gddlmZddlZddlZddlZddlmZddlmZddl Z ddl Z ddl m Z m Z m Z ddl mZddlmZmZmZmZddlmZdd lmZmZmZ dd lmZ dd lmZmZ dd l m!Z!d Z"dZ#dZ$dZ%dZ&dZ'dZ(dZ)dZ*dZ+iZ,e-e Dcic]}e.e/e |e0se/e ||c}Z1GddeZ2GddeZ3GddeZ4GddeZ5ejjle jndejjpe jnd ejjre jnd!ejjte jnd"ejjveejjxeejjzeiZ>ejjpe j~de j#e j~d$e j#fejjle j~d%e j#e j~d&e j#fejjte j~de j#e j~d'e j#fejjre j~d(e j#e j~d)e j#fejjze j~de#e j~d*e#fejjxe j~d+e#e j~d,e#fiZDejjpe j~de j#e j~d-e j#fejjle j~d.e j#e j~d&e j#fejjte j~de j#e j~d/e j#fejjre j~d0e j#e j~d)e j#fiZEejjpe j~de j#e j~d&e j#fejjle j~d1e j#e j~d2e j#fejjte j~de j#e j~d)e j#fejjre j~d3e j#e j~d4e j#fejjze j~de#e j~d,e#fejjxe j~d5e#e j~d6e#fiZFd7d8d9ZGdad:ZHdbd;ZId<ZJ dc ddd=ZK dc ded>ZL da dfd?ZMdgd@ZNdgdAZOdhdBZPdidCZQGdDdEZRGdFdGZSGdHdIZTdJZUdKZVdLZWdMZXdjdNZYdOZZdkdPZ[dldQZ\dmdRZ]dndSZ^dodTZ_dpdUZ`dodVZadpdWZbdqdXZcdrdYZddsdZZedtd[Zfdtd\Zgdud]Zhdud^Zidud_Zjdud`Zky#e$rdZYwxYw#e$rdZdZYwxYw#e$rdZ!YwxYwcc}w)v) annotationsN)Enum)Path) ModelProto TensorProtoexternal_data_helper)onnx_pb) make_graph make_model make_nodemake_tensor_value_info)ReferenceEvaluator)GraphOptimizationLevelInferenceSessionSessionOptions float8e4m3fn)int4uint4)to_array_extendedz onnx.quantizez0.1.0zai.onnxz com.microsoftQuantizeLinear_QuantizeLinear_InputDequantizeLinear_DequantizeLinear_Output _quantizedlc*eZdZdZdZdZedZy)QuantizationModerc|jSNnameselfs Y/var/www/openai/venv/lib/python3.12/site-packages/onnxruntime/quantization/quant_utils.py__str__zQuantizationMode.__str__C yycD t|S#t$r twxYwr )rKeyError ValueError)modes r% from_stringzQuantizationMode.from_stringFs) #D) ) ,   N)__name__ __module__ __qualname__ IntegerOps QLinearOpsr& staticmethodr-r(r%rr?s%JJr(rc*eZdZdZdZdZedZy)QuantizedValueTyperrc|jSr r!r#s r%r&zQuantizedValueType.__str__Rr'r(cD t|S#t$r twxYwr )r7r*r+)vs r%r-zQuantizedValueType.from_stringUs) %a( ( ,  r.N)r/r0r1Input Initializerr&r4r-r5r(r%r7r7Ns% EKr(r7cNeZdZdZdZdZdZdZdZdZ dZ e d Z e d Zy ) QuantTyperrc|jSr r!r#s r%r&zQuantType.__str__fr'r(cD t|S#t$r twxYwr )r>r*r+)ts r%r-zQuantType.from_stringis( Q<  ,  r.c |tjk(rtjS|tjk(rtj S|tj k(rtjS|tjk(rtjS|tjk(rtjS|tjk(rtjS|tjk(rtjSt!d|d)NzUnexpected value qtype=.)r>QInt8rINT8QUInt8UINT8QUInt16UINT16QInt16INT16 QFLOAT8E4M3FN FLOAT8E4M3FNQUInt4UINT4QInt4INT4r+r#s r% tensor_typezQuantType.tensor_typeps 9?? "## # 9## #$$ $ 9$$ $%% % 9## #$$ $ 9** *++ + 9## #$$ $ 9?? "## #24(!<==r(N)r/r0r1rIrKrQrOrMrUrSr&r4r-propertyrWr5r(r%r>r>]sR E FM FG E F >>r(r>c*eZdZdZdZdZedZy) QuantFormatrrc|jSr r!r#s r%r&zQuantFormat.__str__r'r(cD t|S#t$r twxYwr )rZr*r+)formats r%r-zQuantFormat.from_strings) v& & ,  r.N)r/r0r1 QOperatorQDQr&r4r-r5r(r%rZrZs%I Cr(rZint8uint8int16uint16dtypeiiiiiiiii@ii@r@zero_point_indexc@g}t|D]\}}tjt|tjr%|j tj |n=t|tjr|j |ntd|d|||k(s|d}|jtjk(s|jtjk(std|jt|dkDr t|S|dS)Nzarg z is not an array: rmzzero_point cannot be rr) enumeratenumpy issubdtypetypenumberappendarray isinstancendarray TypeErrorrefloat32float16lentuple)roargsnew_argsiar:s r% _check_typersH$1   DGU\\ 2 OOEKKN + 5== ) OOA d1#%7s;< <   Aww%--'177emm+C"7y ABB "(ma/5?@Xa[@r(c|tvs Jd|d|tjjtjjtjj tjj fvr/|dk7rtd|d|jtjk(rtj}nG|jtjk(rtj}ntd|jdtt!t#dgdgt$j&j)d|gdg t#d gd d ggd t+d|dt+d|dgt+d |dg}t-|}t/|j1d||ddSt|} t3|dd\} } | t5| |n| } | t7| |n| } tj8|j;tj|z j=|z}tj>|| | |t/|j;| S)NUnexpected data type > requested. Only INT8, UINT8, INT16, and UINT16 are supported.rz2zero_point is expected to be null for float 8 not rHzUnexpected dtype Constant zero_point)valuer)XscalerYqurr)rrF) reduce_range symmetric)out) ONNX_TYPE_TO_NP_TYPE onnx_protorrRFLOAT8E4M3FNUZ FLOAT8E5M2FLOAT8E5M2FNUZNotImplementedErrorrerrr{FLOATr|FLOAT16r+r r r onnxhelper make_tensorr rrrunget_qmin_qmax_for_qTypemaxminasarrayastyperoundclip)qTypearrrrlowhigh onnx_type onnx_modelrefreqminqmaxcliplowcliphigharr_fp32s r%quantize_nparrayrs- %%e ug%cde% ++--))--  ?%(Z[eZhhi&jk k 99 %#))I YY%-- '#++I0 1=> > "Bdkk>U>UVbdikmpqor>s.0LseT  *3 4@*7ItD(UD9:  !,3774sU)CDQGHH%U+,URWX d$'O#dC.&*&63tT?D==#**U]]";e"C!J!J!Lz!YZ 8WhH=8??5122r(c |dkDs|dkrtd|d|tj|tjd|j}tj |tjd|j}|.t ||tj||jz}|rBtj tj|tj|}| }|}||ks Jd|d|tj||z tj}tj|tjtj|tjz }tj||z } | dk\sJd| tj|jjkrFtjd|j} tjd|j} | | gS|r^tjtj||ztjd tjz |j} n:tjtj||| z z |j} | j|j} | | gS) aCalculate the scale s and zero point z for the quantization relation r = s(q-z), where r are the original values and q are the corresponding quantized values. r and z are calculated such that every value within [rmin,rmax] has an approximate representation within [qmin,qmax]. In addition, qmin <= z <= qmax is enforced. If the symmetric flag is set to True, the interval [rmin,rmax] is symmetrized to [-absmax, +absmax], where absmax = max(abs(rmin), abs(rmax)). :parameter rmin: minimum value of r :parameter rmax: maximum value of r :parameter qmin: minimum value representable by the target quantization data type :parameter qmax: maximum value representable by the target quantization data type :parameter symmetric: True if the floating-point range should be made symmetric. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :return: zero and scale [z, s] rBqmin and qmax must meet requirement: qmin <= 0 <= qmax while qmin:, qmmax:rdzqmin=z > qmax=z scale isse?g@)r+rrminimumrwremaximumrrabsfloat64finfotinyrr) rminrmaxrrrmin_real_rangeabsmaxdrdqrrs r%compute_scale_zprs ( ax4!8]^b]ccklpkqrss ==u{{1DJJ? @D ==u{{1DJJ? @D!4 nDJJ OOPuyy $@ww 4<55htf55< TD[ 6B T /%++d%--2X XB KKR E A:#|#: u{{4::&+++ Ctzz2[[$**5      TD[EKK5==,QQRZ^ZdZdJU[[u 1D%ETZZXJ TZZ(  r(cd}|tvr|tjk(rddlm}ddlm}|}tdDcgc] }|| }}tj|Dcgc]0}tj|rtj|r/|2c}tj}ntd|d|t|<n|tjk(rddlm}|}|td |d tjt|} tjd|} tj|| z |j } | | gScc}wcc}w) arCalculate the scale s for a float8 type (E4M3FN). The function assumes the coefficient distribution and the float 8 distribution are similar to two gaussian laws. :return: zero and scale [z, s] More details in notebook `quantization_fp8.ipynb `_. Nr)float8e4m3_to_float32rrdzQuantization to element_type=z not implemented.zUnexpected element_type rH)FLOAT8_DISTRIBUTIONSrrRonnx.numpy_helperr#onnx.reference.custom_element_typesrrangerrrwisnanisinfr{r+rzstdre) element_typerzp_dtyperrr all_valuesfvaluesstd_f8zerors r%compute_scale_zp_float8r9s*H// ;33 3 ? H#H :math:`[0, 2^{b-1}]` and - when data `type == int8`, from `[-m , m]` -> :math:`[-(2^{b-1}-1), 2^{b-1}-1]` where `m = max(abs(rmin), abs(rmax))` and add necessary intermediate nodes to transform quantized weight to full weight using the equation :math:`r = S(q-z)`, where - *r*: real original value - *q*: quantized value - *S*: scale - *z*: zero point rgz+One of the quantized value is NaN data in [z, z], quantized_data in [z].zUnexpected value for qType=rH)rrrRranyrrrraravelrrrrrJrLrPrNrVrTr+) rrrrrrrrrquantized_datanp_datas r% quantize_datarsZ82  J ((()%ujI %%ekk288:S@SH ImmD)G=gkkm_Bw{{}o^&&4&8&8&:%;2n>P>P>R=SSUW 5.00  *%ujI5.00 25'; <fvrt3d |d tAtC|j-}tjDjG|||j$|d }|StjDjI|}tj||j|j$}tjJjM||}|S)aG Returns a quantized version of the given ONNX initializer. :param weight: The ONNX initializer to quantize. :param quant_type: The final quantized data type. :param zero_point: The zero-point value to use for quantization. :param scale: The scale value to use for quantization. :param axis: The quantization axis if quantizing per-channel. Defaults to None. :param quant_weight_name: The name of the quantized initializer. If not specified, the quantized name is generated. :return: The quantized ONNX initializer. NrzThe initializer of shape z! could not be created, expecting z, got z and shape=z raw=rHzQuantized weights for z. must be 8-bit before packing as 4-bit values.T)rawrd)'tensor_proto_to_arrayrrshapelistrtakervrrrreshape concatenater"TENSOR_NAME_QUANT_SUFFIXrrrR data_typedimsextendflattencopytobytesraw_datarrstrrVrTrer`rabytespack_bytes_to_4bitrrtensor_dtype_to_np_dtype numpy_helper from_array)weightrrraxisquant_weight_name weight_data q_weight_data channel_count channel_dimsquantized_channel_data_listr channel_data channel_scalechannel_zero_pointquantized_channel_data q_weight_nameq_weight_initializercheck packed_dataquant_np_dtypes r%quantize_onnx_initializerr s-((/K*.M |([5F5F5H%Q[\ #))$/ K--.  T&(#}%A&++At4L!!HM!+A %5L..0-AS& " ( . .u}}=S/T/\/\]i/j k&))*EtL ):%6;;-PhOi@jMT%%222#//1)3&!!((5$1!(5(=(=(?(D(D(F(N(N(P%  (&&:;E{{k///5==?mF[F[F]3]"/ 0A0A/BBc$,,.s34F5==?3B;O:PP[\b\h\h[iS!56t<=Q@(  ((--t/?/?/E/EF F   uzz5;;&? ?!7 Ftuv v.}/D/D/FGH  ${{66}jRXR]R]_jpt6u  ==jI m>JRRSYS^S^_ #00;;M=Y r(cj|tjjk(r tdd}|rtj |}n)|r|t vr t |}ntj |}|std|d|\}}|dkDs|dkr'td|d|d|jd |d |d | |S) z Return qmin and qmax, the minimum and maximum value representable by the given qType :parameter qType: onnx.onnx_pb.TensorProto.UINT8 or onnx.onnx_pb.TensorProto.UINT8 :return: qmin, qmax z;This function is not implemented for float 8 as not needed.Nrrrrrz, dtype=z, reduce_range=z , symmetric=z, qType=) rrrRrONNX_INT_TYPE_REDUCED_RANGEgetONNX_INT_TYPE_SYMMETRIC_RANGEONNX_INT_TYPE_RANGEr+re)rrrqrangerrs r%rr(s  &&333!"_`` F,007 u ==.u5$((/ 07uvwwJD$ ax4!86$x |?<.Y" 8E7 4  Mr(c.t|||\}}||z S)z Helper function to get the quantization range for a type. parameter qType: quantization type. return: quantization range. r)r)rrrrrs r%get_qrange_for_qTyperHs ) RJD$ $;r(c:|dkr||zn|}|dk\xr||k}||fS)z Helper function that tries to return a normalized axis in the range [0, rank - 1]. :parameter axis: The axis to normalize. :parameter rank: The tensor rank (number of dimensions). :return (is_valid, axis_norm) rr5)rrank axis_normis_valids r%normalize_axisrRs3 $axt TIA~2)d"2H Y r(ct|}|dk(r tS|dzdz}t|}d}d}||dz kr-||dzdzdz||dzz||<|dz }|dz }||dz kr-||kr ||dz||<|S)aB Copies a source array of 8-bit values into a destination bytearray of packed 4-bit values. Assumes that the source values are already in the appropriate int4 range. :parameter src_8bit: The 8-bit element values to pack. :return A bytearray with every two 8-bit src elements packed into a single byte. rrr?rhrA)r} bytearray)src_8bit num_elemsdst_sizedstsrc_idst_is r%rr^sH IA~{A !#H H CEE )a-  *S0Q68E?S;PQE      )a-   ye_s*E Jr(ceZdZdZggdfdZy)QuantizedInitializerzJ Represents a linearly quantized weight input from ONNX operators Nc ||_||_||_||_||_||_||_||_| |_yr ) r" initializerrminsrmaxs zero_pointsscalesrrr) r$r"r$r%r&r'r(rrrs r%__init__zQuantizedInitializer.__init__sF &  &  , r(r/r0r1__doc__r)r5r(r%r"r"|s r(r"c eZdZdZ ddZy)QuantizedValuezI Represents a linearly quantized value (input\output\intializer) Nc ||_||_||_||_||_||_||_||_| |_yr ) original_nameq_name scale_namezp_name value_typer node_type node_qtype scale_type) r$r"new_quantized_namer1zero_point_namequantized_value_typerr4r5r6s r%r)zQuantizedValue.__init__sD"( $& . "$$r()NNNNr*r5r(r%r-r-s%r(r-ceZdZdZdZy)BiasToQuantizez+ Represents a bias to be quantized c.||_||_||_yr ) bias_name input_name weight_name)r$r=r>r?s r%r)zBiasToQuantize.__init__s"$&r(Nr*r5r(r%r;r;s 'r(r;c|jdk(rtd|jd|jdk(r|j}n#|jdk(r|j}n|jdk(r |j }n|jdk(r |j }n|jdk(r |j}n|jd k(r |j}n|jd k(r |j}nz|jd k(r |j}n^|jd k(r |j}nB|jd k(r |j}n&td|jd|jd|j|iS)z Convert attribute to kwarg format for use with onnx.helper.make_node. :parameter attribute: attribute in AttributeProto format. :return: attribute in {key: value} format. rz attribute z does not have type specified.rr?r@rArBrCri rz has unsupported type rH) rtr+r"rrsrFgfloatsintsstringstensorsgraphs) attributers r%attribute_to_kwargrKs; ~~:inn%55STUU~~  1   1   1   1   1    1  1 !! 1 !! 2   :inn%55KINNK[[\]^^ NNE ""r(ct|Dcgc]}|j|k(s|}}t|dkDr|dSdScc}w)z Helper function to find item by name in a list. parameter item_name: name of the item. parameter item_list: list of items. return: item if found. None otherwise. rN)r"r}) item_name item_listitemitemss r% find_by_namerQsB( Bid499 +ATiE B5zA~58/4/ Cs55cRd}tt|D] }|||k(s |}|S)zC Helper function to return index of an item in a node list rm)rr}) elem_name elem_listelem_idxrs r%get_elem_indexrVs4H 3y> " Q<9 $H# Or(cHtjjd||g|S)z Helper function to create a Mul node. parameter inputs: list of input names. parameter output: output name. parameter name: name of the node. return: Mul node in NodeProto format. Mul)rrr )inputsoutputr"s r% get_mul_noder[s! ;; $ ??r(cl|jj|j|z|jzS)zp Helper function to generate a identifiable filepath by concatenating the given identifier as a suffix. )parentjoinpathstemsuffix)filename identifiers r%generate_identified_filenamercs+ ?? # #HMMJ$>$P QQr(c`ddl}ddlm}ddl}|j|j t dt |t dt ||j||d|jd|jd|jd |jy) Nr) thresholdz Histogram:zHistogram Edges:T)fillz Tensor valueCountszTensor value V.S. Counts) sysmatplotlib.pyplotpyplotrrset_printoptionsmaxsizeprintstairsxlabelylabeltitleshow)hist hist_edgesrhpltrrs r% apply_plotrvs#ES[[1 , $K  *JJtZdJ+JJ~JJxII()HHJr(c B ddlddl}ddlddlmcmcm}ddlmcmcm}ddl m m m tjd|Gfddj}j!||}t#t$j&j)|dd 5}|j+|dddj-d}|j/d } g} t1|j3D]} || } | j5} t7| j9d |j;t7| j9d |j;g}t=t?|}| jA| }| jA|}|jC| |jE| ||jG| ||jI| }| jK||jM| tO| | D]}| jQ|| jS}|jU| |jW| ||jY| }| j[|| j]}t#t$j&j)|d d5}|j+|dddt$j^j9dddvr|jja|d}|jc}te|D]Y}|jg|}tj|jitj|jk[t#t$j&j)|dd 5}t1|j3D]} || } | j5} t7| j9d |j;t7| j9d |j;g}| dzt=t?|z}|j+||j+d dddy#1swYxYw#1swYxYw#1swYyxYw)z> Helper function to write calibration table to files. rN)CalibrationMethod TensorData TensorsDatazcalibration cache: c eZdZfdZy)*write_calibration_table..MyEncodercNt|fr|jSt|jr'|jt |j ddSt|r"|j jt |dSjj||S)Nz numpy.array)rreCLS)r~r) rxto_dictrytolistrre __class__r/ JSONEncoderdefault)r$objrxryrzjsonnps r%rz2write_calibration_table..MyEncoder.default2s# K89{{}$#rzz* # s399~m\\#01"}}55CII##++D#6 6r(N)r/r0r1r)rxryrzrrsr% MyEncoderr|1s  7 7r(r)clszcalibration.jsonwihighestlowestzcalibration.flatbufferswbQUANTIZATION_DEBUG)r1zcalibration.cache  )6r flatbuffersrr5onnxruntime.quantization.CalTableFlatBuffers.KeyValue quantizationCalTableFlatBuffersKeyValue5onnxruntime.quantization.CalTableFlatBuffers.TrtTableTrtTable"onnxruntime.quantization.calibraterxryrzlogginginfordumpsopenospathjoinwriterwBuildersortedkeysrfloatrrOrr CreateString KeyValueStartKeyValueAddKeyKeyValueAddValue KeyValueEndrvTrtTableStartDictVectorr}PrependUOffsetTRelative EndVector TrtTableStartTrtTableAddDict TrtTableEndFinishOutputenvironGetRootAsTrtTable DictLengthrDictKeyValue)calibration_cachedirrrrr json_datafilerbuilderkey_value_listkeyrd_valuesrErflat_key flat_value key_value main_dict cal_tablebufdict_lenrrxryrzrrs @@@@@r%write_calibration_tabler!s LLLL]] LL&'8&9:;77D$$7 ,) ># (,,y$/446 7 (,,x.335 6 CK '',))%0 w'2!!':6((1 i(#0& $$Wc..AB# '' 2$!!#I 7# Wi0$$W-I NN9 .. C bggll3 9:D AT 3 B zz~~*A.(:%%77Q? '')xA!q)I LL ) LL* +! bggll3 34c :d+0023C&s+F~~'Hhll9d388:;hll8T2779:F#ICK 00E JJu  JJt 4 ; :g : 9L B A ; :s%$Q;6R/CR;RRRc|dk(jtj}|dk7jtj}|j}|j|z }|sy|t |zt |z }|dks Jd|||fz|jtj}|||z| |zzz }|dkjdk(sJ|S)a~Given a discrete distribution (may have not been normalized to 1), smooth it by replacing zeros with eps multiplied by a scaling factor and taking the corresponding amount off the non-zero values. Ref: http://web.engr.illinois.edu/~hanj/cs412/bk3/KL-divergence.pdf https://github.com//apache/incubator-mxnet/blob/master/python/mxnet/contrib/quantization.py rNrz"n_zeros=%d, n_nonzeros=%d, eps1=%f)rrrr{sumsizer)pepsis_zeros is_nonzerosn_zeros n_nonzeroseps1rss r%smooth_distributionr}sQu}}-H6//%--0KllnG'!J  w % "3 3D #:; ?: 88EMM "DC(Nte{2 22D AI??  !! ! Kr(ctj|jd}|jjD]}t j |syy)NF)load_external_dataT)rloadas_posixgraphr$ruses_external_data) model_pathmodel intializers r%model_has_external_datarsE IIj))+ FEkk--  2 2: >. r(ct}|j|_tj|_i}dg|d<t |j|fddgi|}y)z Generate model that applies graph optimization (constant folding, etc.) parameter model_path: path to the original onnx model parameter opt_model_path: path to the optimized onnx model :return: optimized onnx model ConstantSharingdisabled_optimizers providersCPUExecutionProviderN)rroptimized_model_filepathrORT_ENABLE_BASICgraph_optimization_levelr)ropt_model_path sess_optionkwargs_s r%optimize_modelrsb!"K+9+B+B+DK(+A+R+RK( F%6$7F !,,. jH^G_jcijAr(cddi}|jr8|jD])}|j|j|ji+tj j ||y)z>Tag the model that it went through quantization pre-processingonnx.quant.pre_processonnxruntime.quantNmetadata_propsupdaterrrrset_model_props)rrprops r%add_pre_process_metadatarsU.0CDN ((D  ! !488TZZ"8 9)KK~6r(c|jr2|jD]#}|jdk(s|jdk(s#yy)zCCheck the model whether it went through quantization pre-processingrrTFrrr)rrs r%model_has_pre_process_metadatars< ((Dxx33 FY8Y) r(cddi}|jr8|jD])}|j|j|ji+tj j ||y)N onnx.inferrr)rrrs r%add_infer_metadatarsU"$78N %%A  ! !155!''"2 3&KK~6r(c|jr2|jD]#}|jdk(s|jdk(s#yy)NrrTFr)rrs r%model_has_infer_metadatars; %%Auu $4G)G& r(ct|d}tjjt |t |tj |j }t||j|S)Nz -inferred) rcrshape_inferenceinfer_shapes_pathrrrrunlink)rinferred_model_pathrs r%load_model_with_shape_inferrs`6z;O**3z?C@SrZrJrerLrPrNrRrVrTrrwrar`rcrbrrr rrrrrrr rrrrr"r-r;rKrQrVr[rcrvrrrrrrrrrrrrrrrrr)ks0r%r8sC #  >>&QQ-PP@ ? 7      ,$2'!47 4Dq4Dq SZ[fhiSjloHp Q'*4Dq  t   #>#>L $  V!4  +%++g"6  +%++g"6!!;5;;x#8''  %  ;5;;q #DkekkRU]b]h]hFi"j+%++d%**"E{u{{SV^c^h^hGi!j!!KEKK$F  TYafamamHn#o  ;5;;vU[[#I;5;;W\didodoKp"q  ;5;;q#>  BV[@\"]+%++b"={u{{1TX?Y!Z   ;5;;q #DkekkRU]b]h]hFi"j+%++d%**"E{u{{SV^c^h^hGi!j!!KEKK$F  TYafamamHn#o  ;5;;vU[[#I;5;;W\didodoKp"q !  ;5;;q #DkekkRU]b]h]hFi"j+%++c"DkekkRT\a\f\fFg!h!!KEKK$F  TYafamamHn#o  ;5;;vU[[#I;5;;W\didodoKp"q  ;5;;q#={u{{1TX?Y"Z+%++b"={u{{1TX?Y!Z )+ A 13h<~"R#'"&"&;G ;G);G;G ;G ! ;G  ;G ;G);G~hl:=7:=D$( L L )L L   L  L " L L ^@ <>%%8''"#J0@R$Yx: k 777+,2-3/QL D E $rsB\7]$] ]# ]#7]] ]]] ]