g ddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z m Z m Z mZddlZddlZddlmZmZmZmZddlZddlmZmZmZdej6d ej6d ej6fd Z d-dej6d ej6d e ed ed ej6f dZGddZ GddZ!GddeZ"GddejFZ$GddZ%Gdde%Z&Gdde%Z'Gdde'Z(Gd d!e'Z)Gd"d#e'Z*Gd$d%ejFZ+Gd&d'e+Z,dd(e"jZd)ifd*ee.efd+e e e.fd,Z/y).N)Enum)Path)DictOptionalSequenceTupleUnion) ModelProto TensorProtohelper numpy_helper) apply_plotload_model_with_shape_infersmooth_distributionpkqkreturnctj|j|j}|ddtj|dd|ddz z|dd|dk(|dk\z}d||<|dkD|dkDz}tj ||<|S)z See https://docs.scipy.org/doc/scipy/reference/generated/scipy.special.rel_entr.html#scipy.special.rel_entr. Python implementation. dtypeNr)npemptyshaperloginf)rrresc2c1s W/var/www/openai/venv/lib/python3.12/site-packages/onnxruntime/quantization/calibrate.pyrel_entrr!s ((288288 ,C URVVBqEBqEM* *CF 'bAg BCG q&R!V BvvCH JbaseaxiscR| |dkDsJd|Jdtj|jtj}d|ztj||dz }tj|jtj}tj ||\}}d|ztj||dz }t ||}tj||}||tj|z}|j|jS)z Simplifeied version of entropy. Source: https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.entropy.html. This avoids taking a dependency on scipy just for this function. rz0base={base} must be a positive number or `None`.z qk is None?T)r$keepdimsr$) rasarrayastypefloat32sumbroadcast_arraysr!rr)rrr#r$vecss r entropyr0's <4!8W%WW # >'<'> B  rzz *B rBFF2D48 8B B  rzz *B  R (FB rBFF2D48 8B 2r C sA  RVVD\ 88BHH r"c`eZdZegdZegdZdZedZedZ dZ y) TensorData)avgstdlowesthighesthist hist_edgesbins)r3r4r5r6r8c t|j|_|jD]\}}|tj vr t d|dtj d|tjvrmt|dst dt|d||jtjtjfvrt d|jd|t|||y)NzUnexpected value z not in .rUnexpected type z for k=zUnexpected dtype )listkeys_attrsitemsr2_allowed ValueError_floatshasattrtyperrfloat16r+setattr)selfkwargskvs r __init__zTensorData.__init__Gs6;;=) LLNDAq +++ #4QE*BUBUAVVW!XYYJ&&&q'*$'7Qyu%MNN772::rzz"::$'8 %NOO D!Q #r"ct|dr t|dstdt|d|j|jfS)Nr5r6z0Attributes 'lowest' and/or 'highest' missing in r;)rDAttributeErrordirr5r6rHs r range_valuezTensorData.range_valueSsFtX&gdI.F #STWX\T]S^^_!`a a T\\**r"ct|dr t|dstdt|d|j|jfS)Nr3r4z)Attributes 'avg' and/or 'std' missing in r;)rDrNrOr3r4rPs r avg_stdzTensorData.avg_stdYsCtU#74+? #LSQUYKWX!YZ Z$((##r"c|jDcic]}|t||}}|jj|d<|Scc}w)NCLS)r?getattr __class____name__)rHrJdatas r to_dictzTensorData.to_dict_sC-1[[9[74##[9nn--U  :sAN) rX __module__ __qualname__ frozensetrArCrLpropertyrQrSrZr"r r2r2CsJZ[HIJG  ++ $$ r"r2c\eZdZdeeeeefffdZdZ dZ dZ dZ dZ dZd Zd Zy ) TensorsDatarYc b||_i|_|jD] \}}t|tst dt |dt|tr|tjk(r/t|dk(r!t|d|d|j|<t|dk(r)t|d|d|d|d |j|<t d |d d t|d |dt|tst dt |d||j|<y)NzKeys must be strings not r;rrr5r6)r5r6r7r9zUnexpected tuple for rz , it has z elements: zValues must be TensorData not ) calibration_methodrYr@ isinstancestr TypeErrorrEtupleCalibrationMethodMinMaxlenr2)rHrhrYrJrKs r rLzTensorsData.__init__gs"4 JJLDAqa%";DG9A FGG!U#%):)A)AAc!fPQk#-QqT1Q4#HDIIaLq6Q;#-QqT1Q4aPQdYZ[\Y]#^DIIaL"7!uIc!fX[YZX[[\ ]^^a,"@a  KLLDIIaL!r"c#8K|jEd{y7wNrYrPs r __iter__zTensorsData.__iter__ys99s c||jvSrqrrrHkeys r __contains__zTensorsData.__contains__|sdiir"c |j|Srqrrrus r __getitem__zTensorsData.__getitem__syy~r"c\||jvrtd|d||j|<y)Nz)Only an existing tensor can be modified, z is not.)rY RuntimeError)rHrvvalues r __setitem__zTensorsData.__setitem__s1 dii !J3'QYZ[ [ #r"c6|jjSrq)rYr>rPs r r>zTensorsData.keyssyy~~r"c6|jjSrq)rYvaluesrPs r rzTensorsData.valuessyy!!r"c6|jjSrq)rYr@rPs r r@zTensorsData.itemssyy  r"cb|jj|j|jd}|S)N)rUrYrh)rWrXrYrh)rHrYs r rZzTensorsData.to_dicts/>>**II"&"9"9   r"N)rXr[r\rrjr r2rrLrsrwryr}r>rr@rZr_r"r rarafsJc5UAR;S6S1T$   "!r"raceZdZdZdZdZdZy)rmrrrcrfN)rXr[r\rnEntropy Percentile Distributionr_r"r rmrms FGJLr"rmcheZdZedZej defdZdZ dZ dZ de de fd Z y ) CalibrationDataReadercXt|dxrt|jxstS)Nget_next)rDcallablerNotImplemented)clssubclasss r __subclasshook__z&CalibrationDataReader.__subclasshook__s%x,L(:K:K1L^P^^r"rct)z9generate the input data dict for ONNXinferenceSession runNotImplementedErrorrPs r rzCalibrationDataReader.get_nexts "!r"c|Srqr_rPs r rszCalibrationDataReader.__iter__s r"c6|j}|t|Srq)r StopIteration)rHresults r __next__zCalibrationDataReader.__next__s >  r"ctrqrrPs r __len__zCalibrationDataReader.__len__!!r" start_index end_indexctrqr)rHrrs r set_rangezCalibrationDataReader.set_rangerr"N)rXr[r\ classmethodrabcabstractmethoddictrrsrrintrr_r"r rrsY__ "$"" ""S"S"r"r) metaclassceZdZ ddeeefdeeefdZdgfdZ dZ de fd Z d Z d Zd efd ZdefdZy)CalibraterBaseN model_pathop_types_to_calibratec"t|trtt||_n,t|trt||_n t d||_||_||_||_ ||_ d|_ d|_ dg|_ y)a :param model_path: ONNX model to calibrate. It should be a model file path :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param symmetric: make range of tensor symmetric (central point is 0). :param use_external_data_format: use external data format to store model which size is >= 2Gb. :param per_channel: whether to compute ranges per each channel. z model_path should be model path.NCPUExecutionProvider)rirjrrmodelrBraugmented_model_path symmetricuse_external_data_format per_channel augment_model infer_sessionexecution_providers)rHrrrrrrs r rLzCalibraterBase.__init__s" j# &4T*5EFDJ  D )4Z@DJ?@ @%:"$8!"(@%&!!$:#; r"rc2||_|jy)zz reset the execution providers to execute the collect_data. It triggers to re-creating inference session. N)rcreate_inference_session)rHrs r set_execution_providersz&CalibraterBase.set_execution_providerss$7  %%'r"ctj}tjj|_tj |j ||j|_y)z9 create an OnnxRuntime InferenceSession. ) sess_options providersN) onnxruntimeSessionOptionsGraphOptimizationLevelORT_DISABLE_ALLgraph_optimization_levelInferenceSessionrrr)rHrs r rz'CalibraterBase.create_inference_sessionsN#113 0;0R0R0b0b -(99  % %%.. r"rc|jjDcic]}|j|}}|j|jjDcic]}|j|c}|j|jj Dcic]}|j|c}|jj Dchc]}|j}}t}tjtjh} |jjD]} |jr| j|jvs(tj| j | jD]a} | |vs|| }|j j#ds)|j j$j&| vsL| |vsQ|j)| c||fScc}wcc}wcc}wcc}w)z select input/output tensors of candidate nodes to calibrate. returns: tensors (set): set of tensor name. value_infos (dict): tensor name to value info. tensor_type)graph value_infonameupdateoutputinput initializersetr FLOATFLOAT16noderop_type itertoolschainrEHasFieldr elem_typeadd) rHrvi value_infosotitinitrtensors_to_calibratetensor_type_to_calibrater tensor_names r select_tensors_to_calibratez*CalibraterBase.select_tensors_to_calibrates.3[[-C-CD-Crrww{-C D%++2D2DE2DBBGGRK2DEF%++2C2CD2CBBGGRK2CDE-2[[-D-DE-DTtyy-D E"u$/$5$5{7J7J#K KK$$D--A[A[1[#,??4::t{{#KK"k1(5GG,,];!#!4!4!>!>BZ!Z!,K!?044[A$L%$[00)EEDEsGGGG#c|jS)zP return: augmented onnx model. Call after calling augment_graph )rrPs r get_augment_modelz CalibraterBase.get_augment_modelszzr"ct)z abstract method: augment the input model to prepare for collecting data. It will: 1. augment the model to be able to collect desired statistics data 2. save augmented model to augmented_model_paths rrPs r augment_graphzCalibraterBase.augment_graphs "!r" data_readerct)z abstract method: collect the tensors that will be used for range computation. It can be called multiple times. r)rHrs r collect_datazCalibraterBase.collect_data "!r"rct)ze abstract method: compute data based on the calibration method stored in TensorsData rrPs r compute_datazCalibraterBase.compute_data"rr")Naugmented_model.onnxFFF)rXr[r\r rjrrrrLrrr rrrrrrarr_r"r rrs:>3!& <#t)$ < ( 6 = 2Gb :param moving_average: compute the moving average of the minimum and maximum values instead of the global minimum and maximum. :param averaging_constant: constant smoothing factor to use when computing the moving average. :param max_intermediate_outputs: maximum number of intermediate outputs before an intermediate range is computed. :param per_channel: whether to compute ranges per each channel. )rrrrrNrrz;Invalid averaging constant, which should not be < 0 or > 1.)superrLintermediate_outputscalibrate_tensors_rangerorrrnum_model_outputsrmodel_original_outputsmoving_averagerBaveraging_constantmax_intermediate_outputs) rHrrrrrrrrrrrWs r rLzMinMaxCalibrater.__init__*s.  "7!5%=#  %'!'+$!$TZZ%5%5%<%.get_op_version^sN % 2 2 99==,*=*=>'///!3!GyPRST Tr"c4d}|dz|z}|dz}tjj||g|g||}tjjd|g|g|}jjj Dcic]}|j |}}|jjjjD cic]} | j | c} |jjjjD cic]} | j | c} ||vr$||jjj} ntd|djr+t||jjj j"} d gt%d | } |jd kr0|j&j)tj*d | nt-t/j0}t3j4t7j8| t6j: |}|jj)|jjj<j)|jjj>jA||gjjjj)tjB|| dgycc}wcc} wcc} w)Nr__Reshape)r'rReshape)inputsoutputsrz'Unable to guess tensor type for tensor zE, running shape inference before quantization may resolve this issue.rrcaxesr)"rr make_noderrrrrrrrErrrBrrordimrange attributeappendmake_attributerjuuiduuid4r from_arrayrarrayint64rrextendmake_tensor_value_info)rreduce_op_namer' reduce_outputintermediate_output reduce_node reshape_noderroi onnx_type tensor_rank reduced_axesreduce_axes_name reduce_axesrreshape_shape_namerHs r add_reduce_min_maxz:MinMaxCalibrater.augment_graph..add_reduce_min_maxdsH(#->M"/*"< ++// 0C/Dx^k0K ;;00+-?@&( 1L261A1A1L1LM1L2277B;1LKM   4::3C3C3J3JK3Ja 3JK L   4::3C3C3I3IJ3Ia 3IJ Kk)' 499EEOO  =k_MZ[ !+k":"?"?"K"K"Q"Q"U"UV !:E![$9: !.$**=B))001F1Fv|1\]'*4::<'8$"."9"9"((JJ$$0077 D JJ   ! ! ( (+|)D E JJ   # # * *6+H+HXadhci+j k3NKJs?L L L ReduceMin ReduceMaxsave_as_external_dataN)rrrjr r r r rr r rrrrsaverr)rHtensorsr reshape_shapertensorrrs` @@r rzMinMaxCalibrater.augment_graphSs 55djjA  .$//"RXX0NPbc  $$++M: U , l\F v{ 3 v{ 3 JJ  % %"&"?"? r"cg|_yrqrrPs r clear_collected_dataz%MinMaxCalibrater.clear_collected_data $&!r"rc |j}|snt|jj|jj d||j 2t |j|j k(r|jt |jdk(r|j td|j}t|tstdt|d|jy)NrNo data is collected.z+compute_data must return a TensorsData not r;)rrrrrunrror(rrBrrirarkrE)rHrrts r rzMinMaxCalibrater.collect_datas ))+F  % % , ,T-?-?-C-CD&-Q R--9112d6S6SS))+ t(( )Q .43O3O3W45 5    ![)I$q'RSTU U !!#r"c>|s|S|jD]\}}t|tr|jd}|jd}n|\}}t||tr%||jd}||jd}n||\}}|jr+||j ||z zz} ||j ||z zz} nt ||} t||} t|tst||trt| | ||<| | f||<|S)Nrrrd)r@rir2rQrrminmax) rH old_range new_rangervr|old_minold_maxnew_minnew_max min_value max_values r merge_rangezMinMaxCalibrater.merge_ranges, #//+JC%,++A.++A.#( )C.*5#C.44Q7#C.44Q7#,S> ""#d&=&=7AR&SS #d&=&=7AR&SS 1 1 %, 9S>:0V!+9i!P #"+Y!7 #3,6r"rc t|jdk(r |jStt|jdDcgc])}|jj |j +}}|jDcgc]}tt||}}i}|D];}|jD]&\}}|j|gj|(=||jd} tdt| dDcgc]}| |jdd} }|Dcic]}||jvs|||} }g} tdt| dD]}|jr>t!j"| | |d} t!j"| | |dzd}n=t!j$| | |d} t!j&| | |dzd}|j(r]t!j&t!j*| t!j*|gd}| jt-| |g| jt-| |gt/t0j2tt| | }|jr-|j5|j||_|jS||_|jScc}wcc}wcc}wcc}w)z Compute the min-max range of tensor :return: dictionary mapping: {added node names: (ReduceMin, ReduceMax) pairs } rNrcrr(r)rorrrr get_outputsrrzipr@ setdefaultrr rpartitionrrrmeanr/r0rabsrlrarmrnr9)rHr output_namesroutput_dicts_listmerged_output_dictdrJrKadded_output_namescalibrate_tensor_namesmerged_added_output_dictpairsmin_value_arraymax_value_arraymax_absolute_valuenew_calibrate_tensors_ranges r rzMinMaxCalibrater.compute_datas t(( )Q .// /JOPSTXTmTmnoTpPqJrsJrQ**668;@@Jr sTXTmTm Tm=PD\#67 8Tm   "A 1"--a4;;A>"#*$*@*@*BC>CAsK]G^`a>b" >b q ! , ,S 1! 4>b " /A$ .@ATMhMhDhA!!$ $.@ !$ q#0115A"""$''*BCUVWCX*Y`a"b"$''*BCUVWZ[V[C\*]de"f"$&&)ABTUVBW)X_`"a"$&&)ABTUVYZUZB[)\cd"e~~%'VVRVVO-Dbff_F],^ef%g" U%7$79K#LMN UO_#EFG6'22C2J2JDQTUkmrQsLt&u#  ' '+/+;+;D3!&!%'A#t)$'A ( 6'ARG R'$(=$(B0,k0,r"rcveZdZ d deeefdeeeffd ZdZ dZ de fdZ de fd ZxZS) HistogramCalibraterrrc t ||||||g|_d|_t |j j j|_|j j jD chc]} | jc} |_ d|_ ||_ ||_ ||_| |_d|_| |_ycc} w)a= :param model_path: ONNX model to calibrate. It is a model path. :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param use_external_data_format: use external data format to store model which size is >= 2Gb :param method: A string. One of ['entropy', 'percentile']. :param symmetric: make range of tensor symmetric (central point is 0). :param num_bins: number of bins to create a new histogram for collecting tensor values. :param num_quantized_bins: number of quantized bins. Default 128. :param percentile: A float number between [0, 100]. Default 99.99. :param scenario: see :class:`DistributionCalibrater` )rrrrN)rrLrrrorrrrrr collectormethodnum_binsnum_quantized_bins percentilerscenario) rHrrrrrTrrUrVrWrXrrWs r rLzHistogramCalibrater.__init__s2  "7!5%=  %'!'+$!$TZZ%5%5%<%9K9K9V9V9XY9XX8==9XY6:6H6H6T6T6VW6V( 6V W ))+F((,,T6:GM(1'(:$ f -@!((6):;!((0 );  % % , ,] ;" t(( )Q .45 5UYTmTm Tm=PD\#67 8Tm   "A 1&&q"-44Q7"#9Df 1qDLeLeGeQ A. f~~/{{..#'#:#:?? DN 01 !!#[ZW, gsH>I2II *I rcn|js tdt|trtj }nZt|t rtj}n9t|trtj}ntdt|dt||jjS)z Compute the min-max range of tensor :return: dictionary mapping: {tensor name: (min value, max value)} z9No collector created and can't generate calibration data.zUnknown calibrater z". This method must be overwritten.)rSrBriEntropyCalibraterrmrPercentileCalibraterrDistributionCalibraterrrkrEracompute_collection_result)rHcals r rz HistogramCalibrater.compute_datazs ~~XY Y d- .#++C 2 3#..C 4 5#00C1$t*=_`a a3 H H JKKr") NrFrWF-X@same)rXr[r\r rjrrrrLrr(rrrarrNrOs@r rQrQsp:>3!&*!#t)$*! ( 6*!X '1$(=1$fLkLr"rQcNeZdZ ddeeefdeeeffd ZxZ S)rhrrc 4t |||||||||y)a :param model_path: ONNX model to calibrate. It is a model path :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param use_external_data_format: use external data format to store model which size is >= 2Gb :param method: A string. One of ['entropy', 'percentile', 'distribution']. :param symmetric: make range of tensor symmetric (central point is 0). :param num_bins: number of bins to create a new histogram for collecting tensor values. :param num_quantized_bins: number of quantized bins. Default 128. )rTrrUrVNrrL) rHrrrrrTrrUrVrWs r rLzEntropyCalibrater.__init__s/*   ! $1  r")NrFr0Frmrm rXr[r\r rjrrrrLrNrOs@r rhrhsH:>3!& #t)$  ( 6  r"rhcNeZdZ ddeeefdeeeffd ZxZ S)rirrc 4t |||||||||y)a :param model_path: ONNX model to calibrate. It is a model path :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param use_external_data_format: use external data format to store model which size is >= 2Gb :param method: A string. One of ['entropy', 'percentile', 'distribution']. :param symmetric: make range of tensor symmetric (central point is 0). :param num_quantized_bins: number of quantized bins. Default 128. :param percentile: A float number between [0, 100]. Default 99.99. )rTrrUrWNrs) rHrrrrrTrrUrWrWs r rLzPercentileCalibrater.__init__s/*   ! $!  r")NrFrWFrnrortrOs@r ririsH:>3!& #t)$  ( 6  r"ricLeZdZ ddeeefdeeeffd ZxZ S)rjrrc 2t||||||||y)a :param model_path: ONNX model to calibrate. It is a model path :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors. :param augmented_model_path: save augmented model to this path. :param use_external_data_format: use external data format to store model which size is >= 2Gb :param method: A string. One of ['entropy', 'percentile', 'distribution']. :param symmetric: make range of tensor symmetric (central point is 0). :param num_bins: number of bins to create a new histogram for collecting tensor values. :param scenario: for float 8 only, if `scenario="same"`, the algorithm weights and float 8 follow the same distribution, if `scenario="p3"`, it assumes the weights follow a gaussian law and float 8 ~ X^3 where X is a gaussian law )rTrUrXNrs) rHrrrrrTrUrXrWs r rLzDistributionCalibrater.__init__s,.   ! $  r")NrF distributionrmrprtrOs@r rjrjsE:>3!& #t)$  ( 6  r"rjcXeZdZdZej dZej dZy)CalibrationDataCollectorzL Base class for collecting data for calibration-based quantization. ct)z Generate informative data based on given data. name_to_arr : dict tensor name to NDArray data rrH name_to_arrs r r`z CalibrationDataCollector.collects "!r"ct)z? Get the optimal result among collection data. rrPs r rkz2CalibrationDataCollector.compute_collection_results "!r"N)rXr[r\__doc__rrr`rkr_r"r r{r{s; "" ""r"r{cdeZdZdZdZdZdZdZdZdZ dZ d Z d Z e dd Zd Zd Zy)r_a` Collecting histogram for each tensor. Percentile and Entropy method are supported. ref: https://github.com//apache/incubator-mxnet/blob/master/python/mxnet/contrib/quantization.py ref: https://docs.nvidia.com/deeplearning/tensorrt/pytorch-quantization-toolkit/docs/_modules/ pytorch_quantization/calib/histogram.html cfi|_||_||_||_||_||_||_yrq)histogram_dictrTrrUrVrWrX)rHrTrrUrVrWrXs r rLzHistogramCollector.__init__s5  "  "4$  r"c|jSrq)rrPs r get_histogram_dictz%HistogramCollector.get_histogram_dicts"""r"ctd|jdvr|j|S|jdk(r.|jr|j |S|j|St d)Nz/Collecting tensor data and making histogram ...>r0ryrWDOnly 'entropy', 'percentile' or 'distribution' methods are supported)printrT collect_valuercollect_absolute_valuerBr}s r r`zHistogramCollector.collectsl ?@ ;;5 5%%k2 2 [[L (~~22;??))+66cd dr"c|jD]D\}}t|trz|D]2}t|tjrJdt |d|t d|D}t|dk(s Jd|d|tj|}n6t|tjstdt |d||}|j}|jdkDr+tj|}tj|}nBtjd|j}tjd|j}tj |}||j"vrxtj$||j&\} } | j)|j} |jtj*k7sJd | | ||f|j"|<|j"|} | d } | d } t-| d sJd t | t-| d sJd t | | d}| d}tj|}||dkDrB|d|dz }tj.|d|z||z|}tj0||f}tj$||\} } | j)|j} | dt|xxx|z ccc|jtj*k7sJd | | t| |t| |f|j"|<Gy)z5 Collect histogram on absolute value r<z for tensor=c34K|]}|jywrqr).0as r z.6s7hQWWhsrz6The calibration expects only one element type but got rr)r9zMonly float32 or float16 is supported, every constant must be explicitly typedrcrfrz'old_min should be a numpy array but is rN)r@rir=rndarrayrError)rBflattensizer/r0r rabsoluter histogramrUr*float64rDarangehstack)rHr~r%data_arrarrdtypes data_arr_npr7r8r7r8 old_histogramr3r4old_histold_hist_edges temp_amaxwidth new_bin_edgess r rz)HistogramCollector.collect_absolute_value.s@!, 1 1 3 FH(D)#C%c2::6l:J4PS9+Uabhak8ll6$7h77K1$kKF8S_`f_ijk$ jj2 "**5 #3DN3C7G] M ),,xX ~WdFefKHa8#GW%GW%  !#%<<#h-Q^P^`mOn#o j "8} .= &)==+HZ*WZ[*[&\#+a2E.EE 3j @= P #%<<,P]~_lNm#n j(<:M+MNRZZNGW%GW%  r"cb|jrt|jdk(r tdtd|jd|jdk(r|j S|jdk(r|j S|jdk(r|jStd) Nrz=Histogram has not been collected. Please run collect() first.z0Finding optimal threshold for each tensor using z algorithm ...r0rWryr)rrorBrrTcompute_entropycompute_percentilecompute_distributionrPs r rkz,HistogramCollector.compute_collection_results""c$*=*=&>!&C\] ] @~^_ ;;) #'') ) [[L (**, , [[N *,,. .cd dr"c|jdks|jdkDr td|j}|j}i}tdt |td|j tdd|z d|d |j D]\}}|d}|d }|j}tj||z } |jrftj| |dz } tj|| |j  tj|| |j f||<nd|z d z } tj| d | z } tj| | } tj|| |j tj|| |j f||<|d} |d}||d| kr | ||d f||<||d |kDr ||d|f||<g|||dd||<tjj!dddvst#|||S)Nrdzr"c|j}|j}i}tdt|td|jdtd|j|j D]^\}}|j ||}|||<g||dd||<tjjdddvsMt|d|d `|S) Nrrz: (The number may increase depends on the data it collects)zNumber of quantized bins : rcrrrr) rrVrrorUr@get_entropy_thresholdrrrr)rHrrVrr%roptimal_thresholds r rz"HistogramCollector.compute_entropys,,!44 $S%8$9:; +DMM?:tuv +D,C,C+DEF!/!5!5!7 FI $ : :9FX Y &7OF #&J(9&JIbqM&JOF #zz~~2A6(B9Q<16"8r"c|dkrtd|d|dd|ddzdz}|dk(r||zj|jz }||dzzj|jz |dzz dz}tj||jtj||jfSt ||k(rt |dzdk(r|||zzj|jz }|||z|z dzzj|jz dz}tj||jtj||jfStj ||z }d|tj|<d|tj|<tj ||z|z}||zj|jz }||dzzj|jz |dzz dz}tj||jtj||jfS) Nrzpower=z <= 0 is invalid.rrg?rcr) rBr,rr rrr@isnanisinf)r7r8powerrr3r4facts r _avg_stdzHistogramCollector._avg_stds A:veW,=>? ?Sb/JqrN2c9 A:&=%%'$((*4C619$))+dhhj836AcIC88Cz'7'78"((3jN^N^:__ _ u: 3u:>Q#6&%-',,.;CFEMC/A55::>RZZ/ //>>RZZ/ //##rzz1 11&0%!~~'"( 'OF #zz~~2A6(B4,3"86r"c |d}|d}|j}|dz}|dz}|dj}tj||z dz} t | jD cgc]0} tj d|tj d|f2} } t ||dzdD]} || z } t || zdz|} || || f| | |z <tj|| | }|j}t|d| }t|| d}|dxx|z cc<|dxx|z cc<|dk7jtj}tj|tj}|j|z}t |D]}||z}||z}t|||||<|dxxt|||zdz cc<tj|jtj}t |D]+}||z}||z}t|||}|dk7s!|||z |||-t|}t|}||&tj tj|}n!tj t|||}|| | |z <tj| }| |}|d}|d}|d|kr||df}|d|kDr|d|f}t!|ddsJt!|ddsJ|Scc} w) aFGiven a dataset, find the optimal threshold for quantizing it. The reference distribution is `q`, and the candidate distribution is `p`. `q` is a truncated version of the original distribution. Ref: http://on-demand.gputechconf.com/gtc/2017/presentation/s7310-8-bit-inference-with-tensorrt.pdf rrrcrNrrfr)rrrzerosrr r/r^deepcopyr,r*r rrr0argminrD)rHrrVr7r8rUzero_bin_indexnum_half_quantized_binr kl_divergencer thresholdsrrsliced_distributionpleft_outliers_countright_outliers_countnonzerosquantized_binsnum_merged_binsindexstartendqnormdivmin_kl_divergence_idxrr7r8s r rz(HistogramCollector.get_entropy_threshold7sh |q\ 99!Q!3q!8! ""2H!H1!LM TYZgZlZlTmnTmqrxx/!51IJTm n -~/A1EA(1,KNQ.2H=I6@6MzZcOd5eJq11 2"&--[0K"L $((*A"%drs  99 >>UU    "!    5/   ZZ 8  F11h"ckk"4k"k"\[,~[,|CL.CLL + D . D  0  F"",E!1E!T 6:/&--" KK d KK#HSM2KKr"