gddlZddlZddlZddlZddlmZddlmZm Z ddl m Z ddl m Z ddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZddlm Z Gd d eZ!y) N)onnx_pb) BaseQuantizerQuantizationParams) TensorData) ONNXModel)TENSOR_NAME_QUANT_SUFFIXQuantizationModeQuantizedValueQuantizedValueType __producer__ __version__add_infer_metadataattribute_to_kwargcompute_scale_zpcompute_scale_zp_float8 find_by_nameget_qmin_qmax_for_qTypeget_qrange_for_qType ms_domain&save_and_reload_model_with_shape_infertensor_proto_to_array)CreateOpQuantizerceZdZ ddZdZdZdZdZdZdZ d Z dd Z d Z d Z d ZdZddZ ddZdZd dZdZddZ d!dZ d"dZd#dZ d$dZdZdZdZy)% ONNXQuantizerNc <tj|||||||| | | | |s |jjt |jj}|j j D cic]} | j| c} |_|jj|j jDcic]}|j|c}|jj|j jDcic]}|j|c}t||_||_ ||_|jdkD|_d|j"vxr|j"d|_g|_d|_i|_|j*j|j jDcic]}|jdc}|j*j|j jDcic]}|jdc}|jjj j,D]6}|j*j|jDcic]}|dc}8|jt.vrt1d|j|j3|_d|_d|_d|_d |_i|_|jjA|_!ycc} wcc}wcc}wcc}wcc}wcc}w) N MatMulConstBOnly/rzunsupported quantization mode fixed_quantization_range_uint8fixed_quantization_range_int8 fixed_zero fixed_zero_zp)"r__init__modelreplace_gemm_with_matmulrgraph value_infoname value_infosupdateoutputinputrmodestatic opset_versionfuse_dynamic_quant extra_optionsq_matmul_const_b_only new_nodes graph_scope tensor_namesnoder ValueErrorcalculate_quantization_paramsquantization_paramsfixed_qrange_uint8_namefixed_qrange_int8_namefixed_zero_namefixed_zero_zp_namequantized_value_mapget_non_initializer_inputsgenerated_value_names)selfr% per_channel reduce_ranger.r/ weight_qTypeactivation_qType tensors_rangenodes_to_quantizenodes_to_excludeop_types_to_quantizer2viotitr7 output_names \/var/www/openai/venv/lib/python3.12/site-packages/onnxruntime/quantization/onnx_quantizer.pyr$zONNXQuantizer.__init__&s             JJ / / 1:4::;K;KLE6;kk6L6LM6L 6LMD     # #5;;;M;M$N;MRRWWb[;M$N O    # #5;;;L;L$M;LRRWWb[;L$M N"5)DJ  "&"4"4r"9%74;M;M%M%xRVRdRdewRx"   u{{7I7I!J7I"''1*7I!JK   u{{7H7H!I7H"''1*7H!IJJJ$$**//D    $ $ %T k1n %T U0 99, ,=dii[IJ J#'#E#E#G (H$&E#+"1$& &*ZZ%J%J%L"K N$N$M"K!I%Ts$4L=LL 8LL+ L c:tjj|d|jjj}t |t ||j|j|j|j|j|j|j|j|j|j |j" }||_|j&|d|_|j)|jjj*S)z generate submodel for the subgraph, so that we re-utilize current quantization implementation. quantize the submodel update subgraph and set it back to node onnx-quantizer) producer_name opset_importsr)onnxhelper make_modelr% opset_importrrrCrDr.r/rErFrGrHrIrJr2parentr5quantize_modelr')rBsubgraph graph_key warped_model sub_quantizers rOquantize_subgraphzONNXQuantizer.quantize_subgraphos {{-- *****77. <(%        II KK     ! !     " "  ! !  % %      $ '+'7'7&8 1$E !$$&""((...c6|jDcgc]R}|jtjjk(s'|jtjj k(r|T}}t |dk(r|S|jr |jn#|jdt |j}i}|jD]}|jtjjk(r8|j|j|j|d|ji}n|jtjj k(r_g}|jD]?}|j|j||d|jdt |gA|j|i}n t|}|j|tj j"|j|j$|j&fd|ji|Scc}w)z| Check subgraph, if any, quantize it and replace it. return new_nodes added for quantizing subgraph r _node_count_:r)) attributetyperTAttributeProtoGRAPHGRAPHSlenr)op_typer4r^ggraphsextendrr+rU make_noder-r,) rBr7attr graph_attrs node_namekwargskvvaluerZs rOquantize_node_with_sub_graphz*ONNXQuantizer.quantize_node_with_sub_graphs &yyD//555dFYFYF`F`9` &  { q K!%DII4<<. SQUQ_Q_M`La0b NNDyyD//555ii!7!79+Qtyyk@Z![\d11888 $ HLL 22 (#,+Qtyyk3u:, G!,ii''- MM" ##${{$$T\\4::t{{eQUQZQZe^dee7 sAHcVtd|jjDS)zQ Detect if model already has QuantizeLinear or DequantizeLinear. c3\K|]$}|jdk(xs|jdk(&yw)QuantizeLinearDequantizeLinearN)ri).0r7s rO z.ONNXQuantizer.has_QDQ_nodes..s2 _qW[DLL, , R @R0R R_qs*,)anyr%nodes)rBs rO has_QDQ_nodeszONNXQuantizer.has_QDQ_nodess- _c_i_i_o_o_q   r_ct||jjy|j|jj |Sy)NTF)rr% initializerrXfind_initializer_in_path)rBinitializer_names rOrz&ONNXQuantizer.find_initializer_in_pathsC ($***@*@*B C O ;; ";;778HI Ir_c|jj||D].}|jD]}|jj |0yN)r4rlr,rAadd)rBr|r7rNs rO add_new_nodeszONNXQuantizer.add_new_nodess@ e$D#{{ **..{; +r_c|jrtjd|jj D]}|j r|j |}t|j}t||}|jt|t|jD];}|j|jD]}|jj|=|j|jj!j#d|jj!j$j'|j|j(B|jj+\}}t|dkDrt-dt/|zt0|jj_t4|jj_|jjj8Dcgc]}|j:t<k(s|} }| sk|jDcgc]}|j:dk(s|} }| r@|jjj8j}d|_t<|_|jjScc}wcc}w)NzPlease check if the model is already quantized. Note you don't need to quantize a QAT model. OnnxRuntime support to run QAT model directly.r7rz0Invalid model with unknown initializers/tensors.z com.microsoftr) r}loggingwarningr%r|enable_subgraph_quantizationrtrhr4rquantizeranger,rAr_dequantize_outputsr' ClearFieldr7rlrXclean_initializers RuntimeErrorstrr rRrproducer_versionrWdomainrversion) rBr7number_of_existing_new_nodes op_quantizerirN_initializers_not_foundopsetms_opsetms_nodess rOrYzONNXQuantizer.quantize_models      OOn  JJ$$&D0088>+.t~~+> (,T48L  ! ! #7T^^9LM#'>>!#4#;#;K..22;?$<N'   " %%f- &&t~~6 ;; (, (E(E(G %A%)*Q."#UX[\rXs#stt)5 &,7 )'+zz'7'7'D'Db'De XaHaE'Db)-Z4;;/;YHZ ((5599; ! ( zzcZsJ6-J6J;J;cd|jvr3tjd||jd|jdStd|d)NDefaultTensorTypezDget_tensor_type returns DefaultTensorType for tensor name %r, use %dz)Unable to find data type for weight_name=a7. shape_inference failed to return a type probably this node is from a different domain or using an input produced by such an operator. This may happen if you quantize a model already quantized. You may use extra_options `DefaultTensorType` to indicate the default weight type, usually `onnx.TensorProto.FLOAT`.)r2rinforrB tensor_names rO_get_default_tensor_typez&ONNXQuantizer._get_default_tensor_typesf $"4"4 4 LLV""#67  %%&9: :7 GI J  r_ct||jj}| |jS||jvr|j|}|j j drV|r4|j jjdk(r|j|S|j jjS|jr |j|r|j|Sy|jj|}||S|jr+|jr|jj|}||S|r|j|Sy)N tensor_typer)rr%r data_typer*rdHasFieldr elem_typerrrXis_valid_quantize_weightget_tensor_type)rBr mandatoryweightrKotyperess rOrzONNXQuantizer.get_tensor_types*k4::+A+A+CD  ## # $** *!!+.Bww .!4!4!>!>!!C88EEww**44411t{{7J44[AA 44[A  L  , ,++--k:C 00= =r_cH|j|r|j|S||jvr|j|}|jj drU|jj j tjjtjjfvrytjd|d|jdy|jr'|jr|jj|Stjd|dy) NrTz)rBrrrrinput_scale_namereduce_min_namereduce_min_nodereduce_max_namereduce_max_nodereduce_min_abs_namereduce_min_abs_nodereduce_max_abs_namereduce_max_abs_node abs_max_name abs_max_nodeinitializer_divscale_div_namescale_div_nodeinitializer_zps rOrz9ONNXQuantizer._get_dynamic_input_quantization_params_int8Ds&&++&0$|3++//  L t # $  0  /*$|3++//  L t # $  0  /*.6"kk33   # #A & ' 4 ' (    -.-6"kk33   # #A & ' 4 ' (    -.!J. {{,,  ' ' *,?,F,Fq,I J D !   ,'++11  ' '   !% (3 . /   ""?3#k1..    #T%@%@ A      .)001H1H%QSVWUXY "">2!8!8"b@@r_c.tjj}|dz}|dz}|dz}tjj d|g|dzg|d}|j ||dz} tjj d |g| dzg| d} |j | tjj|j|gt|g} |jj| tjj|j|gd g} |jj| |d z} tjj d | jd|jdg| dzg| }|j ||d z}tjj d|jd|jg|g|}|j ||dz}tjj d |j|jdg|dzg|}|j ||dz}tjj d|jd|g|dzg|}|j ||dz}tjj d|j|dzg|}|j ||dz}tjj d|j|g||}|j |||ggfS)a{ Create nodes for dynamic quantization of input to uint8 and add them to nodes_list parameter input_name: Name of the input. parameter nodes_list: new nodes are appended to this list. parameter initial_type: initial weight type (FLAOT or FLOAT16) return: scale_name, zero_point_name, scale_shape, zero_point_shape. r _zero_pointrrrrrrrg _scale_SubSub _scale_Divr_zero_point_Sub_zero_point_Div_zero_point_FloorFloor_zero_point_CastCast)to)rrrrTrUrmrrr;rr%rr=r,)rBrrrrr input_zp_namerrrrinitializer_qrangeinitializer_qvaluescale_sub_namescale_sub_noderr zp_sub_name zp_sub_node zp_div_name zp_div_node zp_floor_name zp_floor_node zp_cast_name zp_cast_nodes rOrz:ONNXQuantizer._get_dynamic_input_quantization_params_uint8s6&&,,%0"]2 $|3++//  L t # $  0  /*$|3++//  L t # $  0  /*"[[44  ( (   !% ( )   ""#56![[44T5I5I(>q(A B d " #    .)#l2..   " "1 %t'C'C D      .)!#44 kk++   ! !?#9#9!#< = 4    +& #44 kk++     "$4 5 4    +&"%88  --g{7I7IM\`L`Kacpq -(!$66 {{,,V]5I5IM?\hmr,s ,'B66r_cH|j}||A|j||jvrtjd|dy|j|}t |t st dt|d|d|t|dk7rtd|d |tj|d g}t|d d r/|d jtjtjfvrtdt|d d |tj|d g}|jtj k7sJ|d}n~tj|g}tj|g}|j|}d |vr |d j}|j#|}|jtj k7sJg} |dz} g} |dz} t$j&j)| || |j+j-} |j.j1| |jtjk(rt2j4j6}nS|jtjk(rt2j4j8}ntd|jd|t$j&j)| || |j;dj-}|j.j1|d| | | | fS)a\ Create initializers and inputs in the graph for zero point and scale of output. Zero point and scale values are obtained from self.quantization_params if specified. parameter param_name: Name of the quantization parameter. return: result, scale_name, zero_point_name, scale_shape, zero_point_shape. z$Quantization parameters for tensor:"z" not specified)FrrrUnexpected type  for rzbQuantization parameters should contain zero point, scale, quant type. Specified values for output z: zero_pointscaledtypez and param_name= quant_typerrzUnexpected dtype=z for param_name=)T)rFr:rr isinstancer TypeErrorrdrhr8nparrayhasattrrfloat32float16float64astyperTrUrraveltolistr%rrrrrreshape)rB param_name use_scale use_zeropointzero_point_typeparamszero_point_values scale_valuesrzero_point_shapezero_point_name scale_shape scale_nameinit_zp scale_type init_scales rO_get_quantization_paramsz&ONNXQuantizer._get_quantization_paramss//   5''/:TE]E]3] CJ<_`,--j9Ff&89"24<.j^ST UVV~V!1 33=,bJ !#&*>)? @ 6'?G4w8M8MVXV`V`bdblblUm8m #3D4I3JJZ[eZh!ijj88VG_$56L%%3 33$\2O "- 9 88YK0L--j9F& w--+2259 %%3 33$}4 (* ++)) _.>@Q@W@W@Y@`@`@b  ""7+    +#//55J   2:: -#//77J01C1C0DDTU_Tbcd d[[,,Z[R^RfRfglRmRtRtRvw  "":.Z+?OOOr_c |j|}|dk7sJd|tz}|dz} | |d||} } } n|j|\} } } } } g}| r'tjj d|| | g|g| }n|j ry|jrN|tjjk(r1|dz} |dz} tjj d |g|| | g| }nU|Jd |d |d |d ||j||||\} } }}tjj d|| | g|g| }t||| | ||j|<g||S)a Given an input for a node (which is not a initializer), this function - add nodes to compute zero point and scale for this input if they don't exist. - add new QuantizeLinear node to quantize the input. :param node: node being quantized in NodeProto format. :param input_index: index of input in node.input. :param qType: type to quantize to. :param given_scale_name: if those inputs need to be quanitzed using this scale tensor. :param given_zp_name: if those inputs to be quantized using this zeropoint tensor. :param initial_type: type of the weight to quantize :return: List of newly created nodes in NodeProto format. rz*Cannot access undefined variable in graph._QuantizeLinearNTrwrrDynamicQuantizeLinearzCCannot quantize input without knowing the initial type, input_name=z, input_index=z, qType=z, node=r)r-r rrTrUrmr/r1rrrrr r?)rBr7 input_indexrgiven_scale_name given_zp_namerrrN ql_node_name data_foundrzp_namerr| qlinear_noderzp_shapes rO_get_quantize_input_nodesz'ONNXQuantizer._get_quantize_input_nodes1s"ZZ , RM!MM #;; !$55  (}/H/35E}G J484Q4QR\4] 1J GQ ;;00 Z1  L{{&&5J4J4J4P4P+P'(2 $}4#{{44+L *g6   $/"",~k](SXRYY`ae`fh/?? ESXgs?t #{{44$W5 M   0>j+Wacjlq/r  ,%% %%r_c||jvr|j|S|j|jj|Syr)r?rXfind_quantized_value)rBrs rOr,z"ONNXQuantizer.find_quantized_valuewsC 11 1++J7 7 ;; ";;33J? ?r_c ||jvr|j|jS|j|j}t||jj }t |}||jvr|j|j}n5||jvr|j|\} }} } } ntd|dt||jj } t | } |j|| ||\} } }}}}||jvsJt|| | |tj|jdkDrdnd||}||j|<| S)z] Quantized the bias. Zero Point == 0 and Scale == Input_Scale * Weight_Scale z Expected z5 to be in quantized value map for static quantizationrrN) node_type node_qtype)r?q_namerrr%rrr:rr8quantize_bias_static_implr r Initializersize)rB bias_namer weight_namebetaweight_scale_nameweight_initializer weight_scalerrinputscale_initializer input_scalequantized_bias_namequantized_bias_scale_namequantized_bias_zp_namebias_scale_datar.r/quantized_values rOquantize_bias_staticz"ONNXQuantizer.quantize_bias_static~s 00 0++I6== =!44[ALL)*;TZZ=S=S=UV,-?@  11 1#77 CNN  433 3+/+H+H+T (AAqy 4ijk k!-.> @V@V@X!Y+,BC   * *9k  +""r_c^||jvxs||jvxs||jvS)zq only check for value info and newly generated tensor names, initializers are checked separately )r*r6rArs rOcontains_tensorzONNXQuantizer.contains_tensors= D,, , ;t000 ;t999 r_c 2|j||dddd|S)NFrr7indicesinitializer_use_weight_qTyperDop_level_per_channelaxis from_subgraph_ONNXQuantizer__quantize_inputs)rBr7rFrJs rOquantize_activationz!ONNXQuantizer.quantize_activations/%%).!&'&  r_c 2|j||d||||S)NTrErK)rBr7rFrDrHrIrJs rOquantize_weightzONNXQuantizer.quantize_weights1%%)-%!5'&  r_c g}g} g} g} |D]4} |j| } | |jvra|j| }|j|j| j|j| j|j | s4| jd|jd| jdt | |jj}||jr=|r;|j|j|r |jn |j||\}}}n/|j||r |jn |j|\}}}| j|| j||j||j| r |jj!| dz|j"|jj%}||j| }||j&vrr|j&|}|j)ds Jd|d|j*j)ds Jd|d|j*j,j.}n(||j0vs Jd|d |j0|}|j3|| |j| }|y |r|j5|n| j7||d }|j8d k(rY| j7|j:|j|jd| j|jdN| j|j:d|j|j:d| j|j:d|j<f|j<j?|| g||||d\}}}}| j|d|j|d| j|dtAd| d|jB| | || fS)a Given a node, this function quantizes the inputs as follows: - If input is an initializer, quantize the initializer data, replace old initializer with new initializer - Else, add QuantizeLinear nodes to perform quantization parameter node: node being quantized in NodeProto format. parameter indices: input indices to quantize. return: (List of quantized input names, List of zero point names used for input quantization, List of scale names used for input quantization, List of new QuantizeLinear nodes created) rrrdz value_info=z has no type.rz is not a tensor.zshape inference failed for zF and attribute 'tensor_names' does not have any value for this tensor.r!)NNNNrrwrrT)rGrDrHrIrJz!Invalid tensor name to quantize: z @graph scope)"r-r?rrr'r0rr%rrCquantize_weight_per_channelr)rErFquantize_initializerrCfind_node_by_namer4r'r*rrdrrr6r*rrlrir,rXrLr8r5)rBr7rFrGrDrHrIrJ scale_nameszero_point_namesquantized_input_namesr|r" node_inputr@r q_weight_namer'rr(rr(rquantize_input_nodesparent_quantized_input_namesparent_zero_point_namesparent_scale_namesrs rO__quantize_inputszONNXQuantizer.__quantize_inputss[.  ""KK0JT555"&":"::"F""?#=#=> ''(?(?@%,,_-C-CD%,,R0""2& ''+&z4::3I3I3KLK&##(< 88#((-I))tOdOd$  %":>9R9R#-I))tOdOd$:6M7J &,,]; ''0"":.%%j1#zz;;!22DNNDJJDTDTDV   '!%K!8J!T%5%55%)%5%5j%A )226:ck*Ub->>9*H+,> (,'8'8'D +/+I+Ik4+@+@|,J,(,37$**+?@ %9:#7#;L''+;;)001D1DE&&|'9'9!'<=$++L,>,>q,AB)001D1DQ1GH&&|':':1'=>$++L,?,?,BC( KK11 M1M!-)="&2 0+&&,,-I!-LM""#5a#89 ''(?(BC!#DZLP]^b^n^n]o!pqqG#J%&6 UJJr_cf|j|jvr<|j|j}|j|j|jfS|j ||||\}}}t |j|||tjd}||j|j<|||fS)a :param weight: TensorProto initializer :param qType: type to quantize to :param keep_float_weight: Whether to quantize the weight. In some cases, we only want to qunatize scale and zero point. If keep_float_weight is False, quantize the weight, or don't quantize the weight. :return: quantized weight name, zero point name, scale name N) r)r?r0r'rquantize_initializer_implr r r2) rBrrrDkeep_float_weightr@rYr'rs rOrSz"ONNXQuantizer.quantize_initializer\s ;;$22 2"66v{{CO&&''**  .2-K-K E<):. * w ) KK     * *   1@  -gz11r_c||jvr2|j|}|j|j|jfS|j |||||\}}} t ||| |t jd}||j|<||| fSr)r?r0r'r quantize_weight_per_channel_implr r r2) rBr5rE channel_axisrDrar@rYr'rs rOrRz)ONNXQuantizer.quantize_weight_per_channel}s $22 2"66{CO&&''**  .2-R-R |\CT. * w )      * *   1@  -gz11r_c||jvrd||jvrU|j|}t|j|jj }|jjj dk7s%|jjj dk(r0|.tjj|jdk(sJ|dz}|jj||j|jj}|H|j|j|jg}tj j#d||g|}|S||j$dk(sJy)a Given a value (input/output) which is quantized, add a DequantizeLinear node to dequantize it back to float32 or float16 parameter value_name: value to dequantize parameter new_nodes_list: List of new nodes created before processing current node return: None if there is already a DequantizeLinear node that dequantizes it A DequantizeLinear node otherwise rQNr_DequantizeLinearrxr)r?rArrr%rrRrT numpy_helperto_arrayr3rTr4r'r0r'rUrmr,)rB value_namer@ scale_init dqlinear_name dqlinear_nodedqlinear_inputsdequantize_nodes rO_dequantize_valuezONNXQuantizer._dequantize_valuesV $22 24KeKe9e"66zBO&o&@&@$**BXBXBZ[Jzz--1AA   ..2BBzG]((11*=BBaGGG&)<>!3DE&**62>>!3DE+// T=Y=YZ .22>5I 4Zlfop d.tT4yRVReRef e/ATY^ku/v  ,/.2#"r_r)F)NN)NNN)g?)FFrF)TFFrF)FF)TF)__name__ __module__ __qualname__r$r^rtr}rrrYrrrrrrrr*r,rArCrMrOrLrSrRrorr9r_rOrr%sFMR/> fD < + Z "22 ARAh\7|9PxaeD&L/#b   ""  .&*" AKF2L 2@%N 7 #r_r)"rnumpyrrTonnx.numpy_helperrrbase_quantizerrr calibrater onnx_modelr quant_utilsr r r r r rrrrrrrrrrrregistryrrrr_rOrsK  &=!!$(K#MK#r_