gddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z m Z m Z m Z ddlZddlZddlZddlmZddlmZddlmZddlmZddlmZdd lmZdd lmZdd lm Z m!Z!m"Z"m#Z#ejHe%Z&gd Z'ejPd ejRdejTdiZ+Gdde"Z,Gdde#Z-Gdde"Z.Gdde!Z/Gdde!Z0e/ddfe0ddfe.ddfdZ1Gdd Z2Gd!d"Z3y)#N)Path)DictListTupleUnion) Precision)float_to_float16_max_diff) FusionOptions)IOBindingHelper) OnnxModel)optimize_model)torch_onnx_export) GPT2ConfigGPT2LMHeadModel GPT2Model TFGPT2Model) distilgpt2gpt2z gpt2-mediumz gpt2-largezgpt2-xlMb@?g?g@c,eZdZdZfdZfdZxZS)GPT2ModelNoPastState2Here we wrap a class to disable past state output.c$t||yNsuper__init__selfconfig __class__s e/var/www/openai/venv/lib/python3.12/site-packages/onnxruntime/transformers/models/gpt2/gpt2_helper.pyrzGPT2ModelNoPastState.__init__+  c(t||ddS)NF) use_cache return_dict)rforwardr input_idsr!s r"r(zGPT2ModelNoPastState.forward.swyEuMMr$__name__ __module__ __qualname____doc__rr( __classcell__r!s@r"rr(s<!NNr$rc,eZdZdZfdZfdZxZS)TFGPT2ModelNoPastStaterc2d|_t| |y)NF)r&rrrs r"rzTFGPT2ModelNoPastState.__init__5s   r$c&t||dS)NF)r&)rcallr)s r"r(zTFGPT2ModelNoPastState.forward9sw|I|77r$r+r1s@r"r3r32s<!88r$r3c<eZdZdZfdZedZfdZxZS) MyGPT2ModelzMHere we wrap a class for Onnx model conversion for GPT2Model with past state.c$t||yrrrs r"rzMyGPT2Model.__init__@r#r$c t|ddttfrt|d|k(rt|dddk(sJg}t |D]Z}|j t j|d|djd|d|djdfd\|dt|fS|S)Nr)dim) isinstancetuplelistlenrangeappendtorchcat unsqueeze)result num_layerpresentis r" post_processzMyGPT2Model.post_processCs fQilUDM 2vay>Y.3vay|3D3I IIG9%II1a2215vay|A7P7PQR7ST&1IuW~. . r$ct|||||d}tj||jj SNF) position_idsattention_maskpast_key_valuesr'rr(r8rKr n_layerrr*rNrOpastrGr!s r"r(zMyGPT2Model.forwardVsD %)  ! '' 0C0CDDr$) r,r-r.r/r staticmethodrKr(r0r1s@r"r8r8=s+W!$EEr$r8c,eZdZdZfdZfdZxZS)MyGPT2LMHeadModelzSHere we wrap a class for Onnx model conversion for GPT2LMHeadModel with past state.c$t||yrrrs r"rzMyGPT2LMHeadModel.__init__dr#r$ct|||||d}tj||jj SrMrQrSs r"r(zMyGPT2LMHeadModel.forwardgsD %)  ! '' 0C0CDDr$r+r1s@r"rWrWas]! E Er$rWc,eZdZdZfdZfdZxZS)MyGPT2LMHeadModel_NoPaddingaHere we wrap a class for Onnx model conversion for GPT2LMHeadModel with past state and no padding. When you always use batch_size=1 in inference, there is no padding in inputs. In such case, position_ids and attention_mask need no be in inputs. c$t||yrrrs r"rz$MyGPT2LMHeadModel_NoPadding.__init__yr#r$c|t|||d}tj||jj S)NF)rPr'rQ)rr*rTrGr!s r"r(z#MyGPT2LMHeadModel_NoPadding.forward|s4DeT'' 0C0CDDr$r+r1s@r"r[r[ss !EEr$r[logitsTF last_state)rGPT2LMHeadModel_NoPaddingrc0eZdZdZdefdZdefdZdZy) Gpt2Inputsc<||_||_||_||_yr)r*rNrOrT)rr*rNrOrTs r"rzGpt2Inputs.__init__s +4.:\jLP r$returnc|j|j|jfDcgc]}|| }}|jr|j |j|Scc}wr)r*rNrOrTextend)rv input_lists r"to_listzGpt2Inputs.to_listsZ"&..$2C2CTEXEX!Yk!YA]^]ja!Y k 99   dii ( ls AAc~td|j|j|j|jfDS)Nc3&K|] }|| ywr).0rgs r" z&Gpt2Inputs.to_tuple..su c1ghgtQ cs)r?r*rNrOrT)rs r"to_tuplezGpt2Inputs.to_tuples3u1B1BDDWDWY]YbYb cuuur$cd}|j]|jjtjk(r*|jj tj n |j}|j Dcgc]"}|j tj $}}t|j|j||Scc}w)N)dtype) rOrqrDfloat16tofloat32rTrbr*rN)rrOprTs r"to_fp32zGpt2Inputs.to_fp32s    *''-->##&&U]]&;((  4899=9a5==)9=$..$*;*;^TRR>s:'CN) r,r-r.rrrirrorvrlr$r"rbrbs(Q v%v Sr$rbc"peZdZdZedddej ej ej dfdededededed ed ed ejd e d e de dejdejdejde de f dZ e dAdededede dedeeeeff dZedZedBdZedBdZedCdZedDdZeddddej ej ej fdede d e d e de dejdejdejfd!Ze dEd#Zegd$fd%ed&eefd'ZedFd(e d)efd*ZedFd(e d)efd+Zed,ZedGd-Ze dHd(e d.eeej>fd/eeeefd)ed0e d1e f d2Z ed3Z!ed4Z"edd5d5d6d7ddddej ej ej d"ddfd8Z#edd9ddddej ej ej d:d7d;f d<Z$edId=Z%edddgd>fdefd?Z&y@)J Gpt2HelperzEA helper class for Gpt2 model conversion, inference and verification.FT batch_sizepast_sequence_lengthsequence_lengthnum_attention_heads hidden_sizerH vocab_sizedevicerrhas_position_idshas_attention_maskinput_ids_dtypeposition_ids_dtypeattention_mask_dtypeleft_side_paddingrdc|rtjntj}d|||t||z g}t |Dcgc] }tj |||dzdz "}}tj d|dz ||f| |}d}| re||z}tj||g| |}|dk\rAt |D]3}tj d|dz }|r d||d|f<(d||||z df<5d}| rQ|jjd dz }|j|dkd|dd|dfj| }t||||Scc}w) zCreate random inputs for GPT2 model. Returns torch tensors of input_ids, position_ids, attention_mask and a list of past state tensors. r<rqrg@?rr;)lowhighsizerqrN)rDrrrtintrBrandrandintonesrandomlongcumsum masked_fill_rsrb)ryrzr{r|r}rHr~rrrrrrrrr float_type past_shape_rTr*rOtotal_sequence_lengthrJpadding_lengthrNs r"get_dummy_inputszGpt2Helper.get_dummy_inputss*'.U]]5==    11 2  `een_op_oZ[JjH3NQTT_opMMao.!   $8?$J !"ZZ23*N %)z*A%+^^A7Lq7P%QN(=>q/>/'9:VWq*?.*P*R'RS + )..077;a?L  % %lQ&6 :'+?+@(@ADDEWXL)\>4HHCqs%Err model_classc |j}|j}|j}|j}t|d} ||| dk(r|n|g} d||||zt ||z g} | | i} t |D]} | | dt| z<| S)zAReturns a dictionary with output name as key, and shape as value.r;r^r<present_)r|r}num_hidden_layersr~ MODEL_CLASSESrrBstr)ryrzr{r rr|r}rHr~ output_namelast_state_shapepresent_state_shape output_shapesrJs r"get_output_shapeszGpt2Helper.get_output_shapess%88(( ,, && #K03   %1J{     ? 2  11 2  %&67 y!A1DM*s1v- ."r$c|D]|}||vsJ||}tj|||jkDs8tjtj|||j |j ||<~y)Nr)numpyprodnelementrDemptyrqr)output_buffersrkeybuffers r"auto_increase_buffer_sizez$Gpt2Helper.auto_increase_buffer_sizess C.( ((#C(Fzz-,-0AA&+kkJJ}S12 ,,!=='s# !r$c|rtjntj}i}|jD]3\}}tjt j |||||<5|S)zpReturns a dictionary of output name as key, and 1D tensor as value. The tensor has enough space for given shape.r)rDrrrtitemsrrr)rr is_float16 data_typernameshapes r"get_output_bufferszGpt2Helper.get_output_bufferssX&0EMMU]] (..0KD%#(;;uzz%/@ Z`#aN4 1r$c|djj}tj||dz }|r.tj|tj|dzz Stj|S)zGReturns the maximum difference between PyTorch and OnnxRuntime outputs.rư>)cpurabsamax) torch_outputs ort_outputsrelativeexpected_outputsdiffs r" diff_outputszGpt2Helper.diff_outputs&sn)+//1779yy)KN:; ::deii0@&AD&HIJ J::d# #r$c tj|d|djj||}tj d||}t |dz }t |D]g}tj|d|z|d|jj||}tj d|d|d||xr|}i|s/tj||} tjd| d |S) zReturns True if torch and ORT outputs are close for given thresholds, and False otherwise. Note: need kwargs since Gpt2BeamSearchHelper.compare_outputs has an extra parameter model_class r)rtolatolz9PyTorch and OnnxRuntime output 0 (last_state) are close: r;zPyTorch and OnnxRuntime layer z state (present_z ) are close:z@PyTorch and OnnxRuntime results are not all close: max_abs_diff=.5f) rallcloserloggerdebugrArBrxrinfo) rrrrkwargsis_close is_all_close num_layerslayer max_abs_diffs r"compare_outputszGpt2Helper.compare_outputs0s >>+a.-2B2F2F2H2N2N2PW[bfg PQYPZ[\ %) :&E~~AI&a '++-335 H LL9%@PQVPWWcdlcmn o'4HL'%22=+NL KKZ[ghkZlm nr$cd}d}g}g}tt|D]}||}|dk(r|dn |d|dz jj} tj|| |d} |j tj tj| |z |xr| }tj| jrtjd|dtj| jrtjd|dtj|jrtjd |dtj|jrtjd |dtj|| z } tj| j| j } |j d | | d d | d || d dt#| | d |dk(stjtj|d|j } tjtj| d| j }tj$| |}|j't)|}|t)||||fS)aCompare outputs from PyTorch and OnnxRuntime Args: torch_outputs (Tuple[Torch.Tensor]): PyTorch model output ort_outputs (List[numpy.ndarray]): OnnxRuntime output atol (float, optional): Absolute tollerance. Defaults to 1e-06. Returns: is_all_close(bool): whether all elements are close. max_abs_diff(float): maximum absolute difference. messages(str): a list of debug message for each output TFrr;)rrzPyTorch output z has nanz has infz ORT output zdiff=z.9fz index=z ort=z torch=N)axis)rBrArrrrCrrisnananyrrisinffabs unravel_indexargmaxrfloat array_equalindexmax)rrrris_top1_matched max_diffsmessagesrJ ort_output torch_outputrridx ort_max_indextorch_max_indexmax_diff_output_indexs r"compare_outputs_v2zGpt2Helper.compare_outputs_v2Ks`  s;'(A$QJ01QM!,M!f\EVEVWXEYE_E_D``hipqrisiyiyhz{DELMNEOPQEREXEXDY Z  _$$**4$*G #,,.,')ww||L+'N$)*1177t7T!z**"& +!-!-"$(,-1# (L$ L14 L6%BL;$L.;Mrc td} t|d||d| d} |r5|rtj| nd|vrd|d<| jdddi|| j ||| S) zHOptimize ONNX model with an option to convert it to use mixed precision.rrF) model_type num_headsr} opt_leveloptimization_optionsuse_gpu keep_io_typesuse_symbolic_shape_inferTrl)r r rxauto_mixed_precisionconvert_float_to_float16save_model_to_file) roptimized_model_pathrr|r}rrstagerrms r" optimize_onnxzGpt2Helper.optimize_onnxs -V4  )#!5  #//2"&0.3F?+***SDSFS 13KLr$)AddLayerNormalizationSkipLayerNormalizationFastGeluEmbedLayerNormalization onnx_model op_block_listcn|jDchc]}|j}}t|}|j|}tj d|d||j jdj}d}|j}||vsJ||}d} |jdk(r|} tj d|jd} |jD]} |j| } | nt| } tjd|jd | | d k}n/tjd |jd |jg} g}|s| |g} | jg}| |||d }tj d||jdddi||Scc}w)a?Convert GPT-2 model to mixed precision. It detects whether original model has fp16 weights, and set parameters for float16 conversion automatically. Args: onnx_model (OnnxModel): optimized ONNX model op_block_list (List[str], optional): operators to compute in fp32. Defaults to ["Add", "LayerNormalization", "SkipLayerNormalization", "FastGelu", "EmbedLayerNormalization"] Returns: parameters(dict): a dictionary of parameters used in float16 conversion z fp32 op: z fp16 op: rFNMatMulz#Found last MatMul node for logits: z3max diff of converting weights in last MatMul node : rz-Failed to find MatMul node for logits. Found z of node )rr%node_block_listforce_fp16_initializersz!auto_mixed_precision parameters: rTrl)nodesop_typeset differencerrgraphoutputroutput_name_to_nodeinputget_initializerr rwarningr)r$r%node op_full_set fp32_op_set fp16_op_setlogits_output_nameis_weight_fp16_precisionr1last_matmul_node initializerr2max_diffrr) parameterss r"rzGpt2Helper.auto_mixed_precisions(1;0@0@0BC0Bt||0B C-( !,,[9  i }J{mDE(--/66q9>>$) (<<>!%8888"#56 <<8 ##  KK=dii[I JK(88? *$1=H LLNtyykY[\d[ef g'/$ $ NNJ4<<.Xabfbkbkalm n (/?/K/0M/445O+*.'?    7 |DE+ ++XTXZXaDsF2inputs total_runsc`tjd|jj}t j 5||}ddd|dk(rSg}t j 5t |D]A}tj}||}|jtj|z C dddt|dzt|z }tjdjt|d|fS#1swYxYw#1swYaxYw)zfRun inference of PyTorch model, and returns average latency in ms when total_runs > 0 besides outputs.zstart pytorch_inferenceNrzPyTorch inference time = {} ms.2f) rrrvrirDrrBtimerCsumrAformat) rr?r@rhr latencyrstartaverage_latencys r"pytorch_inferencezGpt2Helper.pytorch_inferencecs  ./^^%--/ ]]_Z(G ?N ]]_:& ,tyy{U23' g,-G < 5<|j>tj|jj j|d<|j>tj|jj j|d<|jd|}|dk(r|Sg}t|D]N}tj} |jd|}|jtj| z Pt|dzt|z } tjd j!t!| d || fS) zcRun inference of ONNX model, and returns average latency in ms when total_runs > 0 besides outputs.zstart onnxruntime_inferencer*NrrOrNrrBz"OnnxRuntime Inference time = {} msrC)rrrascontiguousarrayr*rrT enumeraterOrNrunrBrDrCrErArF) ort_sessionr?r@ ort_inputsrJpast_irrGrrHrIs r"onnxruntime_inferencez Gpt2Helper.onnxruntime_inference}s  23!5#:#:6;K;K;O;O;Q;W;W;Y#Z[ ;; "&v{{3 6*/*A*A&**,BTBTBV*W U1#;'4  ,+0+B+B6CXCXC\C\C^CdCdCf+gJ' (    *).)@)@ATATAXAXAZA`A`Ab)cJ~ &!oodJ7 ? z"AIIKE%//$ ;K NN499;. /# g,-G < 9@@Y^A_`aO++r$c 8tj|||||||S)z)Returnas IO binding object for a session.)r prepare_io_binding)rOr*rNrOrTrrs r"rTzGpt2Helper.prepare_io_bindings,11         r$c2tj||||S)z3Copy results to cpu. Returns a list of numpy array.)r "get_outputs_from_io_binding_buffer)rOrr return_numpys r"rVz-Gpt2Helper.get_outputs_from_io_binding_buffers AA    r$rrrWinclude_copy_output_latencyc htjdtj||j|j |j |j||}|j|tj||||}|dk(r|Sg} t|D]g} tj} |j||rtj||||} | jtj| z it| dzt| z } tjd| || fS)zUInference with IO binding. Returns outputs, and optional latency when total_runs > 0.z*start onnxruntime_inference_with_binded_iorrBz4OnnxRuntime with IO binding inference time = %.2f ms)rrrxrTr*rNrOrTrun_with_iobindingrVrBrDrCrErA) rOr?rrr@rWrX io_bindingrrGrrHrIs r"$onnxruntime_inference_with_binded_ioz/Gpt2Helper.onnxruntime_inference_with_binded_ios  AB 22         ! ! KK    &&z2!CC    ? z"AIIKE  * *: 6*AA  NN499;. /#g,-G < K_]O++r$cTtd|dd5}tj||dddtj d|dtd|dd5}tj||dddtj d|dy#1swYkxYw#1swY/xYw)N ort_outputs_.picklewbz$ORT output are saved to ort_outputs_torch_outputs_z(Torch output are saved to torch_outputs_openpickledumprr)rJrrrs r" save_outputszGpt2Helper.save_outputss L7+T 2a KK Q '3 :1#WEF N1#W-t 4 KK q )5 >qcIJ 3 25 4sBBBB'ctd|dd5}tj||dddtj d|dy#1swY#xYw)N dummy_inputs_r_r`z!inputs are saved to dummy_inputs_rb)rJr rrrs r" save_inputszGpt2Helper.save_inputssI M!G,d 3q KK a (4 7s'BC4 3s A  Ari'r;c|j}tjd|d|d|d|d| d|d d}d }d }d }|r0tj ||||| }tj |||}d }d }g}d g|z}||z}t |D]E}t||z }tjd |}|d k(rd ntjd |} tjd |}!tjd|!d| dtj|!| ||j|j|j|j||| | | | |d}"tj!||"}#|rtj#||"}$n1tj |!| ||| }%tj%||"||%}$tj'|#|$|\}&}'}(})}*t)j*|'s|j-|'|&r|d z }|*r|d z }||xxd z cc<|rr|&sptjd|d|!d| d|d|' t/|)D]>\}}+tjd|d|j1|j2d|+@|st)j*|'s |'d|zkDstj5||"tj7||$|#H|r*dD,cic]},d|,t)j8||,d }-},ndD,cic]},d|,d }-},|d z|z |-d!<|D.cgc] }.|.d z|z  c}.|-d"<|d z|z |-d#<|t;|z d z|z |-d$<tjd%|d&|d'|t;|z d(||d)|zkDr)tjd*t|dz|z d+d,|-Scc},wcc},wcc}.w)-zKGenerate random inputs and compare the results of PyTorch and Onnx Runtime.zRunning parity test (atol=z , test_cases=z, runs=z, use_io_binding=z, model_class=z , is_float16=z) ...r<Nrr;z#Running parity test for batch_size=z past_sequence_length=z...T)rrrr)rz test_case=z batch_size=z sequence_length=z MaxDiff= z: Name=z, d)2Z_cmax_diff_percentile_rnanrtop1_match_ratetop1_match_rate_per_rundiff_pass_ratenan_ratezParity Test Cases=z ; Passed=z; Nan=z; Top1_Matched=gffffff?zParity is good: passed rate=z.0f%)r rrrxrrrBrrrrrr|r}rRr~rJrRr\rrrrCrM get_outputsrrirf percentilerA)/rOrrrrrtest_cases_per_runr@use_io_bindingrrrrrrrrenable_pickle_outputr max_batch_sizemax_past_seq_len max_seq_lenrmax_output_shapespassed_test_casestop1_matched_casesmax_abs_diff_listtop1_matched_cases_per_runtotal_test_casesrJrun_idr{rzryr r rrrrrrrmessagerurGxs/ r" test_parityzGpt2Helper.test_paritys.#\\ (m:J LL5j\AWXlWmmpq &66$**""!! " /#5%9"&7L"!225,GG(>>{LY * < <(# ! )MM~} --g{-N  %;;|,!((6!Q&!"a'"*62a72|  < |;QRfQggxzIyJJST`Sab#,H"5JAwKK"QCw{/F/F/H/K/P/P.QQST[S\ ]^#6$\)BlUX[_U_F_&&q,7'';@G)J eueu`a&qc*u/?/?@QST/UVY.Z[eu BRRAQA,QC0%7AQFR$6$"Ornrk c|j}d}|r0tj| || ||}tj|||}tj | || |j |j |j|j||||| | | }|rtj|||\}}|Stj||||\}}|S)zCGenerate random inputs and measure average latency of Onnx Runtime.N)rrr) r rxrrrr|r}rRr~rRr\)rOrrrr@r}rrrrrrryr{rzr rrr rrGs r"test_performancezGpt2Helper.test_performances(#\\ &880/6;M(::=&R\]N!22    & &    NN       +1!53 " #99+|U_`JAw  $HH\>=*JAwr$c tjddd|j|j|j|j |d|| j }tjj||S)zJIT trace for TorchScript.r;F) ryrzr{r|r}rHr~rrrrr) rxrr|r}rRr~rirDjittrace)rr rrrrhs r" torchscriptzGpt2Helper.torchscriptss 00!" & : :**nn((-11   ') yyuj11r$rawfp32fp16int8c |}tjj|rt|jd}n|j dd|dk7r|d|zz }|r|dz }|rdddd d }d D]}tjj ||||z} tjj| sI||vr/ tj| tjd | |tjd|d| tjj tjj |||dztjj tjj ||dz|dztjj tjj ||dz|dztjj tjj ||d z|dzd Stjj ||dztjj ||dztjj ||dztjj ||dzd S#t$r0} tjd | d | jYd} ~ dd} ~ wwxYw)z=Build a path name for given model based on given attributes.r/rr_past_fp32_fp16_int8rzRemoved the existed directory: zFailed to remove the directory r(NzDirectory for z existed: z.onnxz _fp32.onnxz _fp16.onnxz _int8.onnx)rrisdirrpartssplitrexistsshutilrmtreerrOSErrorstrerror) output_dirmodel_name_or_pathrhas_past new_folderremove_existing model_namesuffixrnew_dires r"get_onnx_pathszGpt2Helper.get_onnx_pathss`( 77==+ ,0177;J   S !" % + + # + +J  ' !J '7SF= '',,z:z@R3RS77>>'*!_4c"MM'2"KK*I'(ST nZL 7)$TU>ww||BGGLLZ$H*W^J^_ GGLLZ'-AB- GGLLZ'-AB- GGLLZ'-AB- "77<< J,@AGGLLZ,-FGGGLLZ,-FGGGLLZ,-FG   - 'c"KK*I'RTUVU_U_T`(abbcs=-J K(%KKN)r)F)MbP?r)r)FFr)r)T)rTF)TT)'r,r-r.r/rUrDint32rrboolrqrbrrrrrrrrrrrrrr rrJrRrTrVTensorr\rfrirrrrrlr$r"rxrxsO!%#'',{{*/++,1KK"&>I>I!>I>I! >I  >I  >I>I >I>I>I!>I>I"KK>I$kk>I >I !>I>I@ -  !      c49n   D  $$43 3 j ).!%#'',{{*/++,1KKuu u #' u  u!uu"KKu$kkuun"'"!!F$ CCCyCCJ((((2,:,3,,>  (   !,10,0,S%,,./0,CcN+ 0,  0,  0,&*0,0,dKKDD     %  ;;"[["%EEN %  ;;"[[44l22"-7 : : : r$rx)4loggingrrdrrrrDpathlibrtypingrrrrrrrDbenchmark_helperrrrr fusion_optionsr io_binding_helperr r$r optimizerr torch_onnx_export_helperr transformersrrrr getLoggerr,rPRETRAINED_GPT2_MODELSFLOAT32FLOAT16INT8DEFAULT_TOLERANCErr3r8rWr[rrbrxrlr$r"rs  ++ &-(- $6LL   8 $Wv s NNCN9N8[8!E)!EHEE$ E/ E"*8T:"=x!O|T2 SS>_  _  r$