gYddlZddlZddlmZddlmZmZmZddlZ ddl Z ddl m Z ddl mZddlmZddlmZddlmZmZmZdd lmZdd lmZmZmZdd lmZmZdd l m!Z!m"Z"dd l#m$Z$ejJe&Z'gdZ(GddZ)y)N)Path)DictTupleUnion)float_to_float16_max_diff) OnnxModel)optimize_model)version) WhisperConfigWhisperForConditionalGenerationWhisperProcessor) __version__)WhisperDecoderWhisperDecoderHelperWhisperDecoderInit)WhisperEncoderWhisperEncoderHelper)WhisperEncoderDecoderInitWhisperEncoderDecoderInitHelper)InferenceSession) z whisper-tinyzwhisper-tiny.enz whisper-basezwhisper-base.enz whisper-smallzwhisper-small.enzwhisper-mediumzwhisper-medium.enz whisper-largezwhisper-large-v2zwhisper-large-v3ceZdZe d)dededededef dZedededejdejjfd Z e d*ded ededejd ed ede eejjffd Z e d+deeeeefdejdededededefdZe d,dedeefdZe d-dededededededededefdZe d.d ed!ejjdejd"ed#ef d$Zed"ed#efd%Ze d.deded&edejd"ed#ef d'Zy()/ WhisperHelper output_dirmodel_name_or_pathsuffix new_folderreturnc6|}tjj|rt|jd}n|j dd}||z }|r tjj ||n|}tjj ||dzS)aBuild onnx path Args: output_dir (str): output directory model_name_or_path (str): pretrained model name, or path to the model checkpoint suffix (str, optional): suffix like "_encoder" or "_decoder_fp16" will be appended to file name. Defaults to None. new_folder (bool, optional): create a new directory for the model. Defaults to False. Returns: str: path of onnx model /z.onnx)ospathisdirrpartssplitjoin)rrrr model_name directorys k/var/www/openai/venv/lib/python3.12/site-packages/onnxruntime/transformers/models/whisper/whisper_helper.py get_onnx_pathzWhisperHelper.get_onnx_path,s$( 77==+ ,0177;J#))#.r2Jf r?rconfigevalrBrrrHr)rrJr,r-rKrL extra_kwargsrH openai_model model_encoder model_decoder passed_modelrUrTrVrWs r) load_modelzWhisperHelper.load_modelpsx& ==- .'--2I I29L. //??@Rx^gxkwx  !(::;MyZ`aL+7+?+?AUAU=M'L+0%=ML   ! !%**_"=e ! L  [gh &! )#< '+%" $ -AWU U$U[[%8%8%,,GG LLN  f %-emmU\\JL     " "6 *"" , r+rHonnx_model_pathverboseuse_external_data_formatuse_decoder_input_idsuse_int32_inputsc t|trtj|||||yt|trt j|||||||yt j||||||yN) isinstancerr export_onnxrrr)rHr-rcrdrerfrgs r)rkzWhisperHelper.export_onnxs} e^ , , ,(  8 9 + 7 7%(   ! , ,(  r+ onnx_model op_block_listct|jDcgc]}|jc}}t|}|j|}tj d|d||j jdj}d}|j}||vsJ||}d} |jdk(r|} tj d|jd} |jD]} |j| } | nt| } tjd|jd | | d k}n/tjd |jd |jg} g}|s| |g} | jg}| t|||d }tj d||j dddi||Scc}w)aConvert model to mixed precision. It detects whether original model has fp16 precision weights, and set parameters for float16 conversion automatically. Args: onnx_model (OnnxModel): optimized ONNX model op_block_list (List[str], optional): . Defaults to ["SimplifiedLayerNormalization", "SkipSimplifiedLayerNormalization", "Relu", "Add"] Returns: parameters(dict): a dictionary of parameters used in float16 conversion z fp32 op: z fp16 op: rFNMatMulz#Found last MatMul node for logits: z3max diff of converting weights in last MatMul node z: gư>z-Failed to find MatMul node for logits. Found z of node ) keep_io_typesrmnode_block_listforce_fp16_initializersz!auto_mixed_precision parameters: use_symbolic_shape_inferTr:)setnodesop_type differenceloggerinfographoutputnameoutput_name_to_nodeinputget_initializerrdebugwarninglistconvert_float_to_float16)rlrmnode op_full_set fp32_op_set fp16_op_setlogits_output_nameis_weight_fp16_precisionr}last_matmul_node initializerr~max_diffrprq parameterss r)auto_mixed_precisionz"WhisperHelper.auto_mixed_precisions$J4D4D4FG4FD4<<4FGH -( !,,[9  i }J{mDE(--/66q9>>$) (<<>!%8888"#56 <<8 ##  KK=dii[I JK(88? *$1=H LLNtyykY[\d[ef g'/$ $ NNJ4<<.Xabfbkbkalm n (/?/K/0M/445O+!-0.'?    7 |DE+ ++XTXZXaHsGoptimized_model_path is_float16num_attention_heads hidden_sizeruse_gpuproviderc ddlm} | d} d| _|dk(| _t |d|||sdnd| |d } |r*|rt j | n| jd | j||d y) zHOptimize ONNX model with an option to convert it to use mixed precision.r) FusionOptionsbartTrocmNF) model_type num_headsr opt_leveloptimization_optionsronly_onnxruntime)cast_input_output)all_tensors_to_one_file) fusion_optionsruse_multi_head_attention!disable_multi_head_attention_biasr rr convert_model_float32_to_float16save_model_to_file) rcrrrrrerrrrrms r) optimize_onnxzWhisperHelper.optimize_onnxs 1,V48<5AIVAS>  )#7aT!5"  #221522U2K 13Keijr+ processorpt_model batch_size prompt_modec   ddlm}ddlm}|d d d }g} |d k(r||dddgdj} ni||dddgdj||dddgdjg} t| |k(sJtj| d| d f} d\} } } }d\}}| j|| | | |||ddd }|rddg}|Dcgc]}|j|}}g}g}t|D]}tj|||d<| |j||d<|j di|j#j%j'}|j)||j)|j+|dd| |d<|d=n`g}|j di|j#j%j'}|j+|ddg}t-|}|d=|d=||||fS#t$rU}tj d|dd}tj d|dt j|Yd}~qd}~wwxYwcc}w)Nr) load_datasetz.An error occurred while importing `datasets`: T)exc_infozpip install datasetszCCould not import `datasets`. Attempting to install `datasets` via `z`.z)hf-internal-testing/librispeech_asr_dummyclean validation)r%audioarraypt)return_tensors)rrr)?r) input_features max_length min_length num_beamsnum_return_sequenceslength_penaltyrepetition_penaltyearly_stopping use_cachezJohn has doubtszMaria has grave doubts prompt_idsrskip_special_tokensrrr:)datasetsr Exceptionrxerrorrr!systemrlenr>catrBget_prompt_idsrange from_numpygeneratedetachcpunumpyappend batch_decoder)rrr-rrre install_cmddsinput_features_rrrrrrrinputspromptsprpt_transcription pt_outputsi pt_outputs r) pt_transcription_for_verify_onnxz.WhisperHelper.pt_transcription_for_verify_onnx<s # - * EwVb c ?&1gw(?'@QUVeeN2a5>'23DIXX2a5>'23DIXXO':5 55"YY(:OA>@DDFLLN !!), '' (>(>y^b(>(cde(fg ' (6F# $|$J***4V4;;=AACIIKJ ) 6 6zW[ 6 \]^ _` j)J # $ ; 'Z??s # LLI!MX\L ]0K NN`al`mmop q IIk " "  #HHsH J I>)A I99I>cJ|dkDr|rd}d}d}d}||||h}|Sd}d}d} ||| h}|S) Nrz{ John has doubts whether Sir Frederick Layton's work is really Greek after all and can discover in it but little of Rocky Izy John has doubts whether Sir Frederick Latins work is really Greek after all and can discover in it but little of Rocky Iz Maria has grave doubts whether Sir Frederick Layton's work is really Greek after all and can discover in it but little of Rockyz Maria has grave doubts whether Sir Frederick Latins work is really Greek after all and can discover in it but little of Rocky IzX Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.zY Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.zZ "Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.r:) rr'expected_transcription_no_comma_prompt1)expected_transcription_misspelled_prompt1'expected_transcription_no_comma_prompt2)expected_transcription_misspelled_prompt2expected_transcription_optionsexpected_transcription_no_comma!expected_transcription_with_comma+expected_transcription_with_quote_and_commas r)select_transcription_optionsz*WhisperHelper.select_transcription_optionss >k7t 39t 57y 39{ 57799 . **.-k ,l .m 801;. * .-r+ ort_sessionc " i}tjttjdk\rd|d<tj|fd|i|j |}t j||}tj||} tj|||||\} } } } | jg}|jdd }ttd |}||z}ttd |j}ttd |j}tj tj"tj$tj&tj(tj*d }d|v}t-||D]5\}}|dk(r3| |j/j1j3| |<?|dk(r(tj4| j6||| |<l|dk(r*tj4|| j6f||| |<|dk(r|s&|r|gn|g}tj8|||| |<g}t;|D]$}|j=| |j?&tAd|D}g}|D]6}g|| jBg|tE|z z}|j=||z8tj8|||| |<j|dk(r tj8dg||| |<|dk(r"tj8ddgg||| |<|dk(r5tjFtj8|g|||d| |<|dk(r tj8dg||| |<tj8| |g||| |<8|jId| ddddddf}|jK|d}tjM||} d}!t;|D]}|!| || vxr||| vz}!d}"|!st;|D]}| |jN||jNk7r!| |||dddtE| |fz }#n | |||z }#tA|#jQ|#jAtR}$tA|"|$}"|"dk7r0tTjWd| tTjWd ||"S)!zRCompare the result from PyTorch and ONNX Runtime to verify the ONNX model is good.rNrOrPr,)r,)rrenglish transcribe)languagetaskc |dS)Nrr:)tokens r)z+WhisperHelper.verify_onnx..sE!Hr+c|jSri)r|entrys r)rz+WhisperHelper.verify_onnx..s5::r+c|jSri)typers r)rz+WhisperHelper.verify_onnx..sEJJr+)z tensor(float)ztensor(float16)z tensor(int64)z tensor(int32)z tensor(int8)z tensor(uint8)extra_decoding_idsr vocab_mask)dtypeprefix_vocab_maskdecoder_input_idsc32K|]}t|ywri)r).0rs r) z,WhisperHelper.verify_onnx..s!>+Q#a&+slogits_processorrcross_qk_layer_headr temperaturerNTr)keyzPyTorch outputs: zONNX Runtime outputs: ),r rXrYr rZrBr r rrrSget_decoder_prompt_idsrmap get_inputsnpfloat32float16int64int32int8uint8ziprrrones vocab_sizerrrtolistmax pad_token_idrrepeatrunrrshapeminabsrxr)%rr,rr-rrr]rrr[rrrdecoder_prompt_idsstart_idrforced_decoder_ids ort_names ort_dtypes ort_to_npuse_extra_decoding_idsr|r raw_input_ids ort_promptsrmax_lenpadded_promptsr padded_prompt ort_outputsort_transcriptionrparityrdiff max_diff_is% r) verify_onnxzWhisperHelper.verify_onnxs  ==- .'--2I I29L. /2BB  *3 7C "V* %445GS\] ../AYWCPCqCq   !# DrD @ *.@11255y|5\ #4jAB % 25{7M7M7OPQ #6 8N8N8PQR ZZ!zzXXXXGGXX  "6!By*5KD%''%d|22488:@@Bt %!wwv'8'8 %@PQt ,,!ww F4E4E'FiX]N^_t ,,"2HXJOaNbM#%88M5AQ#RF4L#%K":.#**+=a+@+G+G+IJ/!!>+!>>G%'N(([!([0C0C/DRUVWRXHX/Y([ &--m>P.PQ)$&88N)EBR#SF4L++!xx9U3CDt ..!xx!Q %8HIt --!yy:,iPUFV)WYcefgt &!xxYu5EFt !xxi>NOt G6H"oodF3A6q!Qw? %22;TX2Y)6)S)ST^`k)l&z"A  #'EEK%a(,JJ F# :&a=&&+a.*>*>>%a=;q>!=Qs:a=?Q=Q:Q+RRD%a=;q>9D TXXZSA x4 ' q= NN./?.@A B NN34E3FG Hr+N)F)Tr+)TFTF))SimplifiedLayerNormalization SkipSimplifiedLayerNormalizationReluAdd)FTFr)rF)__name__ __module__ __qualname__ staticmethodstrboolr*r>r-nnModulerIrrbrrrrrrkrrrintrr rrrr*r:r+r)rr+s ==== = ==8$ $ $  $   $ $ L 04! 8888 8 )- 8  8 c588??" #88t ).&*!&#^^5GIbbc# ## # #' # $ ###J% AASzAAF*/%)$k$k!$k$k! $k  $k #' $k#$k$k$k$kL ! C@#C@((//C@ C@ C@  C@C@J....@ ! hhh&h h  h  hhr+r)*loggingr!pathlibrtypingrrrrrr>r rrlr optimizerr packagingr transformersr r r rrYwhisper_decoderrrrwhisper_encoderrrwhisper_encoder_decoder_initrr onnxruntimer getLoggerr0rxPRETRAINED_WHISPER_MODELSrr:r+r)rEsf %% - $YY<TT@c(   8 $ aar+