gaddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl ZddlZddlmZddlmZmZddlmZmZmZmZddlZej8eZddZdZ dZ!d Z"d Z#d Z$d Z%ed k(re%yy)) annotationsN) setup_logger)add_io_bindings_as_tensorsget_initial_inputs_and_outputs) AutoConfigAutoModelForCausalLM AutoTokenizerBitsAndBytesConfigc |jdvrd}|jdk(r|jdk(rtdddtj }t j|jdk7r |jn |j|j|j|j|jdd||jd i }n t j|jdk7r |jn |j|j|j|j|jd|jdk(rdnd j|j }|j'|jdk(rt j(|}|St+j,}|jdk(rdd|jifnd}t+j.|j0||g}|S#t"$r}t%d |t j|jdk7r |jn |j|j|j|j|jdd j|j }Yd}~-d}~wwxYw)Npt-eager pt-compileint4cudaTnf4) load_in_4bitbnb_4bit_use_double_quantbnb_4bit_quant_typebnb_4bit_compute_dtypeflash_attention_280GB) cache_dir torch_dtypeuse_auth_tokentrust_remote_code use_cacheattn_implementationquantization_config max_memorysdpa)rrrrrrz&Try to load a model using eager mode: eagerrCUDAExecutionProvider device_idCPUExecutionProvider) sess_options providers)benchmark_typeonnx_precisiondevicer torchfloat16rfrom_pretrained hf_dir_path model_namerrauthtrustr$to target_device ExceptionprintevalcompileortSessionOptionsInferenceSessiononnx_model_path)argsmodel bnb_configer&eps h/var/www/openai/venv/lib/python3.12/site-packages/onnxruntime/transformers/models/llama/benchmark_e2e.py get_modelrB8s3 88   & (T[[F-B+!*.$)',}} J)88$($4$4$:  .. ,,#yy"&**$7$. NNF3 E ),<<(,(8(8B(>D$$DOO"nn $ 0 0#'99&*jj"@D v@U)<[a"T''(.    , .MM%(E L))+ {{f$%{DNN&C D' $$T%9%9 ac`de L9 )>B,<<(,(8(8B(>D$$DOO"nn $ 0 0#'99&*jj"(/"T''( )sBG)) J 2BJJ c6|jdk(r%tj5|di|}dddd}|jdvr9|jdk7r]tjj |j n3t||||j|j}|jtj}t|D]}|jdvr^tj5|di|}|jdk7r)tjj |j dddo|j||jtj}||z |z } | |fS#1swYWxYw#1swYxYw)Nrr cpu)r(r+no_gradr*r synchronizer3ruse_fp16use_buffer_sharesynchronize_inputstime perf_counterrangerun_with_iobindingsynchronize_outputs) r<r=runsinputsoutputs io_bindingstart_endavgs rA run_inferencerXxsL l* ]]_ofoGJ 88 ;;%  JJ " "4#5#5 6/vw W[WlWlm %%'    E 4[   "< </&/;;%'JJ**4+=+=>!  $ $Z 0  * * ,    C ;$ C <5_ !s F5AFF F c tt|||||j|j|j|j \}}t |||j||\}}||fSN) clear_cacherr3rHrIenginerX warmup_runs) r<r=config tokenizer prompt_lengthpromptrQrRrUs rAprepare_model_for_inferencerbsfM4 =&$2D2DdmmUYUjUjlplwlwOFGtUD,<,nd?d?d>g@|jdAdBtdCD|jdEdFtdGD|jdHdItdJD|jdKtdLD|j}tjj|jtj|jdM|j vrVt#|dN|j$j'dO|j(dPk(r|j(dQ|j*if|_|j dMk(r|j,sJdR|j.j1dS|_|j2j1dS|_t#|dT|j4|j4dUvs|j4dVk(r|j$d?k(rd4ndW|_|j$d?k7rdX|j*n |j$}|j4dWk(rtj6ntj8}|j dMk(rdMndY}t#|dZ|t#|d[|t#|d\|t#|d]|j4dWk(|j:xr|dMk(|_|S)^Nz-btz--benchmark-typeT)r rr8)typerequiredchoicesz-mz --model-nameFzrrrrzcuda:ptr3rr\rH)argparseArgumentParser add_argumentstrospathjoinintr+r is_available parse_argsnprandomseed manual_seedr(setattrr*upperrr$r; batch_sizessplitprompt_lengths precisionr,float32rI)parserr<r3rr\s rAget_argsrs  $ $ &F   1     K    D    t     S-0D   p    !    S(G^DC   B  P           0 t    /    **113  }3B oCC lcB sA6    DIINN499 dii  ###*t{{/@/@/B.CCT,UV  " "&= ='+'>'>dnn@]&^D # e###S%SS#''--c2D--33C8D D"DNN3..$4469QVZVaVaejVjqw N15 u0DeDNN+,$++M#'>>V#;%--K))U2UF D/=1 D-- D(F# D*dnn67 11EfoD Krcc9:t}tdtj|jd}t |j 5}tj|d}dddtj|jdk7r |jn |j|j|j|j}t!j|jdk7r |jn |j|j|j|j}t#|}g}t%j&|j(|j*D]^\}}t-|t-|}}tjd|d|t/||j0z} ||vr[t3t5j6d|d |j d |j d |d |d |d|d|d|j d|d||g|z} ||g} tjdt9|||||| \} } t;|||j<| | \}} |dz}|||z z}tjd|dtjd|||z zd| j?||gtjdt/t9|||||| \} } | djA}|jBd}|jD}tG|dr |jHn|jJ|jLz}tOjP||jRtNjT}g}g}tWjX}|| kryt;||d| | \}} |j[|tWjX}| djBddkDr| d j]ddz }|j_d!jad|jbje|d|jb}tOjf| dd|ji}n| ddddddf}tOjj|d!}||z|jlk(}|jo||jljq|dg}tWjX} |j[| |z tOjr||gd!}|dz }|| d<tOjr| d |jutNjvjq|dgd| d <d"| vr3tOjx| d"d!d#jq|ddz| d"<| djBddk7r"| dddddddfj{| d<| dj}|j~d$k(r | d%| d%<n |jst|jD]"}!| d&|!d'| d(|!d'<| d&|!d)| d(|!d)<$| d jBd}"t|jD]}!tOjP|||"||jR|j}#tOjP|||"||jR|j}$| jd&|!d'|#j{d&|!d)|$j{i|| krytWjX}%|jd#|jrad*9t|:t|}&tt9:fd+|}t|}'tjd,|&|'z d-9d.:dzd/t]|t|z }(|(dz})|d|(z z}*tjd0|)dtjd1|*d|d#}+|+dz},|d|+z z}-tjd2|,dtjd3|-d|j0d4z}.t]|d|.t|d|.z }/|/dz}0|d|/z z}1tjd5|.d6|0dtjd7|.d6|1dt]|t|z }2|2dz}3|d|2z z}4tjd5|j0d6|3dtjd7|j0d6|4d|%|z }5|||j0z|5z z}6tjd8|5d9tjd:|||j0z|5z zdtjd;| j?|)|*|,|-|0|1|3|4|5|6g |j[| ad>|j~d?tjjd@dA}8t||8|j0y#1swY xYw#t$r)}7tjd<|d|d=|7Yd}7~7d}7~7wwxYw)BNFch|jDcic]\}}t||c}}Scc}}wrZ)itemsr)dkvs rAzmain..hs.STSZSZS\<]S\41aSVQYS\<]<]s.) object_hookr)rrrzRunning batch size = z, prompt length = z2 A prompt of size z was not found in 'zv'. There are a couple of solutions to fix this. 1) You can change one of the keys in 'z' to be z). If za < actual prompt's length, the benchmark E2E tool will repeat the first word in the prompt until zB = actual prompt's length. If zm > actual prompt's length, the benchmark E2E tool will automatically trim the actual prompt's length so that zd = actual prompt's length. 2) You can add a new key-value entry in 'z' of the form 'z,': 'your prompt goes here'. zMeasuring prompt processing...iz&Average Latency of Prompt Processing: z msz)Average Throughput of Prompt Processing: z tpszMeasuring token generation... input_idshead_dim)r*dtypelogitsattention_mask)dim position_idsrrpast_key_valueszpresent.z.keyzpast_key_values.z.value c|zkSrZrE)acc_timeanomaly_threshold_factor min_time_ss rArzmain.. sH7OR\7\,\rcz Filtered out z$ anomaly accelerator times that are zx greater than z ms...zAverage Latency of Sampling: z Average Throughput of Sampling: z"Latency of First Token Generated: z%Throughput of First Token Generated: rjriz Tokens Generated: rkzWall-Clock Latency: z szWall-Clock Throughput: zAdding results to CSVz$Could not benchmark at batch size = z - benchmark__e2e_z%Y-%m-%d_%H:%M:%Sz.csv)Orrrrrs__dict__open prompts_filejsonloadrr-r.r/rr0r1r rB itertoolsproductrrrr[generation_lengthNotImplementedErrortextwrapdedentrbrXnum_runsextendcloneshapenum_key_value_headshasattrr hidden_sizenum_attention_headsr+zerosr3boolrKrLappendsum unsqueezerepeat vocab_sizeviewgathersqueezeargmax eos_token_id masked_fillreshapecatr2int64max contiguouszero_r\rIrMnum_hidden_layersrupdatepopanomaly_filteringminlenlistfilterr4datetimenowrx);r<size_to_promptfr^r_r=all_csv_metrics batch_sizer` max_lengthra csv_metricsrQrRaccelerator_prompt_latency_saccelerator_prompt_latency_msaccelerator_prompt_thrpt all_token_idscurrent_length num_heads head_sizehas_eosaccelerator_timessampling_timeswall_clock_start_timeaccelerator_time_latency_ssampling_start_timeprompt_end_indicesidxsnext_token_logits next_tokens tokens_to_addsampling_end_timeinew_sequence_length present_key present_valuewall_clock_end_time orig_sizenew_sizeavg_sampling_latency_savg_sampling_latency_msavg_sampling_thrptfirst_token_latency_sfirst_token_latency_msfirst_token_thrpthalfwayhalfway_token_latency_shalfway_token_latency_mshalfway_token_thrptall_token_latency_sall_token_latency_msall_token_thrptwall_clock_latency_swall_clock_thrptr?rurrs; @@rAmainr`sH :D KK N d A12]^ ! ' ' ,,2..yy** F -- ,,2..yy** I dOEO%.%6%6t7G7GI\I\%]! M$' OS5GM  +J<7I-YZ "T%;%;;  .%22?@STXTeTeSfgGGKGXGXFYYaboapq((57XYfXgg((57derdssJJNJ[J[I\\klykz{   !/0:=!=1 H t KK8 99$vyZgiopOFG4A$t}}^dfm4n 1 ('-I4,O )'1]Ea5a'b $ KK@A^@__bc d KK;J-ZvJvFDVDVZ`ZtZtDt kk*T5G5GuzzZG!# N$($5$5$7 ! J.6CD%QRTZ\c6d3*G!(()CD'+&7&7&9#8$**1-1)/0@)A)E)Ea)H1)L&*444;6#4#45j!V->->? ). WX5F4(P(X(X(Z%(/(9!R((C%#ll+<"E "K/93I3II!, 7 7AWAW X ` `blnoap q $($5$5$7!%%&7:M&MN % =-*Hb Q !#'4{#+099,-}}U[[/I/Q/QR\^_/`acd,'("V+-2YYvn7MST-UVW-X-`-`akmn-ors-sF>*8$**1-2(/(9!RaR((C(N(N(PGH%!'');;$&078I0JF,-.."6#;#;<=DxPQsRVEW=X!1!D9:?FRSQTTZG[?\!1!F;<=+11A*B*H*H*K'"6#;#;<&+kk&%/%#'#5#5"&"2"2 ' ). &%/%#'#5#5"&"2"2 )  "*1#T 2K4J4J4L"*1#V 4m6N6N6P#=o!J.^#'"3"3"5   ! !! $%%+-( !23  12 $(\^op%!01 #I$8#99]^v]wxGHRUYHYGZZ`a&)%83~;N%N "&'E $",4K0K"L  KK3G9T>T.TXl-lm  KK./C.DBG H KK)*I_I_9_cw8w*x)yy}~  KK/ 0   +&*%,'(#($    " "; /u&^~DKK=h.?.?.C.C.EFW-XX\]H(D,B,BCg ! ^ t KK>zlJ\]j\kknopnqr s s ts,hS2hJ-hh iii__main__)r<zargparse.Namespace)& __future__rrrrerrloggingrrrKnumpyrpandasror+benchmark_helperr llama_inputsrr transformersrrr r onnxruntimer8 getLogger__name__rrrBrXrbr[rxrrrErcrAr*s@#     )S\\   8 $=@> 12_DzDz zFrc