
    gI                         d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 d dl
mZ d dlmZmZ  ej                  e      Zd Zd Zd Zd Zd	 Zed
k(  r e        yy)    N)setup_logger)BenchmarkRecord)WhisperConfigWhisperProcessorc                     t        j                         } | j                  ddt        dd       | j                  ddd d	       | j                  d
dd ddgd       | j                  ddt        d       | j                  ddt        d       | j                  dddd       | j                  dddd       | j                  dt        d        | j                  d!t        d"        | j                  d#t        dd$       | j                  d%t        dg d&d'(       | j                  d)t        dg d*d+(       | j                  d,t        d-d./       | j                  d0ddd1       | j                  d2t        dd3/       | j                  d4t        d d5/       | j                  d6dd7       | j                         }t        |d8|j                  j                  d9      d:   j                  d;d<             d=|j                   d<|j                   }|j                  s||_        t        j                  |j                  d>       |xj                  d?z  c_        |S )@Nz-a--audio-pathTz0Path to folder of audio files for E2E evaluation)typerequiredhelpz-lz
--languagezLanguage of audio file)defaultr   z-tz--task
transcribe	translatezTask to complete)r   choicesr   z-w--warmup-runs   )r	   r   z-n
--num-runs
   z--hf-pt-eagerF
store_truez,Benchmark in PyTorch without `torch.compile`)r   actionr   z--hf-pt-compilez)Benchmark in PyTorch with `torch.compile`--hf-ort-dir-pathzDPath to folder containing ONNX models for Optimum + ORT benchmarking)r	   r   --ort-model-pathz'Path to ONNX model for ORT benchmarking--model-namez9Model name in Hugging Face (e.g. openai/whisper-large-v2)--precision)int8fp16fp32zPrecision to run model)r	   r
   r   r   --device)cpucudarocmzDevice to benchmark models--device-idr   zGPU device ID)r	   r   r   z	--verbosezPrint detailed logsz	--timeoutz8Number of mins to attempt the benchmark before moving on--log-folderz'Path to folder to save logs and results--tune)r   r   
model_size/.-z./)exist_ok<   )argparseArgumentParseradd_argumentstrint
parse_argssetattr
model_namesplitreplacer$   	precision
log_folderosmakedirstimeout)parserargslog_folder_names      j/var/www/openai/venv/lib/python3.12/site-packages/onnxruntime/transformers/models/whisper/benchmark_all.pyget_argsr>      s   $$&F
?   %	   {+   	   	   ;	   8	   S   6   H	   (%   ')   	   "	   G	   6	   %EDD, 5 5c :2 > F FsC PQ4??+1T^^,<=O??)KK$/ 	LLBLK    c                    g }d }d}d}d}d}d\  }	}
d\  }}d\  }}}}d\  }}d}d}d	}d
}t        |      5 }|D ]  }|j                  dd      }||v rd}n||v rd}n
||v s||v rd}||v r't        |t        |      |j	                  d             }Y||v r>t        |t        |      |j	                  d             }|dk(  r||}
}	d }|dk(  s||}}d }||v r1t        |t        |      |j	                  d             }||z  }|dz  }||v sd|v r3t        ||j	                  d      dz   |j	                  d             dz  }nY||j                  |      t        |      z   d  j                  dd      }t        j                  |      |    d   }t        |      dz  }|	r|	nd|r|ndz   |r|ndz   }|d   }|r||z  nd}t        j                  d| d       t        j                  d| d       t        j                  d|        |||	|
|r|nd|r|nd|||||g
z   }|j                  |        	 d d d        |S # 1 sw Y   |S xY w)NzLoad audio: zFeature extraction: zEvaluating PyTorch...zEvaluating ONNX Runtime...)NN)NNNNz	Latency: zThroughput: zGenerated token length: zpeak=
 z
load-audiozfeature-extractionprocess   CPU=   z MB'"max_used_MBr   r&   zTotal latency: z szAudio duration: zReal-time factor: )openr4   floatlenrfindr/   findjsonloadsloggerinfoappend) 	device_idlog_filebase_resultsentriesstepload_audio_patternfeat_ext_patternpytorch_patternonnxruntime_patternload_audio_latency_sload_audio_throughput_sfeat_ext_latency_sfeat_ext_throughput_stoken_length	latency_sper_token_latency_sper_token_latency_ms
throughputmemorylatency_patternthroughput_patterntoken_length_patternmemory_patternf
input_linelinepeakusagetotal_latencyaudio_durationrtfentrys                                    r=   process_log_filerv      s   G D'--O64>110:--I_FL)02F#J "O'5N	h1J%%dB/D "T)#!T)+ D(,?4,G  $&!$s?';djjo"NO	#t+"4,>(?$**S/#RS
<'DMz*A(D//@I:(=&D%-"4,@(ADJJsO#TU&/,&>#':T'A$4'D= #4

3!(;djj>O#PQTXXF  		. 9C<O O QRZZ[^`cdD JJt,Y7FE"5\D0F
 .B)q-?)QH$-y16 
 ".b!1:H}~5bom_B?@.~.>bAB067 % (+*<&"-B)((  u%A  
F NG 
F Ns   BI A I D'I  I
c           	      $   dd l }|j                  | g d      }|d   j                  d      |d<   |d   j                  d      |d<   |d   j                  d      |d<   |d	   j                  d      |d	<   |d
   j                  d      |d
<   |d   j                  d      |d<   |d   j                  d      |d<   |d   j                  d      |d<   |d   j                  d      |d<   |d   j                  d      |d<   |d   j                  d      |d<   |d   j                  d      |d<   |d   j                  d      |d<   dd l}|j                  }t        |D cg c],  }|j                  dv s|j                   d|j                   . c}      }d}d}	|r.|d   j                  d      d   }|d   j                  d      d   }	g }
|j                         D ]  \  }}|d   dk(  rt        |d   |d   |d   |d   ||	      }n8t        |d   |d   |d   |d   t        j                  t        j                        }|d   |j                  j                  d<   |d   |j                  _        |d   |j                  _        |d   |j$                  j                  d<   |d	   |j$                  j                  d<   |d
   |j$                  j                  d <   |d   |j$                  j                  d!<   |d   |j$                  j                  d"<   |d   |j$                  j                  d#<   |d   |j$                  j                  d$<   |d   |j$                  j                  d%<   |d   d&z  |j$                  _        |d   |j$                  _        |d   |j$                  _        |
j-                  |        t        j.                  ||
       t        j0                  |j3                  d'd(      |
       t4        j7                  d)| d*       y c c}w )+Nr   )Warmup RunsMeasured Runs
Model NameEngine	PrecisionDevice
Audio FileDuration (s)Token LengthLoad Audio Latency (s)Load Audio Throughput (qps)Feature Extractor Latency (s)"Feature Extractor Throughput (qps)Latency (s)Per Token Latency (ms/token)Throughput (qps)Memory (GB)Real Time Factor (RTF))columnsrx   r/   ry   r   rM   r   r   r   r   r   r   r   r   r   r   )onnxruntimezonnxruntime-gpuz==rB   rH   r{   r   rz   r|   r}   r~   
audio_filedurationrc   load_audio_latencyload_audio_throughputfeature_extractor_latency_s feature_extractor_throughput_qpsrf   rt   rE   .csvz.jsonzResults saved in !)pandas	DataFrameastypepkg_resourcesworking_setsortedkeyversionr3   iterrowsr   torch__name____version__config
customizedwarmup_runsmeasured_runsmetricslatency_ms_meanthroughput_qpsmax_memory_usage_GBrU   save_as_csvsave_as_jsonr4   rS   rT   )resultsfilenamepddfr   installed_packagesiinstalled_packages_listort_pkg_nameort_pkg_versionrecords_rowrecords                 r=   save_resultsr      sD   	
 
 
B2 =)007B}_-44U;BN+227;B~N+2259B~#%&>#?#F#Fw#OB (*+H(I(P(PQX(YB$%*,-L*M*T*TU\*]B&'/12V/W/^/^_f/gB+,=)009B})+,J)K)R)RSZ)[B%& 23::7CB=)009B}#%&>#?#F#Fw#OB  &22$*<l*<QIk@kAEE7"QYYK	 *<l LO.q177=a@1!4::4@C G++-3x=M)$L!3{#3S]CMS_apF %L!3{#3S]CMSXSaSachctctF 25\1B  .$'$6!&)/&:#03N0C!!*-474G!!.1:=>V:W!!"67=@A^=_!!"9:CFGfCg!!"?@HKLpHq!!"DE<?@^<_!!"89+./G+H!!%(),]);d)B&(+,>(?%-0-?*v5  8 '2  !1!1&'!BGL
KK#H:Q/0Q 	ms   P2Pc                 p   | dt         j                   j                         dd}t        j                  j	                  | j
                  |      }t        |d      5 }t        j                  |||      }	 |j                  | j                         d d d        t        j                  d       | j                  | j                   | j"                  || j$                  | j&                  ||g}	t)        | j*                  ||	      }
|
S # t        j                  $ r |j                          Y w xY w# 1 sw Y   xY w)Nr   %Y-%m-%d_%H:%M:%Sz.logw)stdoutstderrz Gathering data from log files...)datetimenowr7   pathjoinr6   rL   
subprocessPopenwaitr9   TimeoutExpiredkillrS   rT   r   num_runsr2   r5   devicerv   rV   )r;   benchmark_cmdenginer   r   log_filenamelog_pathrW   rC   rX   r   s              r=   	benchmarkr   Q  s    XQx004467HINLww||DOO\:H	h	""=(S	LL& 
 KK23	L t~~xFGN# (( 	LLN		 
	s*   D,6D#D)&D,(D))D,,D5c                  $   t               } t        | j                         t        j	                  | j
                         dt        j                  j                  _	        t        j                  | j                        }t        j                  | j                        }|j                  | j                  | j                         }|j"                  gt%        t'        d |            z   }| j                  r| j                   rdt)        |      gng }| j                  r| j                   rdt)        |      gng }| j*                  rdgng }g }t-        j.                  | j0                        D ]j  }	t,        j2                  j5                  | j0                  |	      }
	 t7        j8                  |
      }t        j	                  d|
 d       | j>                  rdddd|
ddd| j                  d| j@                  d| jB                  dt)        | jD                        dt)        | jF                        dt)        | jH                        d| jJ                  g|z   }t        j	                  d       t        | |d|	|      }|jM                  |       | jN                  rdddd|
ddd| j                  d| j@                  d| jB                  dt)        | jD                        dt)        | jF                        dt)        | jH                        d| jJ                  g|z   }t        j	                  d       t        | |d|	|      }|jM                  |       | jP                  rdddd|
dd d!| jP                  d| j                  d| j@                  d| jB                  dt)        | jD                        dt)        | jF                        dt)        | jH                        d| jJ                  g|z   }t        j	                  d"       t        | |d#|	|      }|jM                  |       | jR                  sdddd|
dd$d%| jR                  d| j                  d| j@                  d| jB                  dt)        | jD                        dt)        | jF                        dt)        | jH                        d| jJ                  g|z   |z   }t        j	                  d&       t        | |d'|	|      }|jM                  |       m | jT                   d(| j@                   d)tV        jV                  jY                         d*d+}t[        |t,        j2                  j5                  | jJ                  |             y # t:        $ r@}d}t        j=                  d| d	       t        j=                  d
|	 d       Y d }~d }~ww xY w),NT)languagetaskc                     | d   S )NrH    )token_ids    r=   <lambda>zmain.<locals>.<lambda>x  s    Xa[r?   z--decoder-input-idsr#   )r   r&   z@An error occurred while trying to calculate the audio duration: )exc_infozJIf you get an error that says:
	soundfile.LibsndfileError: Error opening 'zt': File contains data in an unknown format.
you may not have installed `ffmpeg` in addition to installing `librosa`.zTesting z...pythonz-mzmodels.whisper.benchmarkr   z--benchmark-typezhf-pt-eagerr   r   r   r!   r   r   r"   z'Benchmark PyTorch without torch.compilezpytorch-eagerzhf-pt-compilez$Benchmark PyTorch with torch.compilezpytorch-compilezhf-ortr   z Benchmark Optimum + ONNX Runtimezoptimum-ortortr   zBenchmark ONNX Runtimer   r(   r   r   r   ).r>   r   verboserS   rT   __dict__r   backendscudnnr   r   from_pretrainedr2   r   get_decoder_prompt_idsr   r   decoder_start_token_idlistmapr.   tuner7   listdir
audio_pathr   r   librosaget_duration	Exceptionwarninghf_pt_eagerr5   r   rV   r   r   r6   extendhf_pt_compilehf_ort_dir_pathort_model_pathr$   r   r   r   )r;   r   	processorhf_forced_decoder_idsort_forced_decoder_idshf_decoder_input_ids_cmdort_decoder_input_ids_cmdort_tune_cmdall_resultsr   r   r   er   r   csv_files                   r=   mainr   l  s(   :D
KK%)ENN"**4??;F 00AI &<<dmmZ^ZcZc<d$;;<t(*?@@  @D}}QUQZQZ	$9 :;`b  AERVR[R[	$: ;<ac  "&H:LKjj1
WW\\$//:>
	++<H 	hzl#./ *"DNN#D$$%DMM"+, )-)M. KKABm_jRZ[Gw' *"DNN#D$$%DMM"+, )-)M. KK>?m5F
T\]Gw' *"#$$DNN#D$$%DMM"/0 )1)M2 KK:;m]JPXYGw'  ."&&''"OO!NNKK!'#(() &"OO/2 ,3,4 5 : KK01m]JPXYGw'M 2P //"!DNN#31X5F5F5J5J5LM^4__cdHbggll4??HEFK  	HNN]^_]`alpNqNN^_i^j  k`  a 	s   U	V5V

V__main__)r+   r   rQ   loggingr7   r   r   r   benchmark_helperr   r   r   transformersr   r   	getLoggerr   rS   r>   rv   r   r   r   r   r?   r=   <module>r      sp        	    ) # 8			8	$BJXvW1t6`GF zF r?   