
    g=                     t   d dl Z d dlZd dlZd dlmZ d dlmZmZmZ d dl	Z	d dl
Z
d dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZ  e j6                  e      Z G d dej<                  j>                        Z  G d dej<                  j>                        Z! G d d      Z" G d d      Z#y)    N)Path)ListOptionalUnion)
TypeHelper)PastKeyValuesHelper)	OnnxModel)torch_onnx_export)WhisperConfig
file_utils)WhisperDecoderInitOpenai)InferenceSessionc                        e Zd ZdZ	 d	dej
                  j                  dedee	   f fdZ
dej                  dej                  fdZ xZS )
WhisperDecoderInitzvA Whisper decoder to create initial past key values.
    This model is only called once during starting decoding.
    decoderconfigdecoder_start_token_idc                     t         |           || _        || _        ||| _        y | j                  j                  | _        y N)super__init__r   r   r   )selfr   r   r   	__class__s       l/var/www/openai/venv/lib/python3.12/site-packages/onnxruntime/transformers/models/whisper/whisper_decoder.pyr   zWhisperDecoderInit.__init__!   sD     	&<&H" 	#NRkkNpNp 	#    decoder_input_idsencoder_hidden_statesc                     t        j                         }||d<   d |d<   d |d<   | j                  j                  d ||d dd      }| j                  j	                  |d         }||j
                  |j                  fS )Nlast_hidden_statehidden_states
attentionsTencoder_outputsr   past_key_values	use_cachereturn_dictr   )r   ModelOutputr   modelproj_outr$   encoder_last_hidden_state)r   r   r   r#   outlogitss         r   forwardzWhisperDecoderInit.forward.   s    
 %002/D+,+/((,%ll  +/  ! 
 &&s1v.s**C,I,IIIr   r   )__name__
__module____qualname____doc__torchnnModuler   r   intr   TensorFloatTensorr-   __classcell__r   s   @r   r   r      s^     15	

 
 !)	
J <<J  %00Jr   r   c                   \     e Zd ZdZddedej                  j                  f fdZd Z	 xZ
S )WhisperDecoderz&A Whisper decoder with past key values
model_implr(   c                 t    t         |           || _        || _        || _        |t        ||      | _        y y r   )r   r   r   r   r<   r   whisper_decoder_openai_init)r   r   r   r<   r(   r   s        r   r   zWhisperDecoder.__init__G   s=    $/Gw/WD, r   c                    t        j                         }t        j                  |j                  d   dt        | j                  j                        f      }||d<   ||d<   d |d<   | j                  dk(  r,|j                  d       | j                  |||      \  }}||fS t        |      dk(  rd }nt        j                  |      }| j                  d |||dd	      }|d   }	t        j                  |j                         \  }
}|	|
fS )
Nr     r   r    r!   openai)pastTr"   )r   r'   r2   randnshaper5   r   d_modelr<   	unsqueezer>   lenr   back_group_by_layerr   group_by_self_and_crossr$   )r   r   rB   r#   dummy_encoder_hidden_statesdec_outpresentr$   decoder_outr,   present_self_s               r   r-   zWhisperDecoder.forwardO   s#   $002&+kk3D3J3J13MtUXY]YdYdYlYlUm2n&o#/J+,+F((,%??h&'11!4#??!#>T  @  GW G##t9>"O1EEdKOll+/+ # 
 Q-EEkFaFaba|##r   )hfN)r.   r/   r0   r1   strr2   r3   r4   r   r-   r8   r9   s   @r   r;   r;   D   s)    0XC Xuxx X$r   r;   c                   v    e Zd Z	 ddZe	 	 	 ddededededej                  de	d	e	d
e
fd       ZdefdZd Zy)WhisperDecoderInputsNc                      || _         || _        y r   )r   r$   )r   r   r$   s      r   r   zWhisperDecoderInputs.__init__p   s    
 4E]lr   r   
batch_sizeencode_sequence_lengthpast_decode_sequence_lengthdevicefloat16use_int32_inputsr<   c                 |   | j                   }| j                  }	| j                  }
| j                  | j                   z  }d}t	        j
                  d|
dz
  ||f|rt        j                  nt        j                  |      }|rt        j                  nt        j                  }|dkD  r||||g}|||dk(  r|n||g}g }t        d|	z        D ])  }|j                  t	        j                  |||             + t        d|	z        D ])  }|j                  t	        j                  |||             + nd}t        ||      S )ad  Create dummy inputs for WhisperDecoder.

        Args:
            decoder: decoder
            batch_size (int): batch size
            encode_sequence_length (int): sequence length of input_ids for encoder
            past_decode_sequence_length (int): past sequence length of input_ids for decoder
            device (torch.device): device of output tensors
            float16 (bool): whether the model uses float32 or float16 in input
            use_int32_inputs(bool): whether use int32 instead of int64 for some inputs

        Returns:
            WhisperDecoderInputs: dummy inputs for decoder
           r   )lowhighsizedtyperX   rP      )r`   rX   N)encoder_attention_headsdecoder_layers
vocab_sizerE   r2   randintint32int64rY   float32rangeappendrandrS   )r   rU   rV   rW   rX   rY   rZ   r<   num_attention_heads
num_layersrd   	head_sizesequence_lengthr   
float_typeself_attention_past_shapecross_attention_past_shaperB   rO   s                      r   create_dummyz!WhisperDecoderInputs.create_dummyx   s>   2 $*#A#A //
 ++
  6+I+II	 !MMao."25;;
 '.U]]5==
&*#+	)% #*4*<&B]	*& D1z>*EJJ'@
[abc + 1z>*EJJ'A\bcd + D#$5t<<r   returnc                 n    | j                   g}| j                  r|j                  | j                         |S r   )r   r$   extend)r   
input_lists     r   to_listzWhisperDecoderInputs.to_list   s3    ,,-
d223r   c                     | j                   r8| j                   D cg c]"  }|j                  t        j                        $ c}nd }t	        | j
                  j                         |      S c c}w )N)r`   )r$   tor2   rh   rS   r   clone)r   prB   s      r   to_fp32zWhisperDecoderInputs.to_fp32   s`    LPL`L`43G3GH3Ga5==)3GHfj#""((*
 	
 Is   'A+r   )FFrP   )r.   r/   r0   r   staticmethodr   r5   r2   rX   boolrQ   rs   r   rx   r}    r   r   rS   rS   o   s     m  !&B=B=B= !$B= &)	B=
 B= B= B= B= B=H 
r   rS   c                       e Zd Ze	 	 	 ddedej                  dedededefd       Z	ede
fd	       Ze	 dd
eeef   dedej                  dedef
d       Zy)WhisperDecoderHelperr   rX   onnx_model_pathverboseuse_external_data_formatrZ   c                    t        | t        t        f      sJ t        j	                  | j
                  ddt        | t              rdnd||| j                        }|j                         }t        j                  | j
                  j                  d      }t        j                  | j
                  j                  d      }	|	d	d| j
                  j                  z   }
t        | t              r|ng }t        | t              r|
n|	}d
g|}dg}|j                  |       ddiddddddd}|D ]  }dd|v rdndd||<    |D ]/  }d|v r	ddd||<   t        | t              r	ddd||<   )ddi||<   1 t        |      j                  j                  dd       t        j                          5 }t"        j$                  j'                  |d      }t        |      j                  j                  dd       t)        | t+        |      |r|n|d|||dd||       |r0t-        j.                  |d      }t1        j2                  ||dd       d	d	d	       y	# 1 sw Y   y	xY w)a  Export decoder to ONNX

        Args:
            decoder (Union[WhisperDecoder, WhisperDecoderNoPastState]): decoder object
            device (torch.device): device of decoder object
            onnx_model_path (str): onnx path
            verbose (bool, optional): print verbose information. Defaults to True.
            use_external_data_format (bool, optional): use external data format or not. Defaults to False.
            use_int32_inputs (bool, optional): use int32 inputs
        ra   r@      r   )rU   rV   rW   rX   rZ   r<   F)rL   TNr,   	input_idsrU   zencode_sequence_length / 2)r   r\   ro   )r   r   r,   r   rW   rV   )r   ra   crosszpast_decode_sequence_length + 1)parentsexist_okzdecoder.onnx   )
argsfexport_paramsinput_namesoutput_namesdynamic_axesopset_versiondo_constant_foldingr   r   )load_external_data)save_as_external_dataall_tensors_to_one_file)
isinstancer;   r   rS   rs   r   r<   rx   r   get_past_namesrc   rv   r   parentmkdirtempfileTemporaryDirectoryospathjoinr
   tupleonnx
load_modelr	   save)r   rX   r   r   r   rZ   inputsrw   
past_namespresent_namespresent_self_namesinput_past_namesoutput_present_namesr   r   r   nametmp_dir_nametemp_onnx_model_pathr(   s                       r   export_onnxz WhisperDecoderHelper.export_onnx   sy   & 'N4F#GHHH%22NN#'-7-PVW-)) 3 
 ^^%

 )778U8U_de
+::7>>;X;Xbfg*+NQ1N1N-NO)3G^)L:RT5?5X1^k 8#78 #m+, \*)5:V%W&+<=
 %D4:dN0H`"L % )D$)5:R%ST"g~6'<*L& <*L& ) 	_$$**4$*G((*l#%77<<n#M %&--33D43P:&*B&"')) $()A ((<QUV#*.,0	' +**s   /BIIr   c                 4   t         j                  d       dt        j                  |j                  j                         j                               i}|j                  rt        |j                        dz  dk(  sJ t        t        |j                        dz        }t        j                  |      }t        |j                        D ]<  \  }}t        j                  |j                         j                               |||   <   > | j                  d|      }|S )zRun inference of ONNX model.zstart onnxruntime_inferencer      r   N)loggerdebugnumpyascontiguousarrayr   cpur$   rG   r5   r   r   	enumeraterun)ort_sessionr   
ort_inputsrm   r   ipast_tensorort_outputss           r   onnxruntime_inferencez*WhisperDecoderHelper.onnxruntime_inference;  s     	23 001I1I1M1M1O1U1U1WX

 !!v--.2a777S!7!781<=J,;;JGJ"+F,B,B"C;,1,C,CKOODUD[D[D],^
:a=) #D "oodJ7r   r(   r   	max_casesc                    t        j                  |d      dk(  }g d}g }|d| D ]  \  }}	}
t        | t              rd}n|
}t        j                  | j                  ||	||||      }|j                         j                         }t        j                         5   | | }ddd       t        j                  ||      }t        j                  t        j                  d   j!                         j                         |d   z
              }|}t"        j%                  d|        t'        d| j                  j(                  z        D ]|  }t        j                  t        j                  |d	   |   j!                         j                         |d	|z      z
              }t"        j%                  d
| d|        t+        ||      }~ t        | t              rt'        d| j                  j(                  z        D ]  }t        j                  t        j                  |d   |   j!                         j                         |d	d| j                  j(                  z  z   |z      z
              }t"        j%                  d| d|        t+        ||      } |j-                  |       t"        j/                  d||	|
|        S # 1 sw Y   "xY w)zQCompare the result from PyTorch and OnnxRuntime to verify the ONNX model is good.past_key_self_0ztensor(float16)))r         )r\   ra      )r   r\   r\   )   r   ra   Nr   )rX   rY   rZ   zlogits max_diff=ra   r\   zself attention past state z
 max_diff=zcross attention past state zUbatch_size=%s, encode_sequence_length=%s, past_decode_sequence_length=%s, max_diff=%s)r   get_input_typer   r   rS   rs   r   r}   rx   r2   no_gradr   r   r   amaxabsr   r   r   ri   rm   maxrj   info)r(   r   rX   rZ   r   rY   
test_casestest_cases_max_diffrU   rV   rW   dec_seq_lenr   rw   torch_outputsr   max_diffmax_diff_allr   s                      r   verify_onnxz WhisperDecoderHelper.verify_onnxN  s    #11+?PQUffB
 
 
#	
"'%!349)66&!1 7 F  )113J  %z 2 ! /DD[RXYKzz%))M!,<,@,@,B,H,H,J[YZ^,["\]H#LLL+H:671u||6667 ::eiia0@0C0G0G0I0O0O0QT_`ade`eTf0f&gh9!JxjQR"<: 8
 %!34q5<<#:#::;A$zz		-"21"5"9"9";"A"A"CkRSVWZ_ZfZfZqZqVqRqtuRuFv"vw H LL#>qcH:!VW#&|X#>L <  &&|4KKg&+W $f ? !s   K  K
	N)TFF)r   )r.   r/   r0   r~   r;   r2   rX   rQ   r   r   rS   r   r   r   r   r5   r   r   r   r   r   r      s    
 ).!&lll l 	l
 #'l l l\ 3G  $  C^%778C%C C 	C
 C Cr   r   )$loggingr   r   pathlibr   typingr   r   r   r   r   r2   io_binding_helperr   models.t5.past_helperr   
onnx_modelr	   torch_onnx_export_helperr
   transformersr   r   whisper_openai_helperr   onnxruntimer   	getLoggerr.   r   r3   r4   r   r;   rS   r   r   r   r   <module>r      s     	   ( (    ( 5   6 2 : (			8	$%J %JP($UXX__ ($VY
 Y
xG Gr   