
    gm                         d dl Z d dlmZmZ d dlZd dlmZmZ d dl	m
Z
 d dlmZmZ d dlmZ d dlmZmZmZ d dlmZ d d	lmZ  e j0                  e      Z G d
 de      Z G d de
      Z G d de      Zy)    N)OptionalUnion)AttentionMaskFusionAttention)Fusion)"FusionSimplifiedLayerNormalization&FusionSkipSimplifiedLayerNormalization)NumpyHelper)	NodeProtoTensorProtohelper)	OnnxModel)BertOnnxModelc                        e Zd ZdZdedededef fdZ	 dded	e	d
e	de	dededededede
e   dee	df   fdZdededededededededededededee	df   fdZd Zd Zd Z xZS )FusionT5Attentionz=
    Fuse T5 Attention subgraph into one Attention node.
    modelhidden_size	num_headsattention_maskc           	      B    t         |   ||||dddg       d| _        y )NF SkipSimplifiedLayerNormalizationAdd)use_multi_head_attentionsearch_op_types   )super__init__	static_kv)selfr   r   r   r   	__class__s        [/var/www/openai/venv/lib/python3.12/site-packages/onnxruntime/transformers/onnx_model_t5.pyr   zFusionT5Attention.__init__   s7     	%*?G 	 	
     N
mask_indexq_matmulk_matmulv_matmulinputoutput
add_qk_strscalereturnc                    |dkD  sJ |dkD  r$||z  dk7  rt         j                  d| d|        y| j                  j                  |j                  d         }| j                  j                  |j                  d         }| j                  j                  |j                  d         }|t        |j                  d    d       yt        j                  |      }t        j                  |      }t        j                  |      }|j                  |j                  k(  sJ |j                  d   }|j                  d   }|j                  d   }||cxk(  r|k(  sJ  J |dkD  r!||k7  rt         j                  d| d| d	       t        j                  |j                  dd       }t        j                  |||fd
      }d|z  }| j                  j                  d      }t        j                  |dz   t         j"                  ||g|j%                         d      }| j                  j'                  || j(                         ||dz   dg}||j+                  |       n|j+                  d       |	"|j+                  d       |j+                  |	       t        j,                  d||g|      }d|_        |j0                  j3                  t        j4                  d|      g       |
0|j0                  j3                  t        j4                  d|
      g       | j6                  C|j0                  j3                  t        j4                  dt9        | j6                              g       |S )a  Create an Attention node.
        Args:
            mask_index (str): mask input
            q_matmul (NodeProto): MatMul node in fully connection for Q
            k_matmul (NodeProto): MatMul node in fully connection for K
            v_matmul (NodeProto): MatMul node in fully connection for V
            num_heads (int): number of attention heads. If a model is pruned, it is the number of heads after pruning.
            hidden_size (int): hidden dimension. If a model is pruned, it is the hidden dimension after pruning.
            input (str): input name
            output (str): output name
        Returns:
            Union[NodeProto, None]: the node created or None if failed.
        r   input hidden size # is not a multiple of num of heads Nr   zl is not an initializer. Please set do_constant_folding=True in torch.onnx.export to unblock attention fusionzInput hidden size (z3) is not same as weight matrix dimension of q,k,v (z:). Please provide a correct input hidden size or pass in 0)axis   	Attention_qkv_weightTname	data_typedimsvalsraw inputsoutputsr4   com.microsoftr   r*   mask_filter_value)loggerdebugr   get_initializerr'   printr
   to_arrayshapewarningnpprodstackcreate_node_namer   make_tensorr   FLOATtobytesadd_initializerthis_graph_nameappend	make_nodedomain	attributeextendmake_attributer>   float)r   r#   r$   r%   r&   r   r   r'   r(   r)   r*   q_weightk_weightv_weightqwkwvw
qw_in_size
kw_in_size
vw_in_sizeqw_out_size
qkv_weightqkv_weight_dimattention_node_nameweightattention_inputsattention_nodes                              r!   create_attention_nodez'FusionT5Attention.create_attention_node*   sA   4 1}}?i 7A=LL-k]:]^g]hij::--hnnQ.?@::--hnnQ.?@::--hnnQ.?@>>!$% &g g !!(+!!(+!!(+ xx288###XXa[
XXa[
XXa[
Z5:55555?{j8NN%k]2efpeq rJ J
 ggbhhqrl+XXr2rl3
["jj99+F##$}4!''n-##%
 	

""64+?+?@ -/

 !##J/##B'!##B'##J/))#H$	
 !0  '')>)>{I)V(WX$$++V-B-B7E-R,ST!!-$$++V-B-BCVX]^b^t^tXu-v,wxr"   querykeyvalueres_pos_biaspast_key
past_valuepresent_keypresent_valuec                    |dkD  sJ |dkD  r$||z  dk7  rt         j                  d| d|        y | j                  j                  d      }||dn||dn|dg}||j	                  |       n|j	                  d       ||j	                  |       n|j	                  d       |&|J |j	                  |       |j	                  |       |g}|	&|
J |j	                  |	       |j	                  |
       t        j                  d|||      }d|_        |j                  j                  t        j                  d|      g       |j                  j                  t        j                  d	d
      g       | j                  C|j                  j                  t        j                  dt        | j                              g       | j                  d       |S )Nr   r-   r.   MultiHeadAttentionr9   r:   r=   r   r*         ?r>   )r?   r@   r   rI   rO   r   rP   rQ   rR   rS   rT   r>   rU   increase_counter)r   rg   rh   ri   r#   rj   rk   rl   r(   rm   rn   r   r   rb   rd   attention_outputsre   s                    r!   create_mha_nodez!FusionT5Attention.create_mha_node   s    1}}?i 7A=LL-k]:]^g]hij"jj99:NO+B3-BU	
 !##J/##B'###L1##B')))##H-##J/#H" ,,,$$[1$$]3)) #%$	
 !0  '')>)>{I)V(WX  '')>)>w)L(MN!!-$$++V-B-BCVX]^b^t^tXu-v,wx23r"   c                 P    | j                  |||       | j                  |||       y N)fuse_t5_encoderfuse_t5_decoder)r   normalize_nodeinput_name_to_nodesoutput_name_to_nodes       r!   fusezFusionT5Attention.fuse   s*    ^-@BUV^-@BUVr"   c                 ^   |j                   dk7  r|j                   dk7  ry | j                  j                  |g dg d      }|y |\  }}}}| j                  j                  |g dg d      }	|	y |	d   }
| j                  j                  |g dg d      }|y |\  }}}| j                  j                  |g d	g d
      }|y |\  }}}d }| j                  j                  |g dg d      }|y |d   }|d   j                   dk7  ry | j                  j                  |      \  }}|dk7  r|| _        | j
                  j                  |d   j                  d         }d }| j                  j                  |ddgddg      }|y |d   }|j                  d   }| j                  j                  |g dg d      }|y |\  }}}| j                  j                  |g dg d
      }|y |\  }}}|j                  d   |
j                  d   k7  ry | j                  |      \  }} | j                  |||||| |
j                  d   |j                  d   |d
      }!|!y | j                  j                  |!       | j                  | j                  |!j                  <   | j                   j#                  |dd         | j                   j#                  |       | j                   j#                  |d d        || j                   j#                  |d d        | j                   j#                  |d d        d| _        y )Nr   r   MatMulReshape	Transposer   r   r   r   r   Concat	UnsqueezeGatherShaper   r   r   r   r   r   Softmaxr   r   r   r   r   r   MulSubCastr   r   r   r   r   r   r   r   r   r   r   RelativePositionBiasrq   T)op_typer   match_parent_pathget_constant_inputr>   r   process_maskr'   get_num_heads_and_hidden_sizerf   r(   nodes_to_addrO   rN   node_name_to_graph_namer4   nodes_to_removerS   prune_graph)"r   ry   rz   r{   	qkv_nodes_reshape_qkvtranspose_qkv
matmul_qkvqkv_shape_nodesinput_shape_nodev_nodes	reshape_vmatmul_vqk_nodesadd_qk	matmul_qkr#   
mask_nodesmul_nodemul_valrj   	rpb_nodesrpb_add_nodek_nodes	reshape_kmatmul_kq_nodestranspose_q	reshape_qmatmul_qq_num_headsq_hidden_sizenew_nodes"                                     r!   rw   z!FusionT5Attention.fuse_t5_encoder   s   !!%GGNLbLbfkLkJJ008
	
 4=1;z**666

 "*2.**...

 ?!(9h :://(

 '69
ZZ11C


 a=a=  E)ZZ228<
7f%,D"((55jn6J6J16MN
JJ00*+F
	
  |#))!,**...

 ?!(9h **...

 ?+2(Y >>! 0 6 6q 99%)%G%G	%R"]--""1%q!
   *6:6J6J$$X]]3##IabM2##H-##GCRL1  ''5##GCRL1r"   c                 r   |j                   dk7  r|j                   dk7  ry | j                  j                  |g dg d      }|y |\  }}}}| j                  j                  |g dg d      }	|	y |	d   }
d }d }d }| j                  j                  |g dg d      }|| j                  j                  |g d	g d
      }|X|\  }}}|j                  d   }|j                  d   }d|vry |j                  d   |
j                  d   k7  rd| _        nsd| _        nk|j                  d   }||v ry d|vry d| _        nJ|\  }}}}|j                  d   }||v ry d|vry |j                  d   }d|vry |j                  d   }d| _        | j                  j                  |g dg d      }|y |\  }}}d }d }| j
                  dk(  r| j                  j                  |g dg d      }|y |d   }|d   j                   dk7  ry | j                  j                  |      \  }}|dk7  r|| _        | j                  j                  |d   j                  d         }nf| j                  j                  |ddgddg      }||j                  d   }n3| j                  j                  |ddgddg      }|y |j                  d   }d }d }d }| j
                  dk(  r| j                  j                  |g d	g d
      }|o|\  } }!}|!j                  d   }||!j                  d      }"|"D ]:  }#| j                  j                  |#j                  d         }$|$.|$j                  } n |y d|vry | j                  j                  |dgdg      }|y |d   } | j                  d   }||v ry d|vry | j                  j                  |g dg dfg dg dfg|      \  }%}}d }&d }"||d   |d    }!}'|!j                  d   }|%dk(  r"||'j                  d      }&|&j                  d   }n|'j                  d   }||v ry d!|vry |%dk(  rR||'j                  d      }"|"D ]:  }#| j                  j                  |#j                  d         }$|$.|$j                  } n n|'j                  d   }|y d"|vry | j                  j                  |g d	g d
      }|y |\  }}!}|!j                  d   }||!j                  d      }"|"D ]:  }#| j                  j                  |#j                  d         }$|$.|$j                  } n |y d"|vry | j                  j                  |g d	g d      }(|(y |(\  })}*}+|+j                  d   |
j                  d   k7  ry | j                  |*      \  },}-| j
                  dk(  r
||}|}d }d }| j                  |+j                  d   |||||||j                  d   |||,|-      }.|.y | j                  j!                  |.       | j"                  | j$                  |.j                  <   | j&                  j)                  |dd         | j&                  j)                  |       | j&                  j)                  |d d        || j&                  j)                  |d d        | j&                  j)                  |(d d        d#| _        y )$Nr   r   r~   r   r   r   )r   r   r   r   )r   r   r   r   r   r   r   rn   r   past_value_crosspast_value_selfpresent_value_selfr   r   r   r   r   r   Slicer   present_key_crossr   past_key_cross)r   r   r   r   )r   r   r   r   )r   r   r   r   r   )r   r   r   r   r   past_key_selfpresent_key_selfT)r   r   r   r'   r(   r   r   r>   r   r   find_graph_outputr4   match_parent_pathsr   rt   r   rO   rN   r   r   rS   r   )/r   ry   rz   r{   r   r   r   r   r   r   r   ri   rl   rn   r   transpose_vr   r   concat_vr   r   r   r#   rj   r   r   r   r   rh   rk   rm   r   transpose_kr   present_key_transpose_nodespresent_key_transpose_nodepresent_key_candidateidxpast_key_transpose_nodeconcat_kr   r   r   r   r   r   r   s/                                                  r!   rx   z!FusionT5Attention.fuse_t5_decoderX  s   !!%GGNLbLbfkLkJJ008
	
 4=1;z**666

 "*2.
**..8

 ?jj222G
 "3:0Y!* + 2 21 5"-7>>!$(8(>(>q(AA%&DN%&DN'--a0
!44%Z7!"(/%HaA!*J00 
2$OOA.M#=8OOA&EDN:://(

 '69
>>Q55G"J
 !!!}H!}$$-66x@JAw& )0&,,99*R.:N:Nq:QRJ

44 AI
 $%||A JJ8823F	
 $%||A>>Qjj222G
 ",3)Yooa(.A)BRBRSTBU.V+2M.,0JJ,H,HIcIjIjklIm,n),8&;&@&@	 3N
 &&k9**66 MC
 ?%aj&,,Q/22#83"jj;;A<PNP_` $OC! '+#*.'"&-aj'"+)ooa(!8.A(..QRBS.T+6<<Q?H'~~a0H22"(2!82EhooVWFX2Y/6Q204

0L0LMgMnMnopMq0r-0<*?*D*DK!	 7R #+//!"4K&%[8**666
 ?")9aooa(.A)BRBRSTBU.V+2M.,0JJ,H,HIcIjIjklIm,n),8&;&@&@	 3N
 &%[8**...

 ?+2(Y>>! 0 6 6q 99%)%G%G	%R"]>>Q8#7CEHJ''OOAq!
   *6:6J6J$$X]]3##IabM2##H-##GCRL1  ''5##GCRL1r"   rv   )__name__
__module____qualname____doc__r   intr   r   strr   r   rU   r   rf   rt   r|   rw   rx   __classcell__r    s   @r!   r   r      se     	
 &8 "&kk k 	k
 k k k k k k k 
y$	kZ?? ? 	?
 ? ? ? ? ? ? ? ? ? 
y$	?BWz xA r"   r   c                   .     e Zd Zdedef fdZd Z xZS )FusionRelativePositionBiasBlockr   max_distancec                 H    t         |   |dddg       || _        d| _        y )Nr   r   r   F)r   r   r   is_bidirectional)r   r   r   r    s      r!   r   z(FusionRelativePositionBiasBlock.__init__]  s*     68HI( %r"   c           	         |j                   dk7  r|j                   dk7  ry | j                  j                  |g dg d      }|$| j                  j                  |g dg d      }|y |d   }|d   }|d	   }| j                  j                  |g d
g d      }|y |d   }	| j                  j                  |	g dg d      }
|
+| j                  j                  |	g dg d      }
d| _        |
y |
d   }| j                  j                  |       | j                  j                  |       | j                  j                  |
       | j                  rdnd}| j                  j                  |j                  d	         }t        j                  |      }t        j                  |      }t        j                  | j                  j                  d|      t        j                   t        j"                  |      d	   t        j"                  |      d   g|j%                         d      }| j                  j'                  || j(                         |j*                  |j                  d   |j                  d   g}|j,                  d	   g}t        j.                  d||| j                  j                  d|            }d|_        |j2                  j                  t        j4                  d| j6                        g       |j2                  j                  t        j4                  d| j                        g       | j8                  j;                  |       | j(                  | j<                  |j*                  <   y )Nr   r   )r   r   r   Where)r   r   r   r   )r   r   r   r   r   )r   r   r   r   r      r   r   )	MinConstantOfShaper   r   r   r   DivLogr   )	r   r   r   r   r   r   r   r   r   )r   Negr   r   r   r   r   Range)r   r   r   r   r   r   r   r   )r   Absr   r   r   )r   r   r   r   r   Tencoderdecoderbias_table_weight)name_prefixr   r3   r   r:   r=   r   r   )r   r   r   r   r   rS   rA   r'   r
   rC   rF   	transposer   rJ   rI   r   rK   rD   rL   rM   rN   r4   r(   rP   rQ   rR   rT   r   r   rO   r   )r   noderz   r{   compute_bias_nodesgatherwhere	unsqueezecompute_buckets_nodesdivrange_nodes
range_nodenode_name_prefixtable_weight_itable_weighttable_weight_t
bias_tabler;   r<   rpb_nodes                       r!   r|   z$FusionRelativePositionBiasBlock.fusec  s    <<5 T\\W%<!ZZ99?
 %!%!=!=JO" ")#A&"2&&q)	 $

 < <Z'!

 !(#B'jj22[$

 **66A?K %)D!" _
##$67##$9:##K0(,(=(=9933FLLOD"++N;l3'',,-@N^,_!''((<(+RXXl-CA-FG'')

 	

"":t/C/CD//:#3#3A#6
8H8H8KL##A&'##",,-CQa,b	
 *!!6#8#8IZIZ#["\]!!6#8#89KTMbMb#c"de  *6:6J6J$$X]]3r"   )r   r   r   r   r   r   r|   r   r   s   @r!   r   r   \  s    &i &s &LKr"   r   c                   H     e Zd Z fdZd Zd Zd Zd Zd Zd Z	d Z
 xZS )	T5OnnxModelc                    t         |   |||       t        |       | _        t	        | | j
                  | j                  | j                        | _        t        |       | _	        t        |       | _        t        | d      | _        y )N   )r   r   r   r   r   r   r   attention_fusionr   layer_norm_fusionr	   skip_layer_norm_fusionr   
rpb_fusion)r   r   r   r   r    s       r!   r   zT5OnnxModel.__init__  so    	;7+D1 1$8H8H$..Z^ZmZm n!CD!I&LT&R# :$Dr"   c                 8    | j                   j                          y rv   )r   applyr   s    r!   fuse_attentionzT5OnnxModel.fuse_attention  s    ##%r"   c                 8    | j                   j                          y rv   )r   r  r  s    r!   fuse_layer_normzT5OnnxModel.fuse_layer_norm  s    $$&r"   c                 8    | j                   j                          y rv   )r   r  r  s    r!   fuse_skip_layer_normz T5OnnxModel.fuse_skip_layer_norm  s    ##))+r"   c                 ^   g }| j                         D ]  }|j                  dk(  s| j                  |g dg d      }|-| j                  |dgdg      }|E|d   }|j                  d   |j                  d<   |j	                  |       |j                  |       | j                  |        y )Nr   )r   r   r   r   r   LessOrEqualTiler   r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   nodesr   r   r(   rS   rO   remove_nodesr   r   r   extended_mask_nodesr   r   s         r!   !remove_extended_mask_decoder_initz-T5OnnxModel.remove_extended_mask_decoder_init  s    JJLD||u$&*&<&< 6'#" '. 224:P9QTUSVW	$$Q<%)[[^"&&':;&&t,!!/2? !r"   c                 b   g }| j                         D ]  }|j                  dk(  s| j                  |g dg d      }|-| j                  |ddgddg      }|G|d   }|j                  d   |j                  d<   |j	                  |       |j                  |       | j                  |        y )Nr   )r   r   r   r   r   r   r  r  r   r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  s         r!   remove_extended_mask_decoderz(T5OnnxModel.remove_extended_mask_decoder  s    JJLD||u$&*&<&< 9!'#$ '. 224'CY9Z]^`a\bc	$$Q<%)[[^"&&':;&&t,!!/2A !r"   c                 X    | j                          | j                  j                          y rv   )adjust_reshape_and_expandr  r  r  s    r!   
preprocesszT5OnnxModel.preprocess  s    &&(r"   c                 d    | j                          | j                          | j                          y rv   )r  r  r   r  s    r!   postprocesszT5OnnxModel.postprocess  s&    ..0))+r"   )r   r   r   r   r  r  r	  r  r  r  r  r   r   s   @r!   r   r     s.    E&',!3F"3H r"   r   )loggingtypingr   r   numpyrF   fusion_attentionr   r   fusion_baser   fusion_simplified_layernormr   r	   fusion_utilsr
   onnxr   r   r   
onnx_modelr   onnx_model_bertr   	getLoggerr   r?   r   r   r    r"   r!   <module>r&     sh   
  "  ;  r $ / /   )			8	$E	  E	 PSKf SKle- er"   