
    g}p                     z    d dl Z d dlZd dlmZmZ d dlmZmZ d dl	m
Z
  e j                  e      Z G d de      Zy)    N)AttentionMaskFusionAttention)TensorProtohelper)	OnnxModelc                   F     e Zd ZdZdedededef fdZd Zd Z	d	 Z
 xZS )
FusionBartAttentionz?
    Fuse Bart Attention subgraph into one Attention node.
    modelhidden_size	num_headsattention_maskc                 *    t         |   ||||       y )N)super__init__)selfr
   r   r   r   	__class__s        c/var/www/openai/venv/lib/python3.12/site-packages/onnxruntime/transformers/fusion_bart_attention.pyr   zFusionBartAttention.__init__   s     	YG    c                    | j                   j                  |dgdg      }|y|d   }| j                   j                  |g dg d      }	| j                   j                  |g dg d      }
|	|
y|	\  }}}|
\  }}}|j                  d   |k7  s|j                  d   |k7  ry| j                   j                  |g dg d      }| j                   j                  |g dg d	      }||y|d
   j                  |j                  k7  s|d
   j                  |j                  k7  ry| j                   j                  |g dg d      }| j                   j                  |g dg d      }| j                   j                  |g dg d      }|||y|d
   }|d
   }|d
   }|j                  d   }|j                  d   |k7  s$|j                  d   |k7  s|j                  d   |k7  ryy)NConcat   Fr   	UnsqueezeGatherShaper   r   r   )r   r   r   )r   r   r   )r      r   )r   r   MulT)r
   match_parent_pathinputnameoutput)r   reshape_qkv_2reshape_qkv_1reshape_q_2reshape_k_2reshape_v_2
root_inputconcat_qkv_2_pathconcat_qkv_2reshape_qkv_2_path_1reshape_qkv_2_path_2_gather_1shape_1gather_2shape_2reshape_qkv_1_path_1reshape_qkv_1_path_2reshape_q_2_pathreshape_k_2_pathreshape_v_2_pathmul_qmul_kmul_vgather_1_outs                            r   check_runtime_shape_pathz,FusionBartAttention.check_runtime_shape_path   s    !JJ88
UVTWX$(+#zz;;LJjluv#zz;;LJjluv'+?+G38W38W==z)W]]1-=-K#zz;;MKlnwx#zz;;MKlnwx'+?+G#((HMM9=QRT=U=Z=Z^f^k^k=k::77Eceno::77Eceno::77Eceno#'7'?CSC[ $ $ $q);;q>\)U[[^|-Ku{{[\~amOmr   c                    | j                   j                  |g dg d      }|y|d   j                  d   |j                  d   k7  ry| j                   j                  |g dg d      }| j                   j                  |g dg d      }||y|d   }	|d   }
|	j                  d	   |
j                  d	   k7  ry|d   j                  d   |j                  d   k7  r#|d   j                  d   |j                  d   k7  ry|| j                   j                  |d
gd	g      }|y| j                   j                  |d   g dg d      }| j                   j                  |d   g dg d      }||y|\  }}}}|\  }}}|j                  d   |j                  d   k7  ry|d   j                  d   |j                  d   k7  r#|d   j                  d   |j                  d   k7  ryy)N)r   Slicer   r   r   r   r   r   Fr   r   )r   PowCastDivr   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r>   )r>   r   r   r   )r   r   r   r   r   )r   r   r   T)r
   r    r!   r#   )r   r$   
matmul_qkvadd_qk	matmul_qkadd_qreshape_qkv_2_pathmatmul_qk_path_1matmul_qk_path_2mul_1mul_2add_qk_pathslice_q_path_1slice_q_path_2r.   unsqueeze_1unsqueeze_2s                    r   check_runtime_shape_path_openaiz3FusionBartAttention.check_runtime_shape_path_openaiM   s    "ZZ99A<
 %!"%++A.*2C2CA2FF::77GI[
  ::77GI[
 #'7'? # #;;q>U[[^+B%%a(ELLO;@PQS@T@Z@Z[\@]afamamnoap@p **66vy1#NK"!ZZ99A I<N "ZZ99+a.JjluvN%.*@#1 A{Aq .KA  #{'8'8';;b!''*ell1o=.QSBTBZBZ[\B]afamamnoapBpr   c                 ^   d}| j                   j                  |g dg d      }| j                   j                  |g dg d      }|
|\  }}}	}
}}n||}|\  }}}	}
}d}ny g }|j                  D ].  }||vr||d   j                  d   k(  r|j	                  |       0 t        |      dk7  ry |d   }	 ||   }|j                  d	k(  r| j                   j                  |      d   }|j                  D ];  }|s||   }|D cg c]  }|j                   }}|j                  d
      dk\  s9|} n t        | j                   j                         j                  D cg c]  }|j                   c}      }t        | j                   j                         j                  D cg c]  }|j                   c}      }| j                   j                  |g dg d      }| j                   j                  |g dg d      }| j                   j                  |g dg d      }| j                   j                  |dgdg      }| j                   j                  |g dg d      }d\  }}d\  }} ||\  }}!}"} }#|!j                  d   }n<||}|\  }!}"} }#| j                   j                  | ddg|"g      }$| j                   j                  | g d      }%|$%|$\  }&}'|'j                  d   |v r|'j                  d   }|%|%\  }(}&})|)j                  d   |v r|)j                  d   }| j                   j                  |(ddgddg      }*|*\  }&}+|+j                  d   }nk|+|\  }}(}!}"} }#|}|(j                  d   }|(j                  d   }n>||d   j                  d   |v r~|}|d   j                  d   }|d   j                  d   }||vrt        t        d | j                   j                         |               },t        |,      dk(  r|,d   j                  d   nd}n||d   j                  d   |v r~|}|d   j                  d   }|d   j                  d   }||vrjt        t        d | j                   j                         |               },t        |,      dk(  r|,d   j                  d   nd}nt         j#                  d       y ||v r|nd}||v r|nd}| j                   j                  |dd
gddg      }-| j                   j                  |g dg d       }.| j                   j                  |g d!g d"      }/d }0|-|-\  }&}1|-}2n|.|.\  }&}&}0}&}1|.}2n|/	|/\  }&}0}1|/}2ny | j                   j                  |1g d#g d$      }3| j                   j                  |1g d%g d&      }4d }5|3
|3\  }5}6}7}8}9}:n|4|4}3|3\  }8}6}7}9}:ny | j                   j                  |1g d'g d(      };| j                   j                  |1g d)g d      }<| j                   j                  |1g d*g d+      }=| j                   j                  |1g d,g d-      }>| j                   j                  |1ddgddg      }?| j                   j                  |1g d.g d+      }@d\  }A}Bd/\  }C}D}E|;|;\  }&}C}F}D}G}E|;}HnS|<|<\  }I}F}D}E|<}H|Ej                  d   }B| j                   j                  |Eddg|Dg      }$| j                   j                  |Eg d      }%|$%|$\  }&}J|Jj                  d   |v rJj                  d   }B|%|%\  }K}&}L|Lj                  d   |v rLj                  d   }B| j                   j                  Kddgddg      }*|*\  }&}M|Mj                  d   }Ans|=|=\  }&}C}F}D}E|=}H|Fj                  d   }BnV|>+|>\  }&}C}K}&}D}E|>}H|Kj                  d   }A|Kj                  d   }Bn)|?|?d   j                  d   |v r~|?}H|Hd   j                  d   }A|Hd   j                  d   }B|B|vrt        t        d0 | j                   j                         A               }Nt        |N      dk(  rNd   j                  d   nd}Bn@@d   j                  d   |v r~@}H|Hd   j                  d   }A|Hd   j                  d   }B|B|vrUt        t        d1 | j                   j                         A               }Nt        |N      dk(  rNd   j                  d   nd}Bny A|v rAnd}AB|v rBnd}BH|<|=|>fv r| j                   j%                  | j                  d         j&                  d   }Od2}P| j                   j%                  |P      }Q|QK| j)                  Pt*        j,                  Ogt/        j0                  d3g|Oz  t.        j2                  4      5       | j                   j5                  d	      }Rt7        j8                  d	PEj                  d   gDj                  g|R      }G|rAs| j;                  |	||0|1|9      sy |sAs| j=                  |	|5C||      sy Axr |xr Ed u xr d6t?               v}S|S xr: Ej                  d   |k(  xr& |:j                  d   |k(  xr #j                  d   |k(  }TS xrT |:j                  d   |k(  xr@ Ej                  d   #j                  d   k(  xr Ej                  d   |:j                  d   k7  }UTxr |2|-k(  }VTxr |2|.|/fv }W|sVnWxr Axr |}XUxr |2|-k(  }YSxr |2|-k(  }Zd }[Wri| j                   j                  |0d7gdg      }\| j                   j                  |0g d8g d      }]|]]d   j                  d   }[n\\d   j                  d   }[Vs	WsXsYsZr|	}^| jA                  |7      \  }_}`|_dk  s`dk  s`_z  dk7  rt         j#                  d9       y d }aXsYsZr[| jB                  rL| jE                  |:YsXrEnAYsXr#n||9YsXrGnd YsXr| nd _`^j                  d   XrAndXr|ndB|X:      nd }anL| jB                  }bd| _!        | jG                  d |:E#|9G| _`|^j                  d   Wr[nd A|B|;      }ab| _!        ay | jH                  j	                  a       | jJ                  | jL                  |aj                  <   | jN                  jQ                  ^|
|g       | jN                  jQ                  |2       XsYsZr|3d   j                  d
k(  r|3jS                          Hd   j                  d
k(  rHjS                          |d   j                  d
k(  r|jS                          | jT                  rjYsZrf|3d   j                  d	k(  r|3jS                          Hd   j                  d	k(  rHjS                          |d   j                  d	k(  r|jS                          | jN                  jQ                  |3       | jN                  jQ                  H       | jN                  jQ                  |       d| _+        y y c c}w c c}w c c}w )<NF)AddMatMulReshape	TransposerV   rU   rC   )rT   rU   rV   rW   rU   )r   r   r   r   r   Tr   r   rT   rU   )rV   rW   rV   rT   rU   )r   r   r   r   N)rW   rV   rT   rU   )r   r   r   N)rV   r   rW   rV   rT   rU   )r   r   r   r   r   NrV   )rW   rV   rV   rW   r?   ) rX   )NNrW   )exclude)r   rV   rW   r   c                      | j                   dk(  S NIdentityop_typenodes    r   <lambda>z*FusionBartAttention.fuse.<locals>.<lambda>       
(Br   rX   c                      | j                   dk(  S r[   r]   r_   s    r   ra   z*FusionBartAttention.fuse.<locals>.<lambda>,  rb   r   z&fuse_attention: failed to match v pathSoftmax)rd   rV   rT   rV   rU   )r   r   r   r   r   )rd   rT   rU   r   )rV   rW   rV   r   rT   rU   )r   r   r   r   r   r   )r   rW   rV   rT   rU   )r   r   r   r   r   )rW   rV   rW   rV   rT   rU   )r   r   r   r   r   r   )r   rW   rV   rU   )rW   rV   rW   rV   rU   )r   r   r   r   r   )rW   rV   r   rW   rV   rU   )r   r   r   r   r   r   )r   rW   rV   rV   rW   )NNNc                      | j                   dk(  S r[   r]   r_   s    r   ra   z*FusionBartAttention.fuse.<locals>.<lambda>  rb   r   c                      | j                   dk(  S r[   r]   r_   s    r   ra   z*FusionBartAttention.fuse.<locals>.<lambda>  rb   r   
empty_biasg        )dtype)dimsvalsmatmul_vWhere)Expandr   r   rl   z9fuse_attention: failed to detect num_heads or hidden_size)past_kpast_v	present_k	present_v
packed_qkv)
add_qk_strrn   ro   rp   rq   ),r
   r    r!   r#   appendlenr^   get_childrencountsetgraphr"   match_child_pathlistfilterinput_name_to_nodesloggerdebugget_initializerri   add_initializerr   FLOATnparrayfloat32create_node_namer   	make_noderR   r<   localsget_num_heads_and_hidden_sizeuse_multi_head_attentioncreate_multihead_attention_nodecreate_attention_nodenodes_to_addthis_graph_namenode_name_to_graph_namenodes_to_removeextendpop!disable_multi_head_attention_biasprune_graph)cr   normalize_noder}   output_name_to_nodemodel_impl_openai	qkv_nodesqkv_nodes_openaiadd_out
matmul_outr$   transpose_qkvr%   rD   other_inputsr!   r)   skip_layernormr#   childrenchildchildren_typesr`   graph_input_namesgraph_output_namesv_nodesv_nodes_openaiv_nodes_with_past_self_attnv_nodes_with_past_cross_attn#v_nodes_with_past_cross_attn_openairo   rq   r(   add_vtranspose_vreshape_v_1rk   reshape_pathconcat_pathr.   transpose_add_vconcat_vtranspose_concat_vconcat_nodestranspose_concat_v_inidentity_node_v
qk_nodes_1
qk_nodes_2qk_nodes_2_openairE   rF   qk_nodesq_nodesq_nodes_openair&   transpose_qreshape_q_1r8   rG   matmul_qk_nodes_with_biask_nodes_with_bias_openaik_nodes_no_bias#k_nodes_no_bias_with_past_self_attn$k_nodes_no_bias_with_past_cross_attn+k_nodes_no_bias_with_past_cross_attn_openairn   rp   r'   reshape_k_1matmul_ktranspose_k_1add_kk_nodesr9   transpose_matmul_kconcat_ktranspose_concat_ktranspose_concat_k_inidentity_node_kbias_dimempty_bias_nameempty_tensoradd_namethree_root_inputsone_root_inputtwo_root_inputsencoder_attentiondecoder_attentiondecoder_attention_with_pastdecoder_cross_attention!decoder_cross_attention_with_past
mask_indexmask_nodes_bartmask_nodes_whisperattention_last_noder   r   new_node%use_multi_head_attention_ground_truthsc                                                                                                      r   fusezFusionBartAttention.fuse   sY   ! JJ00J
	
  ::77?

   )(I  !%#))E//	!++A..& * |!!!_
	 -Z8 !!U*!ZZ44^DQGN$++F*62H9ABemmNB##H-2#
 ,  tzz7G7G7I7O7O P7Ot7O PQ 

8H8H8J8Q8Q!R8Q$))8Q!RS**..@

 555

 '+jj&B&BJ!	'
# (,zz'C'CKC	(
$ /3jj.J.J</
+
 #	'UGND[+{E8#**1-I'$G:A7[+uh  ::66K($ 7 L **55e=_`K''3$O"))!,0BB / 6 6q 9I&4?110%,,Q/3EE 2 9 9! <I#zz;;HyR]F^abde`fg+7((.44Q7(4QlN[(KeX1G^^A&F *I(49UVX9Y9_9_`a9bfw9w2GR[&&q)F**1-I 22"&BDJJDbDbDdekDlm# =@<PTU<UOA.55a8[]	/;3B7==a@DUU9GR[&&q)F**1-I 22"&BDJJDbDbDdekDlm# =@<PTU<UOA.55a8[]	LLAB!%66B!*.@!@Ib	ZZ11*y(>SVWYZU[\
ZZ11JO

 !JJ88Eaclm!%LAy!H#)3&Aq&!Y!H*#4 Avy(H**..G

 55<

 LSIKk5%'$G?F<E;UH JJ88M

 $(::#?#?5$
 
 **66F

 /3jj.J.JP	/
+ 04zz/K/K)$F	0
, 7;jj6R6RC	7
3 #	-=*[((J[GA{M;x'G%1:R7E=+x.G *I  ::66K($ 7 L **55h@bcK'*6'&%,,Q/3EE 2 9 9! <I&4?110%,,Q/3EE 2 9 9! <I#zz;;HyR]F^abde`fg+7((.44Q7(CR@A{M;%G%,,Q/I0<Ad>A{Hah9G^^A&F *I0<4R8>>qAEVV:GR[&&q)F**1-I 22"&BDJJDbDbDdekDlm# =@<PTU<UOA.55a8[]	7C;B?EEaHL]]AGR[&&q)F**1-I 22"&BDJJDbDbDdekDlm# =@<PTU<UOA.55a8[]	!%66B!*.@!@Ib	/Beffzz11%++a.AFFqIH*O::55oFL#$$#%%"3%("2"**E	 %  zz2259H$$U_hooa>P,QT_TdTdSegopE 88 !11 "ava(d2BazY_YaGa!! 0q!Z/0q!Z/0 q!Z/	 	 "! 7q!Z/7q!X^^A%667 q!X^^A%66	 	 +Ex:/E*ZxJHY;Z/Z&7=NeTZe_e 	$ #2"Lh*6L,=,X(jBX) 
"jj::	O
 "&!=!=="
 "-/299"=
 ,,Q/66r:
  *&0"/%)%G%G%T"I{A~!1kI6MRS5SXYH*.EIj* 44! 88 $;?Z`f$;?Z`f!8<W]a!8<W]a!#+2215)Dv")Dv""+"+#> 9 " % , 9=8U8U505-55'..q1->zD!!''! 6 $ 1V-$$X.:>:N:ND((7  '')<mZ(XY  ''1 +.EIj2;&&(2KKM2;&&(2KKM2;&&(2KKM99+/Pr{**e3r{**e3r{**e3  ''0  ''0  ''0  $Ds 1A C
 !Q!Rs   x x%!x*)__name__
__module____qualname____doc__r   intr   r   r<   rR   r   __classcell__)r   s   @r   r	   r	      sJ    HH H 	H
 &H.`3j~$r   r	   )loggingnumpyr   fusion_attentionr   r   onnxr   r   
onnx_modelr   	getLoggerr   r~   r	    r   r   <module>r      s6   
   ; $  			8	$q	$/ q	$r   