
    ga                     v    d dl Z d dlZd dlZd dlmZmZmZ d dlmZ  e j                  e
      Z G d de      Zy)    N)TensorProtohelpernumpy_helper)BertOnnxModelc                   r     e Zd Z fdZd Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zd Zd Z xZS )BertOnnxModelTFc                 (    t         |   |||       y N)super__init__)selfmodel	num_headshidden_size	__class__s       `/var/www/openai/venv/lib/python3.12/site-packages/onnxruntime/transformers/onnx_model_bert_tf.pyr   zBertOnnxModelTF.__init__   s    	;7    c                 n   g }| j                         D ]n  }|j                  dk(  s| j                  |j                  d         r2| j	                  |j                  d   |j
                  d          |j                  |       p | j                  |       t        j                  dt        |              y )NIdentityr   zRemoved Identity count: )nodesop_typefind_graph_outputoutputreplace_input_of_all_nodesinputappendremove_nodesloggerinfolen)r   nodes_to_removenodes      r   remove_identityzBertOnnxModelTF.remove_identity   s    JJLD||z)--dkk!n=33DKKNDJJqMR#**40	 !
 	/*.s?/C.DEFr   c                     | j                  |g dg d      }||S | j                  |g dg d      }||S | j                  |g dg d      }|S )N)MulSubReshapeCast)   Nr)   r   )r%   r&   r(   Slice	Unsqueeze)r)   r   r)   r   r   )r%   r&   r(   r+   r+   )r)   Nr)   r   r   )match_parent_path)r   add_or_sub_before_softmax
mask_nodess      r   match_mask_pathzBertOnnxModelTF.match_mask_path   sq    ++%-


 !++%8


 !++%<

 r   c                    | j                  |g       }i }|D ]n  }|j                  D ]]  }| j                  |      }|st        j                  |      }t        |j                        dk(  sE|j                  ||j                  <   _ p |S )zi
        Find initializers that is 2D. Returns a dictionary with name as key and shape as value.
           )get_parent_subgraph_nodesr   get_initializerr   to_arrayr    shapename)r   current_nodeparent_nodesinitializersr"   r   initializertemps           r   )get_2d_initializers_from_parent_subgraphsz9BertOnnxModelTF.get_2d_initializers_from_parent_subgraphs7   s     55lBG D"2259'00=D4::!+9=[%5%56 $ ! r   c                    | j                         }||vry ||   }t        |      dk7  ry | j                  |d   d      }t        |      dkD  rt        d|       y t        |      dk(  r|d   |k7  r|d   S |d   j                  d   }| j                  |d   g dg dfg dg d	fgd       \  }}}|r|r||d
   j                  d   k(  rt        j                  d       |d   }	| j                  |	      j                  }
| j                  t        j                  d|gdg      |
       t        j                  |	j                  d         }| j                  t        j                  ddgdg|      |
       d}|S )Nr)   r   T	recursivez(Found multiple candidates of segment_ids)ConstantOfShaper(   Concatr*   r(   Shape)r)   r   r   r   r   r   r@   r(   rA   r+   Squeezer*   r(   rB   )r)   r   r   r   r   r   r   r   Simplify semgent id path...rB   input_shapeinputsoutputsr@   zeros_for_input_shaperI   rJ   value)input_name_to_nodesr    get_graph_inputsprintr   match_parent_pathsr   debugget_graph_by_noder6   add_noder   	make_nodeget_attribute_value	attribute)r   segment_embedding	input_idsrN   r   graph_inputssegment_ids_segment_id_pathconstantofshape_node
graph_nameconstantofshape_values               r   find_segment_idsz BertOnnxModelTF.find_segment_idsG   s   "668$77#$56u:?,,U1X,F|q <lK|!l1o&B?" AhnnQ' $ 7 7!H T&
	 -& +!
?A0 yY/":M:S:STU:V-VLL67#21#5 //0DEJJJMM  )}oV %+$>$>?S?]?]^_?`$a!MM  %)?45/	  2Kr   c                     | j                         }||vry ||   }t        |      dk7  ry | j                  |d   d      }t        |      dk(  r|d   S t        d|       y )Nr)   r   Tr>   z&Found multiple candidates of input_ids)rN   r    rO   rP   )r   word_embeddingrN   r   rZ   s        r   find_input_idszBertOnnxModelTF.find_input_ids   ss    "668!44#N3u:?,,U1X,F|!?"6Er   c           	         | j                         D ]  }|j                  dk(  s| j                  |g dg d      }|.|\  }}}}}}	| j                  |d      sJ| j                  |d      s]| j	                  |d      }
|
D cg c]	  }||vs| }}t        |      dkD  rt        d|        y t        |      dk(  r|d	   c S | j                  |d
   g dg d      }|
D cg c]	  }||v s| }}|r|rt        |      dk(  r|d	   |d
   j                  d	   k(  rt        j                  d       |d	   }t        j                  |j                  d	         }| j                  |      j                  }| j                  t        j                   d|d	   gdg      |       | j                  t        j                   ddg|	j                  d	   g|      |       |	j                  d	   c S  y c c}w c c}w )NSoftmax)Addr%   r&   r(   r*   r+   )r   r)   Nr)   r   r   ir)   Tr>   z'Found multiple candidates of mask inputr   rE   rC   )r   r   r   r   r   r   r   r   rF   rB   input_shape_for_maskrH   r@   rL   )r   r   r,   has_constant_inputrO   r    rP   r   r   rR   r   rV   rW   rS   r6   rT   rU   )r   excluded_graph_inputsr"   	mask_pathrT   mul_nodesub_node	cast_node
slice_nodeunsqueeze_noderZ   r   rI   path_to_be_simplifiedduplicated_inputsr^   r`   r_   s                     r   find_mask_inputzBertOnnxModelTF.find_mask_input   s;   JJLD||y( 22G)	
 $ "**8V<AXAXYacdAe#'#8#8T#8#RL1=dNcAceFd6{QGP#6{a'%ay(,0,B,B!"	 1-) =I(kL5EUjLjL%(k .- 12a7-a04I"4M4S4STU4VV%BC/DQ/G,060J0JK_KiKijkKl0m-%)%;%;<P%Q%V%V
",, '(9!(<'=)?(@
 ' ",, 1(>'?)7)=)=a)@(A&;	 ' *//22K !L g e* )ls   	G-G-$	G2.G2c                    | j                  |      }|t        j                  d       y| j                  ||      }|t        j                  d       y| j	                  ||g      }|t        j                  d       y|||g| _        | j                  d      }| j                  j                  ||       | j                  |      j                  j                  j                  t        j                  k7  r| j                  j!                  |      \  }	}| j                  |      r| j                  j!                  |      \  }	}n| j                  j#                  |      \  }}
| j                  |      r| j                  j!                  |      \  }	}n| j                  j#                  |      \  }}| j                  d      }t$        j&                  j)                  d||||||j*                  d   |j*                  d	   |g||gd
      }d|_        | j/                  |j0                  d   |       | j3                  || j5                  |      j6                         y )Nz6Failed to find input_ids. Cannot fuse embedding layer.Fz8Failed to find segment_ids. Cannot fuse embedding layer.z7Failed to find input_mask. Cannot fuse embedding layer.
mask_indexembed_outputEmbedLayerNormalizationr)   r1   
EmbedLayerrI   rJ   r6   zcom.microsoftr   )rd   r   r   ra   rs   bert_inputscreate_node_nameattention_maskset_mask_indicefind_graph_inputtypetensor_type	elem_typer   INT32utilscast_graph_input_to_int32cast_input_to_int32onnxr   rU   r   domainr   r   rT   rS   r6   )r   normalize_noderc   rX   position_embeddingrY   r[   
mask_inputru   castedsegment_id_cast_nodemask_input_cast_noderv   
embed_nodes                 r   create_embedding_subgraphz)BertOnnxModelTF.create_embedding_subgraph   s'   ''7	KKPQ++,=yIKKRS));	*BC
KKQR%{J?**<8
++J
C  +00<<FF+J[J[[ $

 D DY OFI  -"&**"F"F{"SFK04

0N0N{0[-K-  ,!%!E!Ej!QFJ/3zz/M/Mj/Y,J,,,^<[[**%"!$$Q'$$Q'	 ":. + 

 ,
''(=(=a(@,Oj$"8"8"H"M"MNr   c                 `   t         j                  d       | j                         }| j                  d      }|D ]  }| j	                  |g dg d|      }||\  }}}| j                  |j                  d         }|Ft        j                  |      }	t        |	j                        dk(  r<t         j                  d|j                   d	|	j                          |j                  }
n1t         j                  d
|j                   d	|	j                           y| j                  |d|      }||j                  dk(  s| j                  |      }t        |      dk7  rt         j                  d|         yd}d}|j!                         D ]H  \  }}|d   dk(  r|}t         j                  d| d	|        ,|}t         j                  d| d	|        J ||t         j                  d        yt         j                  d       | j#                  ||||
       | j%                           y y)zM
        Automatically detect word, segment and position embeddings.
        z#start processing embedding layer...LayerNormalization)rg   r'   r*   )r   r)   r   Nr   r1   zFound position embedding. name:z, shape:z(Failed to find position embedding. name:rg   zFFailed to find two embeddings (word and segment) from Add node. Found zFound segment embedding. name:zFound words embedding. name:z.Failed to find both word and segment embeddingzCreate Embedding node)r   r   output_name_to_nodeget_nodes_by_op_typer,   r3   r   r   r4   r    r5   r6   
get_parentr   r<   warningitemsr   prune_graph)r   r   layer_norm_nodeslayer_norm_nodepos_embed_pathrT   reshape_nodero   r:   r;   r   first_parent
embeddingsrc   rX   r6   r5   s                    r   process_embeddingz!BertOnnxModelTF.process_embedding  s;    	9:"668445IJ/O!33+#	N %1?.HlJ..z/?/?/BCK"((5D4::!#=k>N>N=OxX\XbXbWcde%0%5%5"F{GWGWFXX`aeakak`lmn??8Q8KLL'L,@,@E,I!KKLY
z?a'NN`ak`lm !%$(!#-#3#3#5KD%Qx1},0)&DTF(SXRY$Z[)-&B4&QVPW$XY $6 ")->-FKK PQ34..#"%&	   "o  0r   c                     |||fD ]E  }|j                   d   }||   }||k(  rt        j                  d| d|j                  d            y y)Nr   zCheck attention input failed:z, FT)r   r   rR   r   )	r   matmul_qmatmul_kmatmul_vparentr   x
root_input	root_nodes	            r   check_attention_inputz%BertOnnxModelTF.check_attention_inputW  sb    Hh/AJ+J7IF"LL8Bv}}UVGWFXYZ 0 r   c                    | j                         }g }d}g }| j                  d      }| j                  d      }|j                  |       |j                  |       |D ]  }| j                  |      j                  }|j
                  dk(  r| j                  |dd      }	|	|	}nG| j                  |d      }
|
|
j
                  dvr8| j                  |d      }
|
|
j
                  dvrt        j                  d       | j                  |g dg d	      }|I| j                  |g d
g d      }|0| j                  |g dg d      }|t        j                  d       |d   }| j                  |g dg d      }|0| j                  |ddgddg      }|t        j                  d       Q|d   }|d   }| j                  |g dg d      }|0| j                  |g dg d      }|t        j                  d       |d   }| j                  |g dg d      }|0| j                  |ddgddg      }|t        j                  d       |d   }|d   }| j                  |g dg d      }|0| j                  |g dg d      }|t        j                  d       E|d   }|d   }| j                  |d         }|t        j                  d       || j                  |d   d      st        j                  d       | j                  |d   dgdg      xs | j                  |d   dgdg      }d }|d!z   }|t        |      d"k(  rr| j                  |d   j                  d         Q|d   j                  d   }| j!                  t#        j$                  d|g|g|dg#      |       ||d   j                  d<   | j'                  ||||
|      }|r$| j(                  j+                  |d   j                  d         }t        j                  d$       | j,                  j/                  |||||||| j0                  | j2                  |
j4                  d   |d%   j4                  d   d       }||d   j
                  dk(  r(t#        j6                  |d   j                  d&z   t8        j:                  d'gt=        j>                  dd| j0                  tA        | j2                  | j0                  z        gg      jC                         d()      } | jE                  | |       t#        j$                  d*|j4                  d   |d   j                  d&z   g|d   j                  d+z   g|d   j                  d,z   -      }!|d   j                  d+z   |d   j                  d<   | j!                  |!|       |
j
                  d*k(  rtG        jH                  | jK                  |
j                  d               d   }"t#        j6                  |
j                  d.z   t8        j:                  d/gt=        j>                  dd|"gg      jC                         d()      } | jE                  | |       |
j                  d.z   |
j                  d<   | j!                  ||       |dz  }|j                  |d%d         |j                  |       |j                  |       |j                  |       |j                  |       |j                  |       t        j                  d0        | jM                  |       | jO                          t        jQ                  d1|        y )2Nr   SkipLayerNormalizationr   rg   r)   )r   r   r'   z(Failed to match parent of normalize_node)rg   MatMulr'   	Transposer   )r   r   r   r   r   )r   r'   r   r   )r)   r   r   r   )rg   Einsumr   )r   r   r   zFailed to match qkv nodesrE   )r   r'   rg   r   r   zFailed to match v path)rf   rg   r%   r   )r   r   r   r   )rf   rg   r   zFailed to match qk_pathszFailed to match q path)r%   rg   r   )r)   r   r   zFailed to match k pathzCannot find mask_nodes.z;Sub node expected to have an input with constant value 1.0.rD   ExpandSqueeze_3d_to_2d_mask_output   )axeszCreate an Attention node.r1   	_newshape   T)r6   	data_typedimsvalsrawr'   _reshape_output_reshapery   	_modified   zRoot node not matched.zFused Attention count:))r   r   extendrS   r6   r   match_parentr   r   rR   r,   r/   ri   r    r~   r   rT   r   rU   r   r|   process_maskattention_fusioncreate_attention_noder   r   r   make_tensorr   INT64npint64inttobytesadd_initializerr   r4   r3   r   update_graphr   )#r   r   r!   attention_countstart_nodesskip_layer_norm_nodesr   r   r_   add_before_layernormr   	qkv_nodes
matmul_qkvv_nodesadd_vr   qk_nodes	matmul_qkq_nodesadd_qr   k_nodesadd_kr   r.   squeeze_nodesqueeze_node_namesqueeze_output_namer   is_same_rootru   attention_nodetensorreshape_r   s#                                      r   fuse_attentionzBertOnnxModelTF.fuse_attentionb  sx   "668 $ 9 9:R S445IJ 	01+,)N//?DDJ%%)=='+'8'8PQ'R$'3%9N__^Q7F~ 8 "
 ;>V^^ < &
 LL!KL..CI
   22"@ 	
 $ $ 6 6~Gbdm nI (%@A "2J,,Z9bdpqG00eX=NQRTUPVW?LL!9:BKEr{H--j:]_klH11*>Z\ef#LL!;< I,,Y8acopG00UH<MPQSTvV?LL!9:BKEr{H,,Y8acopG00<TV_`?LL!9:BKEr{H--hqk:J!67**:a=!<Z[  11*R.9+PQsS W[WmWm2
QCXL !8"3i"?#J1(<AVAVWabdWeWkWklmWnAoAw'^11!4
$$!#,-)S 	 +>
2$$Q'55h(TZ\opL!00==jn>R>RST>UV
89 "&!6!6!L!LNN$$MM!$aL''*" ")Q<''83#//&q\..<"-"3"3SXX %&$%$(NN$'(8(84>>(I$J	!"	 "') F  ((<%//!*11!4%aL--;  "+1!2!25F!F G&q\..; H -6aL,=,=@Q,QIaL&&q)MM(J7>>Y."."7"78L8LV\\Z[_8]"^_`"aK#//#[[;6"-"3"3SXX2{';&<=EEG F ((<&,kkK&?FLLOnj91$&&y}5&&x0&&w/&&w/&&w/&&z256s *t 	/*,_,=>?r   c                 d    | j                          | j                          | j                          y r
   )r#   r   skip_reshaper   s    r   
preprocesszBertOnnxModelTF.preprocess.  s&     r   c                     d}| j                  d      }|D ]H  }| j                  |d      }||j                  dk(  s(|j                  d   |j                  d<   |dz  }J |dkD  rt        j                  d|        y y )Nr   r'   r)   zSkip consequent Reshape count: )r   r   r   r   r   r   )r   countreshape_nodesr   r   s        r   r   zBertOnnxModelTF.skip_reshape3  s    11)<)L__\15F!fnn	&A(.Q""1%
	 * 19KK9%AB r   c                    | j                  d      }|D ]t  }| j                  |ddgddg      }|t        j                  d       |\  }}| j	                  |j
                  d   |j                  d          | j                  |        y  y )N	Attentionr'   rw   r   z+Remove Reshape before first Attention node.)r   r,   r   r   r   r   r   remove_node)r   attention_nodesr   pathreshaper\   s         r   %remove_reshape_before_first_attentionz5BertOnnxModelTF.remove_reshape_before_first_attention?  s    33K@-N)).9F_:`cdfgbhiD|KKEFJGQ++GNN1,=w}}Q?OPW% .r   c                 D    | j                          | j                          y r
   )r   r   r   s    r   postprocesszBertOnnxModelTF.postprocessK  s    224r   )__name__
__module____qualname__r   r#   r/   r<   ra   rd   rs   r   r   r   r   r   r   r   r   __classcell__)r   s   @r   r   r      sY    8G2 >@ GR4Ol?B	J@X

C
r   r   )loggingnumpyr   r   r   r   r   onnx_model_bertr   	getLoggerr   r   r    r   r   <module>r      s9       2 2 )			8	$}m }r   