gL0ZddlmZddlmZGddZGddeZGddZy ) )ArgumentParser)EnumceZdZdZdZdZdZy)AttentionMaskFormatrN)__name__ __module__ __qualname__ MaskIndexEndMaskIndexEndAndStart AttentionMaskNoMask\/var/www/openai/venv/lib/python3.12/site-packages/onnxruntime/transformers/fusion_options.pyrr sLMFrrc.eZdZdZdZdZdZdZdZdZ y) AttentionOpType AttentionMultiHeadAttentionGroupQueryAttentionPagedAttentionc|jSNvalueselfs r__str__zAttentionOpType.__str__s zzrc,t|jSr)hashrrs r__hash__zAttentionOpType.__hash__!sDJJrc4|j|jk(Srr)rothers r__eq__zAttentionOpType.__eq__$s{{djj((rN) r r r rrrrr r#r&rrrrrs'I-/%N )rrcVeZdZdZdZd dZdZdefdZe dZ e de fd Z y ) FusionOptionsz'Options of fusion in graph optimizationcd|_d|_d|_d|_d|_d|_d|_d|_d|_d|_ d|_ d|_ d|_ d|_ d|_|dk(rd|_tj |_|dk(rtj$|_n|dk(rtj&|_d|_|dvr2d|_d|_d|_d|_d|_d|_d|_yy)NTFclipbertvitunetvaer*) enable_geluenable_layer_normenable_attentionenable_rotary_embeddingsuse_multi_head_attention!disable_multi_head_attention_biasenable_skip_layer_normenable_embed_layer_normenable_bias_skip_layer_normenable_bias_geluenable_gelu_approximationenable_qordered_matmulenable_shape_inferenceenable_gemm_fast_gelugroup_norm_channels_lastrrattention_mask_formatr rattention_op_typeenable_nhwc_convenable_group_normenable_skip_group_normenable_bias_splitgeluenable_packed_qkvenable_packed_kvenable_bias_add)r model_types r__init__zFusionOptions.__init__+s!% $(,% ).%16.&*#'+$+/( $).&&*#&*#%*"(,%  +0D (&9%F%F"  )<)I)ID & 5 )<)C)CD &!% 0 0$(D !%)D "*.D ')-D &%)D "$(D !#'D  1rc^|rtj|_ytj|_yr)rrr?r )r use_raw_masks ruse_raw_attention_maskz$FusionOptions.use_raw_attention_maskZs! )<)J)JD &)<)I)ID &rc.tj|_yr)rrr?rs rdisable_attention_maskz$FusionOptions.disable_attention_mask`s%8%?%?"r attn_op_typec||_yr)r@)rrOs rset_attention_op_typez#FusionOptions.set_attention_op_typecs !-rct|j}|jrd|_|jrd|_|j rd|_|jrd|_ |jrd|_ |jrd|_ |jrd|_|jrd|_|j"rd|_|j&rd|_|j(rd|_|j,rd|_|j.r|j1d|j0r|j1d|j2r|j5|jdvr|j6rd|_|j:rd|_|j>rd|_ |jBrd|_"|jFrd|_$|jJrd|_&|jNrd|_(|jRrd|_*|S)NFTr-)+r(rH disable_gelur0disable_layer_normr1disable_rotary_embeddingsr3disable_attentionr2r4disable_skip_layer_normr6disable_embed_layer_normr7disable_bias_skip_layer_normr8disable_bias_gelur9r:disable_shape_inferencer<r=use_mask_indexrLno_attention_maskrNuse_group_norm_channels_firstr>disable_nhwc_convrAdisable_group_normrBdisable_skip_group_normrCdisable_bias_splitgelurDdisable_packed_qkvrEdisable_packed_kvrFdisable_bias_addrG)argsoptionss rparsezFusionOptions.parsefs0   "'G   " "(-G %  ) )/4G ,  ! !',G $  ( (/3G ,  ' '-2G *  ( (.3G +  , ,27G /  ! !',G $  ) )04G -  ' '-2G *  % %,0G )     * *5 1  & &  * *4 0  ! !  * * , ??5 511380%%+0(&&,1)++16.**05-&&,1)%%+0($$*/'rparserc|jdddd|jd|jdddd|jd |jd ddd |jd |jd ddd|jd|jdddd|jd|jdddd|jd|jdddd|jd|jdddd|jd|jdddd|jd|jdddd |jd!|jd"ddd#|jd$|jd%ddd&|jd'|jd(ddd)|jd*|jd+ddd,|jd-|jd.ddd/|jd0|jd1ddd2|jd3|jd4ddd5|jd6|jd7ddd8|jd9|jd:ddd;|jd<|jd=ddd>|jd?|jd@dddA|jdB|jdCdddD|jdE|jdFdddGy)HNz--disable_attentionF store_truezdisable Attention fusion)requiredactionhelp)rVz--disable_skip_layer_normz%disable SkipLayerNormalization fusion)rWz--disable_embed_layer_normz&disable EmbedLayerNormalization fusion)rXz--disable_bias_skip_layer_normz2disable Add Bias and SkipLayerNormalization fusion)rYz--disable_bias_geluz)disable Add Bias and Gelu/FastGelu fusion)rZz--disable_layer_normz!disable LayerNormalization fusion)rTz--disable_geluzdisable Gelu fusion)rSz--enable_gelu_approximationz+enable Gelu/BiasGelu to FastGelu conversion)r:z--disable_shape_inferencez disable symbolic shape inference)r[z--enable_gemm_fast_geluzenable GemmfastGelu fusion)r=z--use_mask_indexzWuse mask index to activate fused attention to speed up. It requires right-side padding!)r\z--use_raw_attention_maskzuse raw attention mask. Use this option if your input is not right-side padding. This might deactivate fused attention and get worse performance.)rLz--no_attention_maskz1no attention mask. Only works for model_type=bert)r]z--use_multi_head_attentionzUse MultiHeadAttention instead of Attention operator for testing purpose. Note that MultiHeadAttention might be slower than Attention when qkv are not packed. )r4z--disable_group_normz9not fuse GroupNorm. Only works for model_type=unet or vae)r`z--disable_skip_group_normzPnot fuse Add + GroupNorm to SkipGroupNorm. Only works for model_type=unet or vae)raz--disable_packed_kvz[not use packed kv for cross attention in MultiHeadAttention. Only works for model_type=unet)rdz--disable_packed_qkvz[not use packed qkv for self attention in MultiHeadAttention. Only works for model_type=unet)rcz--disable_bias_addz0not fuse BiasAdd. Only works for model_type=unet)rez--disable_bias_splitgeluz6not fuse BiasSplitGelu. Only works for model_type=unet)rbz--disable_nhwc_convz:Do not use NhwcConv. Only works for model_type=unet or vae)r_z--use_group_norm_channels_firstznUse channels_first (NCHW) instead of channels_last (NHWC) for GroupNorm. Only works for model_type=unet or vae)r^z--disable_rotary_embeddingsz5Do not fuse rotary embeddings into RotaryEmbedding op) add_argument set_defaults)ris r add_argumentszFusionOptions.add_argumentssv !+  e4 '8  E: (9  U; ,E  ? !<  e4 "4  u5 &  / )>  e< '3  E: %-  %8 j  51 &e  59 !D  e4 (d   U; "L  u5 'c  E: !n  e4 "n  u5 C  U3 &I  59 !M  e4 -B  %@ )H  rN)T) r r r __doc__rIrLrNrrQ staticmethodrhrrqrrrr(r((sV1-(^J @./.33jw nw w rr(N)argparserenumrrrr(rrrrvs/ $  )d)"l l r