o hB6#@s UddlZddlZddlmZmZddlZddlmZejdedZiZ e ej j efe d<ehdZded ed eegeffd d Zed d d;dddddejdejdejdeejdededed ejfddZdeeded efddZdejdeded ejfd d!Zd"ejd#ejd$ed%edeed&ed ejfd'd(Zd$ed%ed dfd)d*Zd"ejd#ejd$ed%edeed ejf d+d,Zed-d   d op_type opset_versionreturncsdtdtffdd }|S)zDDecorator to register an ONNX operator with a custom implementation.funcrcsPd}tjjdd|dd|}|ttttjj|<|||S)NZopsetzonnx::.)Z mutates_args)torchZlibraryZ custom_oprgetattropsZonnxZ register_fake)roverloadZtorch_opr r rZ/home/www/facesmatcher.com/frenv_anti/lib/python3.10/site-packages/torch/onnx/ops/_impl.py decorators  z_onnx_op..decorator)r)r r rrrr_onnx_ops rZRotaryEmbeddingF) interleaved num_headsrotary_embedding_dimx cos_cache sin_cache position_idsrrrcsNjd}jd}tjdkr2jd} t|dkfdd| |} |||| g} t| ttjdkddjd} |dkrJ| }d d d d d d d |f} d d d d d d |d f} |d}|d ur{||}||}n|}|}|d d d d d |f}|d d d d d |f}t|d}t|d}|r| d d d d d d dd df}| d d d d d d dd df}n tj| dd d \}}||||}||||}|rt|d }t|d }tj||fd d }t|| j} n tj||fd d } tj| | fd d }tjdkr%t|j}|S) z_RotaryEmbedding-23 https://onnx.ai/onnx/operators/onnx__RotaryEmbedding.html#rotaryembedding-23rr cs djS)NzKnum_heads must be provided for 3D inputs. Received input tensor with shape )shaperrrr>s z%rotary_embedding_23..cSdS)Nzx should be a 4D tensor by nowrrrrrr&CNdim)r$lenr_checkZreshapeZ unsqueezechunkcat)rrr r!rrr batch_sizesequence_length hidden_size head_sizeZ new_shapeZx_rotateZ x_not_rotateZrotary_embedding_dim_halfcossinx1Zx2realimagZx_rotate_concatoutputrr%rrotary_embedding_23+sn        "$  r;scaler4cCs|dur|Sdt|S)z/Get the scale factor for attention computation.Ng?)mathsqrt)r<r4rrr_get_scale_factorsr?tensorr1cCs:|jd|jd}}||}|||||ddS)z1Reshape 3D tensor to 4D for multi-head attention.r r#)r$view transpose contiguous)r@r1rr2r3r4rrr_reshape_3d_to_4ds rDQKcurrent_q_num_headscurrent_kv_num_headsqk_matmul_output_modec Cs2|dkr t|||||Stt||ddS)z1Get QK output tensor based on the specified mode.rr*)_compute_qk_output_for_mode_0rZ zeros_likematmulrB)rErFrGrHr<rIrrr_get_qk_output_for_aten_spdas   rMcs"tdkfdddS)z-Validate Group Query Attention configuration.rcsdddS)Nz q_num_heads (z%) must be divisible by kv_num_heads (z ) for GQArrrHrGrrr&sz-_validate_gqa_configuration..N)rr.)rGrHrrNr_validate_gqa_configurations  rOc Cs`|}||kr||}|j|dd}t||jd}t|}||} ||} t| | ddS)zDHelper function to compute QK output for qk_matmul_output_mode == 0.r r+r"rJr*)repeat_interleaver?r$r=r>rrLrB) rErFrGrHr<ZK_for_qk repeat_factor scale_factor sqrt_scaleQ_scaledK_scaledrrrrKs  rKZ Attention) is_causal kv_num_heads q_num_headsrIr<softcapsoftmax_precisionV attn_maskpast_key past_valuerWrXrYrZr[c( Csd\} }}t|j}|jd}t|jdkr;t|dko|dkdd|jd}t|||}t|||}t|||}tt|jdkoQt|jdkoQt|jdkdd|j|}t| |} |d urmtj||g|d n|}|d ur~tj||g|d n|}||}}|j| }|j| }|j|}|j|}| d ko| dko| d uo|d up|jtj k}t |||rd }|d ur|jtj kr|n|}tj j j ||||d || t ||kd }t||||| | }n||kr||}|j|| d }|j|| d }tj|||j|jd }|r+t|d uddttj||tj |jd }||td}|d urE|jtj krA||td}n||}t| |jd} t| }!||!}"||!}#t|"|#dd}$|$}|$|}%| dkrq|%}| d kr| t|%| }%| dkr|%}| d ur| tvr|%j}&|%tj| }%tj|%dd }'|'|&}'qtj|%dd }'ntj|%dd }'| dkr|'}t|'|}|dkr|dd !||d}||||fS)zMAttention-23 https://onnx.ai/onnx/operators/onnx__Attention.html#attention-23)r r#r"rr"cSr()Nz;q_num_heads and kv_num_heads must be provided for 3D inputsrrrrrr&r)zattention_23..r r'cSr()Nz'Q, K, and V should be 4D tensors by nowrrrrrr&r)Nr+rV)r]Z dropout_prWr<Z enable_gqa)dtypedevicecSr()Nz'Cannot use both is_causal and attn_maskrrrrrr&@r)z-infrJr*r#)"r-r$rr.rDr?r0cloner`boolrOnnZ functionalZscaled_dot_product_attentionrMrPZzerosraZtrilZonesZ masked_fillfloatr=r>rLrBtanh-_ATTENTION_23_ALLOWED_INTERMEDIATE_PRECISIONStorZONNX_DTYPE_TO_TORCH_DTYPEZsoftmaxrCrA)(rErFr\r]r^r_rWrXrYrIr<rZr[Z num_head_dimZ sequence_dimZhead_dimZinput_shape_lenr1Zq_sequence_lengthZ q_head_sizeZ present_keyZ present_valuerGrHZkv_sequence_lengthZ can_use_sdpaZsdpa_attn_maskr:Z qk_outputrQZ attn_biasZ causal_maskrRrSrTrUZqk_matmul_outputZ qk_with_biasZoriginal_dtypeZ qk_softmaxrrr attention_23s       (                       ri)N)NNN)r=typingrrrZtorch.onnx.opsrTypeVarrrdictZ_opsZ OpOverload__annotations__ frozensetrgstrintrZTensorrcr;rer?rDrMrOrKtuplerirrrrs    U