
    ^j                         d Z ddlZddlmZmZ ddlZddlmZ ddlmc m	Z
 ddlmZmZ ddlmZ ddlmZ  G d d	ej$                        Zy)
zDifferential Attention

Paper: 'Differential Transformer' - https://arxiv.org/abs/2410.05258

Reference impl: https://github.com/microsoft/unilm/tree/master/Diff-Transformer

Hacked together by / Copyright 2024, Ross Wightman
    N)OptionalType   )maybe_add_maskresolve_self_attn_mask)use_fused_attn)RmsNormc                   Z    e Zd ZU dZej
                  j                  e   ed<   	 	 	 	 	 	 	 	 	 	 	 	 dde	de	dededed	ed
e
de
deeej                        de	deddf fdZde	fdZd Zdej$                  fdZ	 	 ddej$                  deej$                     dedej$                  fdZ xZS )DiffAttentiona0  Differential Attention module.

    Computes attention as the difference between two softmax attention maps, which helps
    cancel out noise and promotes sparse attention patterns. The module splits Q and K
    into two groups, computes separate attention maps, and subtracts one from the other
    scaled by a learnable lambda parameter.

    The attention output is computed as:
        Attn = softmax(Q1 @ K1^T) - lambda * softmax(Q2 @ K2^T)
        Output = Attn @ V

    Supports both fused (scaled_dot_product_attention) and manual implementations.
    
fused_attnNdim	num_headsqkv_biasqk_norm
scale_norm	proj_bias	attn_drop	proj_drop
norm_layerdepthdual_lambdareturnc                    t         |           ||d}||z  dk(  sJ d       |	t        }	|| _        ||z  dz  | _        | j                  dz  | _        t               | _        t        j                  ||dz  fd|i|| _
        |r |	| j                  fi |nt        j                         | _        |r |	| j                  fi |nt        j                         | _        t        j                  |      | _        || _        |r	 |	|fi |nt        j                         | _        t        j                  ||fd|i|| _        t        j                  |      | _        || _        |rt        j*                  t-        j.                  d	t,        j0                  |
            | _        t        j*                  t-        j.                  d	t,        j0                  |
            | _        dx| _        x| _        x| _        | _        n.dx| _        | _        t        j*                  t-        j.                  | j                  t,        j0                  |
            | _        t        j*                  t-        j.                  | j                  t,        j0                  |
            | _        t        j*                  t-        j.                  | j                  t,        j0                  |
            | _        t        j*                  t-        j.                  | j                  t,        j0                  |
            | _        t        d| j                  z  fddi|| _        d| _         | jC                  |
       | jE                          y)a  Initialize the DiffAttention module.

        Args:
            dim: Input dimension of the token embeddings.
            num_heads: Number of attention heads.
            qkv_bias: Whether to use bias in the query, key, value projections.
            qk_norm: Whether to apply normalization to query and key vectors.
            scale_norm: Whether to apply normalization before the output projection.
            proj_bias: Whether to use bias in the output projection.
            attn_drop: Dropout rate applied to the attention weights.
            proj_drop: Dropout rate applied after the output projection.
            norm_layer: Normalization layer constructor (defaults to RmsNorm).
            depth: Block depth index, used to compute depth-dependent lambda_init.
            dual_lambda: If True, use simplified dual scalar lambda parameterization
                (2 params). If False, use the paper's original formulation with
                lambda_q/k vectors (4 * head_dim params).
        )devicedtyper   z$dim should be divisible by num_headsN   g         bias )r   r   epsgh㈵>皙?)#super__init__r	   r   head_dimscaler   r   nnLinearqkvIdentityq_normk_normDropoutr   attn_drop_pnormprojr   r   	Parametertorchemptyfloat32lambda_alambda_b	lambda_q1	lambda_k1	lambda_q2	lambda_k2sub_normlambda_initset_lambda_initreset_parameters)selfr   r   r   r   r   r   r   r   r   r   r   r   r   dd	__class__s                  e/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/diff_attention.pyr#   zDiffAttention.__init__%   s   B 	/Y!#K%KK# J"y(A-]]d*
(*99S#'??B?9@j5"5bkkm9@j5"5bkkmI.$-7Js)b)R[[]	IIc3=Y="=	I.&LLRu}}U[)\]DMLLRu}}U[)\]DMPTTDNTT^Tdnt~,00DMDM\\%++dmm5==ag*hiDN\\%++dmm5==ag*hiDN\\%++dmm5==ag*hiDN\\%++dmm5==ag*hiDNDMM 1BtBrBU#    c                 J    ddt        j                  d|z        z  z
  | _        y )Nr!   g333333?g333333ӿ)mathexpr;   )r>   r   s     rA   r<   zDiffAttention.set_lambda_initk   s!    txxu'=!==rB   c                 "   | j                   rSt        j                  j                  | j                         t        j                  j                  | j
                         y t        j                  j                  | j                  dd       t        j                  j                  | j                  dd       t        j                  j                  | j                  dd       t        j                  j                  | j                  dd       y )Nr   g?)meanstd)r   r&   initzeros_r4   r5   normal_r6   r7   r8   r9   )r>   s    rA   r=   zDiffAttention.reset_parametersn   s    GGNN4==)GGNN4==)GGOODNNO<GGOODNNO<GGOODNNO<GGOODNNO<rB   c                    | j                   ?t        j                  | j                         }t        j                  | j                        }nt        j                  t        j                  | j
                  | j                  z  d      j                               }t        j                  t        j                  | j                  | j                  z  d      j                               }||z
  | j                  z   S )Nr   )r4   r1   rE   r5   sumr6   r7   floatr8   r9   r;   )r>   lambda_1lambda_2s      rA   _compute_lambdazDiffAttention._compute_lambdax   s    ==$yy/Hyy/Hyy4>>DNN+JPR!S!Y!Y![\Hyy4>>DNN+JPR!S!Y!Y![\H("T%5%555rB   x	attn_mask	is_causalc                 J   |j                   \  }}}| j                  |      j                  dd      \  }}}	|j                  ||d| j                  z  | j
                        j                  dd      }|j                  ||d| j                  z  | j
                        j                  dd      }|	j                  ||| j                  d| j
                  z        j                  dd      }	| j                  |      | j                  |      }}| j                         j                  |      }
| j                  r|j                  || j                  d|| j
                        }|j                  || j                  d|| j
                        }|j                  d      \  }}|j                  d      \  }}| j                  r| j                  nd}t        j                   |||	|||      }t        j                   |||	|||      }||
|z  z
  }n|| j"                  z  }||j                  dd      z  }t%        ||||	      }t'        ||      }|j)                  d      }| j+                  |      }|j-                  || j                  d||      }|d d d d d
f   |
|d d d d df   z  z
  }||	z  }| j/                  |      }|d| j0                  z
  z  }|j                  dd      j                  |||      }| j3                  |      }| j5                  |      }| j7                  |      }|S )Nr   r   rN   r           )rU   	dropout_prV   rM   )rV   r   )shaper(   chunkreshaper   r$   	transposer*   r+   rS   type_asr   unbindtrainingr-   Fscaled_dot_product_attentionr%   r   r   softmaxr   viewr:   r;   r.   r/   r   )r>   rT   rU   rV   BNCqkvlambda_fullq1q2k1k2rY   attn1attn2attn	attn_biass                       rA   forwardzDiffAttention.forward   s    ''1a((1+##A1#-1aIIaA.>HHANIIaA.>HHANIIaDNNA,=>HHAN{{1~t{{1~1**,44Q7??		!T^^Q4==AA		!T^^Q4==AAXXa[FBXXa[FB,0MM((sI22BY)yZE22BY)yZE e++ADJJAq{{2r**D.q$	YWI!$	2D<<B<'D>>$'D99Q1a8D1a=;aAg#>>DqAMM!T%%%&KK1%%aA.IIaLIIaLNN1rB   )   FFFTrX   rX   Nr   FNN)NF)__name__
__module____qualname____doc__r1   jitFinalbool__annotations__intrP   r   r   r&   Moduler#   r<   r=   TensorrS   ru   __classcell__)r@   s   @rA   r   r      s>    		%%
 "!$"!!48 %D D  D  	D 
 D  D  D  D  D  !bii1D  D  D  
D L>S >=6 6 15#	2||2  -2 	2
 
2rB   r   )rz   rD   typingr   r   r1   torch.nnr&   torch.nn.functional
functionalrb   	attentionr   r   configr   r.   r	   r   r   r   rB   rA   <module>r      s9     !     = " ^BII ^rB   