Ë
    éÓ^jª  ã                   ó˜   — d Z ddlZddlmZmZ ddlZddlmZ ddlmc m	Z
 ddlmZmZ ddlmZ ddlmZ  G d„ d	ej$                  «      Zy)
zâDifferential Attention

Paper: 'Differential Transformer' - https://arxiv.org/abs/2410.05258

Reference impl: https://github.com/microsoft/unilm/tree/master/Diff-Transformer

Hacked together by / Copyright 2024, Ross Wightman
é    N)ÚOptionalÚTypeé   )Úmaybe_add_maskÚresolve_self_attn_mask)Úuse_fused_attn)ÚRmsNormc                   óZ  ‡ — e Zd ZU dZej
                  j                  e   ed<   	 	 	 	 	 	 	 	 	 	 	 	 dde	de	dededed	ed
e
de
deeej                        de	deddfˆ fd„Zde	fd„Zd„ Zdej$                  fd„Z	 	 ddej$                  deej$                     dedej$                  fd„Zˆ xZS )ÚDiffAttentiona0  Differential Attention module.

    Computes attention as the difference between two softmax attention maps, which helps
    cancel out noise and promotes sparse attention patterns. The module splits Q and K
    into two groups, computes separate attention maps, and subtracts one from the other
    scaled by a learnable lambda parameter.

    The attention output is computed as:
        Attn = softmax(Q1 @ K1^T) - lambda * softmax(Q2 @ K2^T)
        Output = Attn @ V

    Supports both fused (scaled_dot_product_attention) and manual implementations.
    Ú
fused_attnNÚdimÚ	num_headsÚqkv_biasÚqk_normÚ
scale_normÚ	proj_biasÚ	attn_dropÚ	proj_dropÚ
norm_layerÚdepthÚdual_lambdaÚreturnc                 óî  •— t         ‰| �  «        ||dœ}||z  dk(  sJ d«       ‚|	€t        }	|| _        ||z  dz  | _        | j                  dz  | _        t        «       | _        t        j                  ||dz  fd|i|¤Ž| _
        |r |	| j                  fi |¤Žnt        j                  «       | _        |r |	| j                  fi |¤Žnt        j                  «       | _        t        j                  |«      | _        || _        |r	 |	|fi |¤Žnt        j                  «       | _        t        j                  ||fd|i|¤Ž| _        t        j                  |«      | _        || _        |ršt        j*                  t-        j.                  d	t,        j0                  |¬
«      «      | _        t        j*                  t-        j.                  d	t,        j0                  |¬
«      «      | _        dx| _        x| _        x| _        | _        �n.dx| _        | _        t        j*                  t-        j.                  | j                  t,        j0                  |¬
«      «      | _        t        j*                  t-        j.                  | j                  t,        j0                  |¬
«      «      | _        t        j*                  t-        j.                  | j                  t,        j0                  |¬
«      «      | _        t        j*                  t-        j.                  | j                  t,        j0                  |¬
«      «      | _        t        d| j                  z  fddi|¤Ž| _        d| _         | jC                  |
«       | jE                  «        y)að  Initialize the DiffAttention module.

        Args:
            dim: Input dimension of the token embeddings.
            num_heads: Number of attention heads.
            qkv_bias: Whether to use bias in the query, key, value projections.
            qk_norm: Whether to apply normalization to query and key vectors.
            scale_norm: Whether to apply normalization before the output projection.
            proj_bias: Whether to use bias in the output projection.
            attn_drop: Dropout rate applied to the attention weights.
            proj_drop: Dropout rate applied after the output projection.
            norm_layer: Normalization layer constructor (defaults to RmsNorm).
            depth: Block depth index, used to compute depth-dependent lambda_init.
            dual_lambda: If True, use simplified dual scalar lambda parameterization
                (2 params). If False, use the paper's original formulation with
                lambda_q/k vectors (4 * head_dim params).
        )ÚdeviceÚdtyper   z$dim should be divisible by num_headsNé   g      à¿é   Úbias© )r   r   Úepsgñhãˆµøä>çš™™™™™é?)#ÚsuperÚ__init__r	   r   Úhead_dimÚscaler   r   ÚnnÚLinearÚqkvÚIdentityÚq_normÚk_normÚDropoutr   Úattn_drop_pÚnormÚprojr   r   Ú	ParameterÚtorchÚemptyÚfloat32Úlambda_aÚlambda_bÚ	lambda_q1Ú	lambda_k1Ú	lambda_q2Ú	lambda_k2Úsub_normÚlambda_initÚset_lambda_initÚreset_parameters)Úselfr   r   r   r   r   r   r   r   r   r   r   r   r   ÚddÚ	__class__s                  €úe/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/diff_attention.pyr#   zDiffAttention.__init__%   s  ø€ ôB 	‰ÑÔØ¨Ñ/ˆØ�Y‰ !Ò#ÐKÐ%KÓKÐ#ØÐÜ ˆJØ"ˆŒØ˜yÑ(¨AÑ-ˆŒØ—]‘] dÑ*ˆŒ
Ü(Ó*ˆŒä—9‘9˜S #¨¡'Ñ?°Ð?¸BÑ?ˆŒÙ9@‘j §¡Ñ5°"Ò5ÄbÇkÁkÃmˆŒÙ9@‘j §¡Ñ5°"Ò5ÄbÇkÁkÃmˆŒÜŸ™ IÓ.ˆŒØ$ˆÔÙ-7‘J˜sÑ) bÒ)¼R¿[¹[»]ˆŒ	Ü—I‘I˜c 3Ñ=¨YÐ=¸"Ñ=ˆŒ	ÜŸ™ IÓ.ˆŒà&ˆÔÙÜŸL™L¬¯©°R¼u¿}¹}ÐU[Ô)\Ó]ˆDŒMÜŸL™L¬¯©°R¼u¿}¹}ÐU[Ô)\Ó]ˆDŒMØPTÐTˆDŒNÐT˜Tœ^ÐT¨d¬n¸t¾~à,0Ð0ˆDŒM˜DœMÜŸ\™\¬%¯+©+°d·m±mÌ5Ï=É=ÐagÔ*hÓiˆDŒNÜŸ\™\¬%¯+©+°d·m±mÌ5Ï=É=ÐagÔ*hÓiˆDŒNÜŸ\™\¬%¯+©+°d·m±mÌ5Ï=É=ÐagÔ*hÓiˆDŒNÜŸ\™\¬%¯+©+°d·m±mÌ5Ï=É=ÐagÔ*hÓiˆDŒNä  D§M¡MÑ 1ÑB°tÐB¸rÑBˆŒàˆÔØ×Ñ˜UÔ#Ø×ÑÕó    c                 óJ   — ddt        j                  d|z  «      z  z
  | _        y )Nr!   g333333ã?g333333Ó¿)ÚmathÚexpr;   )r>   r   s     rA   r<   zDiffAttention.set_lambda_initk   s!   € Ø ¤t§x¡x°°u±Ó'=Ñ!=Ñ=ˆÕrB   c                 ó"  — | j                   rSt        j                  j                  | j                  «       t        j                  j                  | j
                  «       y t        j                  j                  | j                  dd¬«       t        j                  j                  | j                  dd¬«       t        j                  j                  | j                  dd¬«       t        j                  j                  | j                  dd¬«       y )Nr   gš™™™™™¹?)ÚmeanÚstd)r   r&   ÚinitÚzeros_r4   r5   Únormal_r6   r7   r8   r9   )r>   s    rA   r=   zDiffAttention.reset_parametersn   s›   € Ø×ÒÜ�G‰G�N‰N˜4Ÿ=™=Ô)Ü�G‰G�N‰N˜4Ÿ=™=Õ)ä�G‰G�O‰O˜DŸN™N°¸ˆOÔ<Ü�G‰G�O‰O˜DŸN™N°¸ˆOÔ<Ü�G‰G�O‰O˜DŸN™N°¸ˆOÔ<Ü�G‰G�O‰O˜DŸN™N°¸ˆOÕ<rB   c                 óø  — | j                   �?t        j                  | j                   «      }t        j                  | j                  «      }nžt        j                  t        j                  | j
                  | j                  z  d¬«      j                  «       «      }t        j                  t        j                  | j                  | j                  z  d¬«      j                  «       «      }||z
  | j                  z   S )Néÿÿÿÿ©r   )r4   r1   rE   r5   Úsumr6   r7   Úfloatr8   r9   r;   )r>   Úlambda_1Úlambda_2s      rA   Ú_compute_lambdazDiffAttention._compute_lambdax   s¦   € Ø�=‰=Ð$Ü—y‘y §¡Ó/ˆHÜ—y‘y §¡Ó/‰Hä—y‘y¤§¡¨4¯>©>¸D¿N¹NÑ+JÐPRÔ!S×!YÑ!YÓ![Ó\ˆHÜ—y‘y¤§¡¨4¯>©>¸D¿N¹NÑ+JÐPRÔ!S×!YÑ!YÓ![Ó\ˆHØ˜(Ñ" T×%5Ñ%5Ñ5Ð5rB   ÚxÚ	attn_maskÚ	is_causalc                 óJ  — |j                   \  }}}| j                  |«      j                  dd¬«      \  }}}	|j                  ||d| j                  z  | j
                  «      j                  dd«      }|j                  ||d| j                  z  | j
                  «      j                  dd«      }|	j                  ||| j                  d| j
                  z  «      j                  dd«      }	| j                  |«      | j                  |«      }}| j                  «       j                  |«      }
| j                  rÓ|j                  || j                  d|| j
                  «      }|j                  || j                  d|| j
                  «      }|j                  d«      \  }}|j                  d«      \  }}| j                  r| j                  nd}t        j                   |||	|||¬«      }t        j                   |||	|||¬«      }||
|z  z
  }n¢|| j"                  z  }||j                  dd«      z  }t%        ||||¬	«      }t'        ||«      }|j)                  d¬«      }| j+                  |«      }|j-                  || j                  d||«      }|d d …d d …d
f   |
|d d …d d …df   z  z
  }||	z  }| j/                  |«      }|d| j0                  z
  z  }|j                  dd«      j                  |||«      }| j3                  |«      }| j5                  |«      }| j7                  |«      }|S )Nr   r   rN   r   ç        )rU   Ú	dropout_prV   éþÿÿÿrM   )rV   r   )Úshaper(   ÚchunkÚreshaper   r$   Ú	transposer*   r+   rS   Útype_asr   ÚunbindÚtrainingr-   ÚFÚscaled_dot_product_attentionr%   r   r   Úsoftmaxr   Úviewr:   r;   r.   r/   r   )r>   rT   rU   rV   ÚBÚNÚCÚqÚkÚvÚlambda_fullÚq1Úq2Úk1Úk2rY   Úattn1Úattn2ÚattnÚ	attn_biass                       rA   ÚforwardzDiffAttention.forward�   sÈ  € ð —'‘'‰ˆˆ1ˆaà—(‘(˜1“+×#Ñ# A¨1Ð#Ó-‰ˆˆ1ˆaØ�I‰I�a˜˜A §¡Ñ.°·±Ó>×HÑHÈÈAÓNˆØ�I‰I�a˜˜A §¡Ñ.°·±Ó>×HÑHÈÈAÓNˆØ�I‰I�a˜˜DŸN™N¨A°·±Ñ,=Ó>×HÑHÈÈAÓNˆà�{‰{˜1‹~˜tŸ{™{¨1›~ˆ1ˆà×*Ñ*Ó,×4Ñ4°QÓ7ˆà�?Š?Ø—	‘	˜!˜TŸ^™^¨Q°°4·=±=ÓAˆAØ—	‘	˜!˜TŸ^™^¨Q°°4·=±=ÓAˆAØ—X‘X˜a“[‰FˆB�Ø—X‘X˜a“[‰FˆB�à,0¯MªM˜×(Ò(¸sˆIÜ×2Ñ2Ø�B˜ Y¸)ÈyôZˆEä×2Ñ2Ø�B˜ Y¸)ÈyôZˆEð ˜ eÑ+Ñ+‰Aà�D—J‘J‘ˆAØ�q—{‘{ 2 rÓ*Ñ*ˆDÜ.¨q°$¸	ÈYÔWˆIÜ! $¨	Ó2ˆDØ—<‘< B�<Ó'ˆDØ—>‘> $Ó'ˆDà—9‘9˜Q §¡°°1°aÓ8ˆDØšš1˜a˜‘= ;°²aº¸A°g±Ñ#>Ñ>ˆDØ�q‘ˆAà�M‰M˜!ÓˆØ��T×%Ñ%Ñ%Ñ&ˆØ�K‰K˜˜1Ó×%Ñ% a¨¨AÓ.ˆà�I‰I�a‹LˆØ�I‰I�a‹LˆØ�N‰N˜1ÓˆàˆrB   )é   FFFTrX   rX   Nr   FNN)NF)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r1   ÚjitÚFinalÚboolÚ__annotations__ÚintrP   r   r   r&   ÚModuler#   r<   r=   ÚTensorrS   ru   Ú__classcell__)r@   s   @rA   r   r      s>  ø… ñð —	‘	—‘ Ñ%Ó%ð
 Ø"Ø!Ø$Ø"Ø!Ø!Ø48ØØ %ØØñD àðD ð ðD ð ð	D ð
 ðD ð ðD ð ðD ð ðD ð ðD ð !  b§i¡i¡Ñ1ðD ð ðD ð ðD ð 
õD ðL> Só >ò=ð6 §¡ó 6ð 15Ø#ñ	2à�|‰|ð2ð   §¡Ñ-ð2ð ð	2ð
 
�‰÷2rB   r   )rz   rD   Útypingr   r   r1   Útorch.nnr&   Útorch.nn.functionalÚ
functionalrb   Ú	attentionr   r   Úconfigr   r.   r	   r€   r   r   rB   rA   ú<module>r‰      s9   ðñó ß !ã Ý ß Ð ç =Ý "Ý ô^�B—I‘Iõ ^rB   