Ë
    éÓ^j¢5  ã                   óê   — d dl mZmZmZmZ d dlZd dlmZ d dlmZ	 ddl
mZ ddlmZ ddlmZ dd	lmZ  G d
„ dej"                  «      Z G d„ dej"                  «      Z G d„ dej"                  «      Zy)é    )ÚListÚOptionalÚTypeÚUnionN)Únn)Ú
functionalé   )Úuse_fused_attn)Úcreate_conv2d)Ú	to_2tuple)Úcreate_pool2dc                   óŒ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 ddedee   dededededefˆ fd	„Zd
„ Zd„ Z	ddee
j                     fd„Zˆ xZS )ÚMultiQueryAttentionV2aÀ  Multi Query Attention.

    Fast Transformer Decoding: One Write-Head is All You Need
    https://arxiv.org/pdf/1911.02150.pdf

    This is an acceletor optimized version - removing multiple unnecessary
    tensor transpose by re-arranging indices according to the following rules: 1)
    contracted indices are at the end, 2) other indices have the same order in the
    input and output tensores.

    Compared to V1, this gives 3x speed up.
    ÚdimÚdim_outÚ	num_headsÚkey_dimÚ	value_dimÚ	attn_dropÚ	proj_dropc
                 óú  •— ||	dœ}
t         ‰| �  «        |xs |}|| _        || _        || _        |dz  | _        t        j                  t        j                  | j                  | j                  |ffi |
¤Ž«      | _
        t        j                  t        j                  || j                  ffi |
¤Ž«      | _        t        j                  t        j                  || j                  ffi |
¤Ž«      | _        t        j                  |«      | _        t        j                  t        j                  || j                  | j                  ffi |
¤Ž«      | _        t        j                  |«      | _        | j#                  «        y)zInitializer.©ÚdeviceÚdtypeç      à¿N)ÚsuperÚ__init__r   r   r   Úscaler   Ú	ParameterÚtorchÚemptyÚ
query_projÚkey_projÚ
value_projÚDropoutr   Úout_projr   Úreset_parameters)Úselfr   r   r   r   r   r   r   r   r   ÚddÚ	__class__s              €úb/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/attention2d.pyr   zMultiQueryAttentionV2.__init__   s  ø€ ð ¨Ñ/ˆÜ‰ÑÔØ’.˜SˆØ"ˆŒØˆŒØ"ˆŒØ ‘_ˆŒ
äŸ,™,¤u§{¡{°D·N±NÀDÇLÁLÐRUÐ3VÑ']ÐZ\Ñ']Ó^ˆŒÜŸ™¤U§[¡[°#°t·|±|Ð1DÑ%KÈÑ%KÓLˆŒÜŸ,™,¤u§{¡{°C¸¿¹Ð3HÑ'OÈBÑ'OÓPˆŒÜŸ™ IÓ.ˆŒÜŸ™¤U§[¡[°'¸4¿>¹>È4Ï>É>Ð1ZÑ%aÐ^`Ñ%aÓbˆŒÜŸ™ IÓ.ˆŒà×ÑÕó    c                 óÈ  — | j                   j                  d   dz  }t        j                  j	                  | j
                  |¬«       t        j                  j	                  | j                   |¬«       t        j                  j	                  | j                  |¬«       t        j                  j	                  | j                  | j                  j                  d   dz  ¬«       y )Nr   r   )Ústd)r#   Úshaper   ÚinitÚnormal_r"   r$   r&   )r(   r   s     r+   r'   z&MultiQueryAttentionV2.reset_parameters9   s�   € Ø—‘×#Ñ# AÑ&¨$Ñ.ˆÜ
�‰�‰˜Ÿ™¨UˆÔ3Ü
�‰�‰˜Ÿ™¨5ˆÔ1Ü
�‰�‰˜Ÿ™¨UˆÔ3Ü
�‰�‰˜Ÿ™¨4¯=©=×+>Ñ+>¸qÑ+AÀTÑ+IˆÕJr,   c                 ól   — |j                   }|j                  |d   |d   d«      j                  dd«      S )zBReshapes a tensor to three dimensions, keeping the first and last.r   r	   éÿÿÿÿé   )r/   ÚreshapeÚ	transpose©r(   ÚtÚss      r+   Ú_reshape_inputz$MultiQueryAttentionV2._reshape_input@   s5   € à�G‰Gˆð �y‰y˜˜1™˜q ™t RÓ(×2Ñ2°1°aÓ8Ð8r,   Úmc                 ó‚  — |j                   \  }}}}|�|n|}| j                  |«      }| j                  |«      }t        j                  d|| j                  «      }	t        j                  d|| j
                  «      }
t        j                  d|	|
«      | j                  z  }|j                  d¬«      }| j                  |«      }t        j                  d|| j                  «      }t        j                  d||«      }t        j                  d|| j                  «      }| j                  |«      }|j                  |d||«      S )	úRun layer computation.zbnd,hkd->bnhkzbmd,dk->bmkzbnhk,bmk->bnhmr3   ©r   zbmd,dv->bmvzbnhm,bmv->bnhvzbnhv,dhv->bdn)r/   r:   r    Úeinsumr"   r#   r   Úsoftmaxr   r$   r&   r   r5   )r(   Úxr;   ÚbÚ_ÚhÚwÚ
reshaped_xÚ
reshaped_mÚqÚkÚattnÚvÚoÚresults                  r+   ÚforwardzMultiQueryAttentionV2.forwardH   s  € à—W‘W‰
ˆˆ1ˆa�Ø�‰A Aˆà×(Ñ(¨Ó+ˆ
Ø×(Ñ(¨Ó+ˆ
ä�L‰L˜¨*°d·o±oÓFˆÜ�L‰L˜¨
°D·M±MÓBˆä�|‰|Ð,¨a°Ó3°d·j±jÑ@ˆØ�|‰| ˆ|Ó#ˆØ�~‰~˜dÓ#ˆä�L‰L˜¨
°D·O±OÓDˆÜ�L‰LÐ)¨4°Ó3ˆÜ—‘˜o¨q°$·-±-Ó@ˆØ—‘ Ó'ˆØ�~‰~˜a  Q¨Ó*Ð*r,   )Né   é@   rP   ç        rQ   NN©N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úintr   Úfloatr   r'   r:   r    ÚTensorrN   Ú__classcell__©r*   s   @r+   r   r      s�   ø„ ñð  &*ØØØØ!Ø!ØØñ àð ð ˜c‘]ð ð ð	 ð
 ð ð ð ð ð ð õ ò<Kò9ñ+˜H U§\¡\Ñ2÷ +r,   r   c                   óª  ‡ — e Zd ZU dZej
                  j                  e   ed<   ddddddddddde	j                  d	ddfd
edee   dedee   dee   dededededeeeee   f   dededee	j$                     defˆ fd„Zd„ Zdej*                  fd„Zdej*                  dedefd„Zdej*                  dededefd„Zd"d eej*                     fd!„Zˆ xZS )#ÚMultiQueryAttention2daª  Multi Query Attention with spatial downsampling.

     3 parameters are introduced for the spatial downsampling:
     1. kv_stride: downsampling factor on Key and Values only.
     2. query_strides: horizontal & vertical strides on Query only.

    This is an optimized version.
    1. Projections in Attention is explicit written out as 1x1 Conv2D.
    2. Additional reshapes are introduced to bring a up to 3x speed up.
    Ú
fused_attnNrO   r	   é   Ú rQ   Fr   r   r   r   r   Úquery_stridesÚ	kv_strideÚdw_kernel_sizeÚdilationÚpaddingr   r   Ú
norm_layerÚuse_biasc                 óÐ  •— ||dœ}t         ‰| �  «        |xs |}|| _        |xs ||z  | _        |xs ||z  | _        t        |«      | _        || _        t        | j                  D �cg c]  }|dkD  ‘Œ	 c}«      | _	        | j                  dz  | _
        t        «       | _        || _        t        j                  «       | _        | j                  r‹|
dk(  r3| j                   j#                  dt%        d| j                  d¬«      «       n0| j                   j#                  dt        j&                  |¬«      «       | j                   j#                  d	 ||fi |¤Ž«       | j                   j#                  d
t)        || j                  | j                  z  fd|dœ|¤Ž«       t        j                  «       | _        |dkD  rP| j*                  j#                  dt)        ||f|||	|
ddœ|¤Ž«       | j*                  j#                  d	 ||fi |¤Ž«       | j*                  j#                  d
t)        || j                  fd|
|dœ|¤Ž«       t        j                  «       | _        |dkD  rP| j,                  j#                  dt)        ||f|||	|
ddœ|¤Ž«       | j,                  j#                  d	 ||fi |¤Ž«       | j,                  j#                  d
t)        || j                  fd|dœ|¤Ž«       t        j.                  |«      | _        t        j                  «       | _        | j                  r<| j2                  j#                  dt        j4                  | j                  dd¬«      «       | j2                  j#                  d
t)        | j                  | j                  z  |fd|dœ|¤Ž«       | j2                  j#                  dt        j.                  |«      «       d| _        | j9                  «        yc c}w )a{  Initializer.

        Args:
          num_heads: Number of attention heads.
          key_dim: Size of the attention key dimension.
          value_dim: Size of the attention value dimension.
          query_strides: Vertical stride size for query only.
          kv_stride: Key and value stride size.
          dw_kernel_size: Spatial dimension of the depthwise kernel.
        r   r	   r   ÚsameÚ	down_poolÚavg)Úkernel_sizere   )rl   ÚnormÚproj)rl   ÚbiasÚ	down_convT)rl   Ústriderd   re   Ú	depthwise)rl   re   ro   ÚupsampleÚbilinearF)Úscale_factorÚmodeÚalign_cornersÚdropN)r   r   r   r   r   r   ra   rb   ÚanyÚhas_query_stridesr   r
   r^   rx   r   Ú
SequentialÚqueryÚ
add_moduler   Ú	AvgPool2dr   ÚkeyÚvaluer%   r   ÚoutputÚUpsampler?   Úinit_weights)r(   r   r   r   r   r   ra   rb   rc   rd   re   r   r   rf   rg   r   r   r)   r9   r*   s                      €r+   r   zMultiQueryAttention2d.__init__k   s¬  ø€ ð: ¨Ñ/ˆÜ‰ÑÔØ’.˜SˆØ"ˆŒØÒ2 #¨Ñ"2ˆŒØ"Ò6 c¨YÑ&6ˆŒÜ& }Ó5ˆÔØ"ˆŒÜ!$°T×5GÑ5GÖ%H° a¨!£eÒ%HÓ!IˆÔØ—\‘\ TÑ)ˆŒ
Ü(Ó*ˆŒØˆŒ	ä—]‘]“_ˆŒ
Ø×!Ò!à˜&Ò Ø—
‘
×%Ñ% k´=ØØ $× 2Ñ 2Ø"ô4õ ð —
‘
×%Ñ% k´2·<±<ÈMÔ3ZÔ[Ø�J‰J×!Ñ! &©*°SÑ*?¸BÑ*?Ô@Ø�
‰
×Ñ˜f¤mØØ�N‰N˜TŸ\™\Ñ)ð'
ð Øñ	'
ð
 ñ'
ô 	ô —=‘=“?ˆŒØ�qŠ=Ø�H‰H×Ñ ¬]ØØð	.ð +Ø Ø!ØØñ	.ð ñ	.ô 	ð �H‰H×Ñ ©
°3Ñ(=¸"Ñ(=Ô>Ø�‰×Ñ˜F¤MØØ�L‰Lð%
ð ØØñ%
ð ñ%
ô 	ô —]‘]“_ˆŒ
Ø�qŠ=Ø�J‰J×!Ñ! +¬}ØØð	0ð +Ø Ø!ØØñ	0ð ñ	0ô 	ð �J‰J×!Ñ! &©*°SÑ*?¸BÑ*?Ô@Ø�
‰
×Ñ˜f¤mØØ�N‰Nð'
ð Øñ	'
ð
 ñ'
ô 	ô Ÿ™ IÓ.ˆŒä—m‘m“oˆŒØ×!Ò!Ø�K‰K×"Ñ" :¬r¯{©{Ø!×/Ñ/ØØ#ô0ô ð
 	�‰×Ñ˜v¤}Ø�N‰N˜TŸ^™^Ñ+Øð(
ð Øñ	(
ð
 ñ(
ô 	ð 	�‰×Ñ˜v¤r§z¡z°)Ó'<Ô=àˆŒØ×ÑÕùòq &Is   Á(O#c                 óþ  — t         j                  j                  | j                  j                  j
                  «       t         j                  j                  | j                  j                  j
                  «       t         j                  j                  | j                  j                  j
                  «       | j                  dkD  rzt         j                  j                  | j                  j                  j
                  «       t         j                  j                  | j                  j                  j
                  «       t         j                  j                  | j                  j                  j
                  «       y )Nr	   )r   r0   Úxavier_uniform_r|   rn   Úweightr   r€   rb   rp   r�   )r(   s    r+   rƒ   z"MultiQueryAttention2d.init_weightsê   sÔ   € ä
�‰×Ñ §
¡
§¡× 6Ñ 6Ô7Ü
�‰×Ñ §¡§¡× 4Ñ 4Ô5Ü
�‰×Ñ §
¡
§¡× 6Ñ 6Ô7Ø�>‰>˜AÒÜ�G‰G×#Ñ# D§H¡H×$6Ñ$6×$=Ñ$=Ô>Ü�G‰G×#Ñ# D§J¡J×$8Ñ$8×$?Ñ$?Ô@Ü
�‰×Ñ §¡× 0Ñ 0× 7Ñ 7Õ8r,   r8   c                 óÆ   — |j                   }|j                  |d   |d   d«      j                  dd«      }| j                  r|S |j	                  d«      j                  «       S )zFReshapes a tensor to three dimensions, keeping the batch and channels.r   r	   r3   r4   )r/   r5   r6   r?   Ú	unsqueezeÚ
contiguousr7   s      r+   r:   z$MultiQueryAttention2d._reshape_inputô   sU   € à�G‰GˆØ�I‰I�a˜‘d˜A˜a™D "Ó%×/Ñ/°°1Ó5ˆØ�;Š;ØˆHà—;‘;˜q“>×,Ñ,Ó.Ð.r,   c                 óä   — |j                   }|j                  |d   ||d«      }| j                  r"|j                  dddd«      j	                  «       S |j                  dd«      j	                  «       S )z?Reshapes projected query: [b, n, n, h x k] -> [b, n x n, h, k].r   r3   r_   r	   r4   éþÿÿÿ)r/   r5   r?   Úpermuter‰   r6   )r(   r8   r   r   r9   s        r+   Ú_reshape_projected_queryz.MultiQueryAttention2d._reshape_projected_queryý   sb   € à�G‰GˆØ�I‰I�a˜‘d˜I w°Ó3ˆØ�;Š;Ø—9‘9˜Q  1 aÓ(×3Ñ3Ó5Ð5à—;‘;˜r 2Ó&×1Ñ1Ó3Ð3r,   Úh_pxÚw_pxc                 óÔ   — |j                   }|d   |z  }| j                  s|j                  dd«      }|j                  |d   |||«      j	                  dddd«      j                  «       S )z2Reshape output:[b, n x n x h, k] -> [b, n, n, hk].r3   r	   r4   r   r_   )r/   r?   r6   r5   rŒ   r‰   )r(   r8   r   rŽ   r�   r9   Úfeat_dims          r+   Ú_reshape_outputz%MultiQueryAttention2d._reshape_output  sc   € à�G‰GˆØ�R‘5˜9Ñ$ˆØ�{Š{Ø—‘˜A˜qÓ!ˆAØ�y‰y˜˜1™˜t T¨8Ó4×<Ñ<¸QÀÀ1ÀaÓH×SÑSÓUÐUr,   Ú	attn_maskc                 óÞ  — |j                   x\  }}}}}| j                  |«      }| j                  || j                  | j                  «      }| j                  |«      }	| j                  |	«      }	| j                  |«      }
| j                  |
«      }
| j                  rft        j                  d||	«      | j                  z  }|�||z   }|j                  d¬«      }| j                  |«      }t        j                  d||
«      }nœ| j                  r=t        j                  ||	|
|| j                   r| j                  j"                  nd¬«      }nS|| j                  z  }||	j%                  dd«      z  }|�||z   }|j                  d¬«      }| j                  |«      }||
z  }| j'                  || j                  || j(                  d   z  || j(                  d	   z  «      }| j+                  |«      }|S )
r=   zblhk,bpk->blhpr3   r>   zblhp,bpk->blhkrQ   ©r“   Ú	dropout_pr‹   r   r	   )r/   r|   r�   r   r   r   r:   r€   r?   r    r   r@   r   r^   ÚFÚscaled_dot_product_attentionÚtrainingÚpr6   r’   ra   r�   )r(   rA   r“   ÚBÚCÚHÚWr9   rH   rI   rK   rJ   rL   s                r+   rN   zMultiQueryAttention2d.forward  sÃ  € àŸ™Ð ‰
ˆˆ1ˆa��Qà�J‰J�q‹Mˆà×)Ñ)¨!¨T¯^©^¸T¿\¹\ÓJˆà�H‰H�Q‹Kˆà×Ñ Ó"ˆà�J‰J�q‹Mˆà×Ñ Ó"ˆð
 �;Š;Ü—<‘<Ð 0°!°QÓ7¸$¿*¹*ÑDˆDØÐ$à˜iÑ'�Ø—<‘< B�<Ó'ˆDØ—>‘> $Ó'ˆDÜ—‘Ð-¨t°QÓ7‰Aà�ŠÜ×2Ñ2Ø�q˜!Ø'Ø26·-²-˜dŸn™n×.Ò.ÀRô‘ð ˜Ÿ
™
‘N�Ø˜1Ÿ;™; r¨2Ó.Ñ.�ØÐ(à )Ñ+�DØ—|‘|¨�|Ó+�Ø—~‘~ dÓ+�Ø˜1‘H�ð × Ñ   D§N¡N°A¸×9KÑ9KÈAÑ9NÑ4NÐPQÐUY×UgÑUgÐhiÑUjÑPjÓkˆØ�K‰K˜‹NˆØˆr,   rR   )rS   rT   rU   rV   r    ÚjitÚFinalÚboolÚ__annotations__r   ÚBatchNorm2drW   r   r   Ústrr   rX   r   ÚModuler   rƒ   rY   r:   r�   r’   rN   rZ   r[   s   @r+   r]   r]   ^   s‡  ø… ñ	ð —	‘	—‘ Ñ%Ó%ð
 &*ØØ%)Ø'+Ø!"ØØ"#ØØ24Ø!Ø!Ø*,¯.©.Ø"ØØñ#}àð}ð ˜c‘]ð}ð ð	}ð
 ˜c‘]ð}ð   ‘}ð}ð ð}ð ð}ð  ð}ð ð}ð ˜3  T¨#¡YÐ.Ñ/ð}ð ð}ð ð}ð ˜RŸY™Y™ð}ð õ}ò~9ð/ §¡ó /ð4¨%¯,©,ð 4À3ð 4ÐQTó 4ðV §¡ð V¸#ð VÀSð VÐPSó Vñ/ H¨U¯\©\Ñ$:÷ /r,   r]   c                   ó¾   ‡ — e Zd ZU ej                  j
                  e   ed<   	 	 	 	 	 	 	 	 	 	 ddede	e   dedededede
d	e
fˆ fd
„Zdde	ej                     fd„Zˆ xZS )ÚAttention2dr^   r   r   r   ro   Úexpand_firstÚ
head_firstr   r   c                 ó|  •— |	|
dœ}t         ‰| �  «        |xs |}|r|n|}|| _        ||z  | _        || _        t        «       | _        t        j                  ||dz  dfd|i|¤Ž| _	        t        j                  |«      | _        t        j                  ||dfd|i|¤Ž| _        t        j                  |«      | _        y )Nr   r_   r	   ro   )r   r   r   Údim_headr©   r
   r^   r   ÚConv2dÚqkvr%   r   rn   r   )r(   r   r   r   ro   r¨   r©   r   r   r   r   r)   Údim_attnr*   s                €r+   r   zAttention2d.__init__D  s°   ø€ ð ¨Ñ/ˆÜ‰ÑÔØ’.˜SˆÙ*‘7°ˆØ"ˆŒØ  IÑ-ˆŒØ$ˆŒÜ(Ó*ˆŒä—9‘9˜S (¨Q¡,°ÑC¸ÐCÀÑCˆŒÜŸ™ IÓ.ˆŒÜ—I‘I˜h¨°ÑD¸ÐDÀÑDˆŒ	ÜŸ™ IÓ.ˆ�r,   r“   c                 óŠ  — |j                   \  }}}}| j                  rP| j                  |«      j                  || j                  | j
                  dz  d«      j                  dd¬«      \  }}}	nK| j                  |«      j                  |d| j                  | j
                  d«      j                  d«      \  }}}	| j                  rÍt        j                  j                  j                  |j                  dd«      j                  «       |j                  dd«      j                  «       |	j                  dd«      j                  «       || j                   r| j"                  j$                  nd¬«      j                  dd«      j                  |d||«      }n�|j                  dd«      }|	j                  dd«      }	||z  |j'                  d«      d	z  z  }
|�|
|z   }
|
j)                  d¬«      }
| j#                  |
«      }
|
|	z  j                  dd«      j                  |d||«      }| j+                  |«      }| j-                  |«      }|S )
Nr_   r3   r4   r>   r	   r‹   rQ   r•   r   )r/   r©   r­   Úviewr   r«   Úchunkr5   Úunbindr^   r    r   r   r˜   r6   r‰   r™   r   rš   Úsizer@   rn   r   )r(   rA   r“   r›   rœ   r�   rž   rH   rI   rK   rJ   s              r+   rN   zAttention2d.forward_  sí  € Ø—W‘W‰
ˆˆ1ˆa�à�?Š?Ø—h‘h˜q“k×&Ñ& q¨$¯.©.¸$¿-¹-È!Ñ:KÈRÓP×VÑVÐWXÐ^_ÐVÓ`‰GˆAˆq‘!à—h‘h˜q“k×)Ñ)¨!¨Q°·±ÀÇÁÈrÓR×YÑYÐZ[Ó\‰GˆAˆq�!à�?Š?Ü—‘×#Ñ#×@Ñ@Ø—‘˜B Ó#×.Ñ.Ó0Ø—‘˜B Ó#×.Ñ.Ó0Ø—‘˜B Ó#×.Ñ.Ó0Ø#Ø.2¯mªm˜$Ÿ.™.×*Ò*Àð Aó ÷ ‰i˜˜BÓ§¡¨¨2¨q°!Ó 4ñ ð —‘˜B Ó#ˆAØ—‘˜B Ó#ˆAØ�q‘5˜1Ÿ6™6 "›:¨Ñ-Ñ-ˆDØÐ$à˜iÑ'�Ø—<‘< B�<Ó'ˆDØ—>‘> $Ó'ˆDØ˜‘×$Ñ$ R¨Ó,×4Ñ4°Q¸¸A¸qÓAˆAà�I‰I�a‹LˆØ�N‰N˜1ÓˆØˆr,   )	Né    TFFrQ   rQ   NNrR   )rS   rT   rU   r    rŸ   r    r¡   r¢   rW   r   rX   r   rY   rN   rZ   r[   s   @r+   r§   r§   @  s¢   ø… Ø—	‘	—‘ Ñ%Ó%à3ð &*ØØØ!&Ø$Ø!Ø!ØØñ/àð/ð ˜c‘]ð/ð ð	/ð
 ð/ð ð/ð ð/ð ð/ð õ/ñ6 H¨U¯\©\Ñ$:÷ r,   r§   )Útypingr   r   r   r   r    r   Útorch.nnr   r—   Úconfigr
   r   Úhelpersr   Úpool2d_samer   r¥   r   r]   r§   © r,   r+   ú<module>r»      sV   ðß .Ó .ã Ý Ý $å "Ý (Ý Ý &ôN+˜BŸI™Iô N+ôb_˜BŸI™Iô _ôD<�"—)‘)õ <r,   