Ë
    éÓ^j/2  ã                   óÄ   — d Z ddlmZmZmZ ddlZddlmZ ddlm	Z	 ddl
mZ ddlmZ ddlmZmZ dd	lmZ  G d
„ dej&                  «      Z G d„ dej&                  «      Zy)a:   Attention Pool 2D

Implementations of 2D spatial feature pooling using multi-head attention instead of average pool.

Based on idea in CLIP by OpenAI, licensed Apache 2.0
https://github.com/openai/CLIP/blob/3b473b0e682c091a9e53623eebc1ca1657385717/clip/model.py

Hacked together by / Copyright 2021 Ross Wightman
é    )ÚOptionalÚUnionÚTupleNé   )Úuse_fused_attn)Ú	to_2tuple)Úresample_abs_pos_embed)Úapply_rot_embed_catÚcreate_rope_embed)Útrunc_normal_c                   óP  ‡ — e Zd ZU dZej
                  j                  e   ed<   	 	 	 	 	 	 	 	 	 	 	 	 	 dde	de
e	   dee	ee	e	f   f   de
e	   de
e	   de
e	   d	ed
ededededefˆ fd„Zddefd„Zdde
e	   de
e   fd„Zdej$                  de	de	dej$                  fd„Zddefd„Zˆ xZS )ÚRotAttentionPool2da�   Attention based 2D feature pooling w/ rotary (relative) pos embedding.
    This is a multi-head attention based replacement for (spatial) average pooling in NN architectures.

    Adapted from the AttentionPool2d in CLIP w/ rotary embedding instead of learned embed.
    https://github.com/openai/CLIP/blob/3b473b0e682c091a9e53623eebc1ca1657385717/clip/model.py

    NOTE: While this impl does not require a fixed feature size, performance at differeing resolutions from
    train varies widely and falls off dramatically. I'm not sure if there is a way around this... -RW

    Setting out_features=0 disables the output projection (pre_logits mode).
    Ú
fused_attnÚin_featuresÚout_featuresÚref_feat_sizeÚ	embed_dimÚhead_dimÚ	num_headsÚqkv_biasÚqkv_separateÚ	pool_typeÚclass_tokenÚ	drop_rateÚ	rope_typec           
      óê  •— ||dœ}t         ‰| �  «        |	dv sJ ‚|xs |x| _        }|| _        |€|| _        n|dkD  r|| _        n|| _        t        |«      }|�||z  dk(  sJ ‚||z  }n||z  dk(  sJ ‚||z  }|| _        || _        |	j                  «       | _	        | j                  dz  | _
        t        «       | _        || _        |
r0t        j                  t!        j"                  d|fi |¤Ž«      | _        nd | _        |rbt        j&                  ||fd|i|¤Ž| _        t        j&                  ||fd|i|¤Ž| _        t        j&                  ||fd|i|¤Ž| _        d | _        n!t        j&                  ||dz  fd|i|¤Ž| _        t        j0                  |«      | _        |dk7  r!t        j&                  || j                  fi |¤Žnt        j4                  «       | _        t9        d
|||d|dd	œ|¤Ž| _        y )N©ÚdeviceÚdtype©Ú Útokenr   ç      à¿r   Úbiasé   F)r   Údimr   Ú	in_pixelsÚref_feat_shapeÚrotate_half© )ÚsuperÚ__init__r   r   r   r   r   r   Úlowerr   Úscaler   r   r   ÚnnÚ	ParameterÚtorchÚzerosÚ	cls_tokenÚLinearÚqÚkÚvÚqkvÚDropoutÚdropÚIdentityÚprojr   Ú	pos_embed)Úselfr   r   r   r   r   r   r   r   r   r   r   r   r   r   ÚddÚ	__class__s                   €úg/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/layers/attention_pool2d.pyr,   zRotAttentionPool2d.__init__$   s  ø€ ð" ¨Ñ/ˆÜ‰ÑÔØ˜MÑ)Ð)Ð)Ø%.Ò%=°+Ð=ˆŒ˜Ø&ˆÔØÐØ +ˆDÕØ˜AÒØ ,ˆDÕà )ˆDÔÜ! -Ó0ˆØÐ Ø˜yÑ(¨AÒ-Ð-Ð-Ø  IÑ-‰Hà˜xÑ'¨1Ò,Ð,Ð,Ø! XÑ-ˆIØ"ˆŒØ ˆŒØ"Ÿ™Ó*ˆŒØ—]‘] dÑ*ˆŒ
Ü(Ó*ˆŒØ"ˆŒáÜŸ\™\¬%¯+©+°a¸Ñ*IÀbÑ*IÓJˆD�Nà!ˆDŒNáÜ—Y‘Y˜{¨IÑK¸HÐKÈÑKˆDŒFÜ—Y‘Y˜{¨IÑK¸HÐKÈÑKˆDŒFÜ—Y‘Y˜{¨IÑK¸HÐKÈÑKˆDŒFØˆD�Hä—y‘y ¨i¸!©mÑQÀ(ÐQÈbÑQˆDŒHÜ—J‘J˜yÓ)ˆŒ	ØEQÐUVÒEV”B—I‘I˜i¨×):Ñ):ÑA¸bÒAÔ\^×\gÑ\gÓ\iˆŒ	ä*ð 
ØØØØØ(Øñ
ð ñ
ˆ�ó    Úzero_init_lastc                 ó0  — | j                   �€| j                  j                  }t        | j                  j                  |dz  ¬«       t
        j                  j                  | j                  j                  «       t        | j                  j                  |dz  ¬«       t
        j                  j                  | j                  j                  «       t        | j                  j                  |dz  ¬«       t
        j                  j                  | j                  j                  «       y | j                   j                  }t        | j                   j                  |dz  ¬«       t
        j                  j                  | j                   j                  «       y ©Nr#   )Ústd)r8   r5   r   r   Úweightr/   ÚinitÚzeros_r$   r6   r7   ©r>   rC   r   s      rA   Úinit_weightszRotAttentionPool2d.init_weightsg   sæ   € Ø�8‰8ÑØŸ&™&×,Ñ,ˆKÜ˜$Ÿ&™&Ÿ-™-¨[¸DÑ-@ÕAÜ�G‰G�N‰N˜4Ÿ6™6Ÿ;™;Ô'Ü˜$Ÿ&™&Ÿ-™-¨[¸DÑ-@ÕAÜ�G‰G�N‰N˜4Ÿ6™6Ÿ;™;Ô'Ü˜$Ÿ&™&Ÿ-™-¨[¸DÑ-@ÕAÜ�G‰G�N‰N˜4Ÿ6™6Ÿ;™;Õ'àŸ(™(×.Ñ.ˆKÜ˜$Ÿ(™(Ÿ/™/¨{¸dÑ/BÕCÜ�G‰G�N‰N˜4Ÿ8™8Ÿ=™=Õ)rB   Únum_classesc                 óÔ   — |�|dv sJ ‚|| _         |�W|dkD  r t        j                  | j                  |«      nt        j                  «       | _        |dkD  r|n| j                  | _        y y ©Nr    r   ©r   r/   r4   r   r;   r<   r   ©r>   rL   r   s      rA   ÚresetzRotAttentionPool2d.resetu   óe   € àÐ Ø Ñ-Ð-Ð-Ø&ˆDŒNØÐ"ØBMÐPQÂ/œŸ	™	 $§.¡.°+Ô>ÔWY×WbÑWbÓWdˆDŒIØ/:¸Qª¡ÀDÇNÁNˆDÕð #rB   ÚxÚHÚWÚreturnc                 ó²   — | j                   dk(  r|d d …df   }|S |d d …dd …f   j                  |j                  d   ||d«      j                  dddd«      }|S ©Nr"   r   r   éÿÿÿÿr%   é   ©r   ÚreshapeÚshapeÚpermute©r>   rS   rT   rU   s       rA   Ú_poolzRotAttentionPool2d._pool~   ób   € Ø�>‰>˜WÒ$Ø’!�Q�$‘ˆAð ˆð ’!�Q‘R�%‘× Ñ  §¡¨¡¨Q°°2Ó6×>Ñ>¸qÀ!ÀQÈÓJˆAØˆrB   Ú
pre_logitsc                 ó~  — |j                   \  }}}}||z  }|j                  d«      j                  dd«      }| j                  €+t	        j
                  |j                  dd¬«      |gd¬«      }nAt	        j
                  | j                  j                  |j                   d   dd«      |gd¬«      }| j                  €ß| j                  |«      j                  ||dz   | j                  | j                  «      j                  dd«      }| j                  |«      j                  ||dz   | j                  | j                  «      j                  dd«      }	| j                  |«      j                  ||dz   | j                  | j                  «      j                  dd«      }
nc| j                  |«      j                  ||dz   d| j                  | j                  «      j                  ddddd	«      }|j!                  d«      \  }}	}
| j"                  j%                  ||f«      }t'        |t(        «      rt	        j
                  |d¬«      }t	        j
                  |d d …d d …d d…d d …f   t+        |d d …d d …dd …d d …f   |«      gd¬«      j-                  |
«      }t	        j
                  |	d d …d d …d d…d d …f   t+        |	d d …d d …dd …d d …f   |«      gd¬«      j-                  |
«      }	| j.                  r"t0        j2                  j5                  ||	|
«      }n;|| j6                  z  }||	j                  d
d«      z  }|j9                  d¬«      }||
z  }|j                  dd«      j                  ||dz   d«      }| j;                  |«      }|r| j=                  |||«      }|S | j?                  |«      }| j=                  |||«      }|S )NrZ   r   T©Úkeepdim©r&   r   rY   r%   é   éþÿÿÿ) r]   ÚflattenÚ	transposer3   r1   ÚcatÚmeanÚexpandr8   r5   r\   r   r   r6   r7   r^   Úunbindr=   Ú	get_embedÚ
isinstanceÚtupler
   Útype_asr   r/   Ú
functionalÚscaled_dot_product_attentionr.   Úsoftmaxr:   r`   r<   )r>   rS   rb   ÚBÚ_rT   rU   ÚNr5   r6   r7   ÚropeÚattns                rA   ÚforwardzRotAttentionPool2d.forward†   s(  € Ø—W‘W‰
ˆˆ1ˆa�Ø�‰EˆØ�I‰I�a‹L×"Ñ" 1 aÓ(ˆØ�>‰>Ð!Ü—	‘	˜1Ÿ6™6 !¨T˜6Ó2°AÐ6¸AÔ>‰Aä—	‘	˜4Ÿ>™>×0Ñ0°·±¸±¸RÀÓDÀaÐHÈaÔPˆAØ�8‰8ÐØ—‘�q“	×!Ñ! ! Q¨¡U¨D¯N©N¸D¿M¹MÓJ×TÑTÐUVÐXYÓZˆAØ—‘�q“	×!Ñ! ! Q¨¡U¨D¯N©N¸D¿M¹MÓJ×TÑTÐUVÐXYÓZˆAØ—‘�q“	×!Ñ! ! Q¨¡U¨D¯N©N¸D¿M¹MÓJ×TÑTÐUVÐXYÓZ‰Aà—‘˜“×#Ñ# A q¨1¡u¨a°·±ÀÇÁÓO×WÑWÐXYÐ[\Ð^_ÐabÐdeÓfˆAØ—h‘h˜q“k‰GˆAˆq�!à�~‰~×'Ñ'¨¨A¨Ó/ˆÜ�dœEÔ"ä—9‘9˜T rÔ*ˆDÜ�I‰I�qššA˜r ˜r¢1˜‘~Ô':¸1ºQÂÀ1Á2Âq¸[¹>È4Ó'PÐQÐWXÔY×aÑaÐbcÓdˆÜ�I‰I�qššA˜r ˜r¢1˜‘~Ô':¸1ºQÂÀ1Á2Âq¸[¹>È4Ó'PÐQÐWXÔY×aÑaÐbcÓdˆà�?Š?Ü—‘×:Ñ:¸1¸aÀÓC‰Aà�D—J‘J‘ˆAØ�q—{‘{ 2 rÓ*Ñ*ˆDØ—<‘< B�<Ó'ˆDØ�q‘ˆAØ�K‰K˜˜1Ó×%Ñ% a¨¨Q©°Ó3ˆØ�I‰I�a‹LˆÙØ—
‘
˜1˜a Ó#ˆAØˆHØ�I‰I�a‹LˆØ�J‰J�q˜!˜QÓˆØˆrB   )Né   Né@   NTFr"   Fç        rk   NN©F©NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r1   ÚjitÚFinalÚboolÚ__annotations__Úintr   r   r   ÚstrÚfloatr,   rK   rQ   ÚTensorr`   r{   Ú__classcell__©r@   s   @rA   r   r      sP  ø… ñ
ð —	‘	—‘ Ñ%Ó%ð
 +/Ø9:Ø'+Ø&(Ø'+Ø!Ø!&Ø$Ø %Ø!Ø"ØØñA
àðA
ð # 3™-ðA
ð !  e¨C°¨H¡oÐ!5Ñ6ð	A
ð
   ‘}ðA
ð ˜s‘mðA
ð   ‘}ðA
ð ðA
ð ðA
ð ðA
ð ðA
ð ðA
ð õA
ñF*¨4ó *ñS ¨#¡ð SÀ(È3Á-ó Sð�u—|‘|ð ¨ð °ð ¸¿¹ó ñ% T÷ %rB   r   c                   óJ  ‡ — e Zd ZU dZej
                  j                  e   ed<   	 	 	 	 	 	 	 	 	 	 	 	 dde	de
e	ee	e	f   f   dee	   dee	   dee	   dee	   d	ed
edededefˆ fd„Zddefd„Zddee	   dee   fd„Zdej$                  de	de	dej$                  fd„Zddefd„Zˆ xZS )ÚAttentionPool2da÷   Attention based 2D feature pooling w/ learned (absolute) pos embedding.
    This is a multi-head attention based replacement for (spatial) average pooling in NN architectures.

    It was based on impl in CLIP by OpenAI
    https://github.com/openai/CLIP/blob/3b473b0e682c091a9e53623eebc1ca1657385717/clip/model.py

    NOTE: This requires feature size upon construction and well prevent adaptive sizing of the network.

    Setting out_features=0 disables the output projection (pre_logits mode).
    r   r   Ú	feat_sizer   r   r   r   r   r   r   r   r   c                 ó¦  •— ||dœ}t         ‰| �  «        |	dv sJ ‚|xs |x| _        }|| _        |€|| _        n|dkD  r|| _        n|| _        |�||z  dk(  sJ ‚||z  }n||z  dk(  sJ ‚||z  }t        |«      | _        | j                  d   | j                  d   z  | _        || _        || _	        |	| _
        | j                  dz  | _        t        «       | _        |
r0t        j                  t!        j"                  d|fi |¤Ž«      | _        nd | _        |rbt        j&                  ||fd|i|¤Ž| _        t        j&                  ||fd|i|¤Ž| _        t        j&                  ||fd|i|¤Ž| _        d | _        n6d x| _        x| _        | _        t        j&                  ||dz  fd|i|¤Ž| _        t        j0                  |«      | _        |dk7  r!t        j&                  || j                  fi |¤Žnt        j4                  «       | _        t        j                  t!        j"                  | j                  dz   |fi |¤Ž«      | _        | j;                  «        y )Nr   r    r   r   r#   r$   r%   )r+   r,   r   r   r   r   r‘   Úseq_lenr   r   r   r.   r   r   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   rK   )r>   r   r‘   r   r   r   r   r   r   r   r   r   r   r   r?   r@   s                  €rA   r,   zAttentionPool2d.__init__»   s4  ø€ ð  ¨Ñ/ˆÜ‰ÑÔØ˜MÑ)Ð)Ð)Ø%.Ò%=°+Ð=ˆŒ˜Ø&ˆÔØÐØ +ˆDÕØ˜AÒØ ,ˆDÕà )ˆDÔØÐ Ø˜yÑ(¨AÒ-Ð-Ð-Ø  IÑ-‰Hà˜xÑ'¨1Ò,Ð,Ð,Ø! XÑ-ˆIÜ" 9Ó-ˆŒØ—~‘~ aÑ(¨4¯>©>¸!Ñ+<Ñ<ˆŒØ"ˆŒØ ˆŒØ"ˆŒØ—]‘] dÑ*ˆŒ
Ü(Ó*ˆŒáÜŸ\™\¬%¯+©+°a¸Ñ*IÀbÑ*IÓJˆD�Nà!ˆDŒNáÜ—Y‘Y˜{¨IÑK¸HÐKÈÑKˆDŒFÜ—Y‘Y˜{¨IÑK¸HÐKÈÑKˆDŒFÜ—Y‘Y˜{¨IÑK¸HÐKÈÑKˆDŒFØˆD�Hà'+Ð+ˆDŒFÐ+�T”V˜dœfÜ—y‘y ¨i¸!©mÑQÀ(ÐQÈbÑQˆDŒHÜ—J‘J˜yÓ)ˆŒ	ØEQÐUVÒEV”B—I‘I˜i¨×):Ñ):ÑA¸bÒAÔ\^×\gÑ\gÓ\iˆŒ	ÜŸ™¤e§k¡k°$·,±,ÀÑ2BÀKÑ&VÐSUÑ&VÓWˆŒà×ÑÕrB   rC   c                 ód  — | j                   �€| j                  j                  }t        | j                  j                  |dz  ¬«       t
        j                  j                  | j                  j                  «       t        | j                  j                  |dz  ¬«       t
        j                  j                  | j                  j                  «       t        | j                  j                  |dz  ¬«       t
        j                  j                  | j                  j                  «       nm| j                   j                  }t        | j                   j                  |dz  ¬«       t
        j                  j                  | j                   j                  «       t        | j                  |dz  ¬«       y rE   )r8   r5   r   r   rG   r/   rH   rI   r$   r6   r7   r=   rJ   s      rA   rK   zAttentionPool2d.init_weights÷   sù   € Ø�8‰8ÑØŸ&™&×,Ñ,ˆKÜ˜$Ÿ&™&Ÿ-™-¨[¸DÑ-@ÕAÜ�G‰G�N‰N˜4Ÿ6™6Ÿ;™;Ô'Ü˜$Ÿ&™&Ÿ-™-¨[¸DÑ-@ÕAÜ�G‰G�N‰N˜4Ÿ6™6Ÿ;™;Ô'Ü˜$Ÿ&™&Ÿ-™-¨[¸DÑ-@ÕAÜ�G‰G�N‰N˜4Ÿ6™6Ÿ;™;Õ'àŸ(™(×.Ñ.ˆKÜ˜$Ÿ(™(Ÿ/™/¨{¸dÑ/BÕCÜ�G‰G�N‰N˜4Ÿ8™8Ÿ=™=Ô)Ü�d—n‘n¨+¸Ñ*=Ö>rB   rL   c                 óÔ   — |�|dv sJ ‚|| _         |�W|dkD  r t        j                  | j                  |«      nt        j                  «       | _        |dkD  r|n| j                  | _        y y rN   rO   rP   s      rA   rQ   zAttentionPool2d.reset  rR   rB   rS   rT   rU   rV   c                 ó²   — | j                   dk(  r|d d …df   }|S |d d …dd …f   j                  |j                  d   ||d«      j                  dddd«      }|S rX   r[   r_   s       rA   r`   zAttentionPool2d._pool  ra   rB   rb   c                 ó  — |j                   \  }}}}||z  }|j                  d«      j                  dd«      }| j                  €+t	        j
                  |j                  dd¬«      |gd¬«      }nAt	        j
                  | j                  j                  |j                   d   dd«      |gd¬«      }t        | j                  j                  d«      ||fd¬«      }||z   }| j                  €ß| j                  |«      j                  ||dz   | j                  | j                  «      j                  dd«      }	| j!                  |«      j                  ||dz   | j                  | j                  «      j                  dd«      }
| j#                  |«      j                  ||dz   | j                  | j                  «      j                  dd«      }n`| j                  |«      j                  |dd	| j                  | j                  «      j%                  ddd	dd
«      }|j'                  d«      \  }	}
}| j(                  r"t*        j,                  j/                  |	|
|«      }n;|	| j0                  z  }	|	|
j                  dd«      z  }|j3                  d¬«      }||z  }|j                  dd«      j                  ||dz   d«      }| j5                  |«      }|r| j7                  |||«      }|S | j9                  |«      }| j7                  |||«      }|S )NrZ   r   Trd   rf   r   rY   )Únum_prefix_tokensr%   rg   rh   )r]   ri   rj   r3   r1   rk   rl   rm   r	   r=   Ú	unsqueezer8   r5   r\   r   r   r6   r7   r^   rn   r   r/   rs   rt   r.   ru   r:   r`   r<   )r>   rS   rb   rv   rw   rT   rU   rx   r=   r5   r6   r7   rz   s                rA   r{   zAttentionPool2d.forward  s�  € Ø—W‘W‰
ˆˆ1ˆa�Ø�‰EˆØ�I‰I�a‹L×"Ñ" 1 aÓ(ˆØ�>‰>Ð!Ü—	‘	˜1Ÿ6™6 !¨T˜6Ó2°AÐ6¸AÔ>‰Aä—	‘	˜4Ÿ>™>×0Ñ0°·±¸±¸RÀÓDÀaÐHÈaÔPˆAÜ*¨4¯>©>×+CÑ+CÀAÓ+FÈÈAÈÐbcÔdˆ	Ø�	‰Mˆà�8‰8ÐØ—‘�q“	×!Ñ! ! Q¨¡U¨D¯N©N¸D¿M¹MÓJ×TÑTÐUVÐXYÓZˆAØ—‘�q“	×!Ñ! ! Q¨¡U¨D¯N©N¸D¿M¹MÓJ×TÑTÐUVÐXYÓZˆAØ—‘�q“	×!Ñ! ! Q¨¡U¨D¯N©N¸D¿M¹MÓJ×TÑTÐUVÐXYÓZ‰Aà—‘˜“×#Ñ# A r¨1¨d¯n©n¸d¿m¹mÓL×TÑTÐUVÐXYÐ[\Ð^_ÐabÓcˆAØ—h‘h˜q“k‰GˆAˆq�!à�?Š?Ü—‘×:Ñ:¸1¸aÀÓC‰Aà�D—J‘J‘ˆAØ�q—{‘{ 2 rÓ*Ñ*ˆDØ—<‘< B�<Ó'ˆDØ�q‘ˆAØ�K‰K˜˜1Ó×%Ñ% a¨¨Q©°Ó3ˆØ�I‰I�a‹LˆÙØ—
‘
˜1˜a Ó#ˆAØˆHØ�I‰I�a‹LˆØ�J‰J�q˜!˜QÓˆØˆrB   )r|   NNr}   NTFr"   Fr~   NNr   r€   )r�   r‚   rƒ   r„   r1   r…   r†   r‡   rˆ   r‰   r   r   r   rŠ   r‹   r,   rK   rQ   rŒ   r`   r{   r�   rŽ   s   @rA   r�   r�   ®   s6  ø… ñ	ð —	‘	—‘ Ñ%Ó%ð
 67Ø*.Ø'+Ø&(Ø'+Ø!Ø!&Ø$Ø %Ø!ØØñ:àð:ð ˜S %¨¨S¨¡/Ð1Ñ2ð:ð # 3™-ð	:ð
   ‘}ð:ð ˜s‘mð:ð   ‘}ð:ð ð:ð ð:ð ð:ð ð:ð õ:ñx?¨4ó ?ñS ¨#¡ð SÀ(È3Á-ó Sð�u—|‘|ð ¨ð °ð ¸¿¹ó ñ! T÷ !rB   r�   )r„   Útypingr   r   r   r1   Útorch.nnr/   Úconfigr   Úhelpersr   r=   r	   Úpos_embed_sincosr
   r   Úweight_initr   ÚModuler   r�   r*   rB   rA   ú<module>r¡      sL   ðñ÷ *Ñ )ã Ý å "Ý Ý -ß DÝ &ôU˜Ÿ™ô UôpJ�b—i‘iõ JrB   