Ë
    éÓ^jÙ\  ã            
       óþ  — d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmc mZ ddlmZmZ ddlmZmZmZmZmZmZ ddlmZ dd	lmZ dd
lm Z  ddl!m"Z"m#Z# ddl$m%Z% dgZ&ee'e'f   Z(e  G d„ dejR                  «      «       Z* G d„ dejR                  «      Z+ G d„ dejR                  «      Z, G d„ dejR                  «      Z- G d„ dejR                  «      Z. G d„ dejR                  «      Z/d%d„Z0d&d„Z1 e# e1d¬«       e1d¬«       e1d¬«       e1d¬«       e1d¬«       e1d¬«      dœ«      Z2e"d%de/fd„«       Z3e"d%de/fd „«       Z4e"d%de/fd!„«       Z5e"d%de/fd"„«       Z6e"d%de/fd#„«       Z7e"d%de/fd$„«       Z8y)'zô Twins
A PyTorch impl of : `Twins: Revisiting the Design of Spatial Attention in Vision Transformers`
    - https://arxiv.org/pdf/2104.13840.pdf

Code/weights from https://github.com/Meituan-AutoML/Twins, original copyright/license info below

é    N)Úpartial)ÚListÚOptionalÚTupleÚUnionÚTypeÚAny©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚMlpÚDropPathÚ	to_2tupleÚtrunc_normal_Úuse_fused_attnÚcalculate_drop_path_ratesé   )Úbuild_model_with_cfg)Úfeature_take_indices)Úregister_notrace_module)Úregister_modelÚgenerate_default_cfgs)Ú	AttentionÚTwinsc                   óŒ   ‡ — e Zd ZU dZej
                  j                  e   ed<   	 	 	 	 	 	 dde	de	de
de
de	f
ˆ fd„Zd	efd
„Zˆ xZS )ÚLocallyGroupedAttnz( LSA: self attention within a group
    Ú
fused_attnÚdimÚ	num_headsÚ	attn_dropÚ	proj_dropÚwsc                 ó®  •— ||dœ}|dk7  sJ ‚t         ‰
| �  «        ||z  dk(  sJ d|› d|› d�«       ‚|| _        || _        ||z  }	|	dz  | _        t        «       | _        t        j                  ||dz  fd	d
i|¤Ž| _	        t        j                  |«      | _        t        j                  ||fi |¤Ž| _        t        j                  |«      | _        || _        y )N©ÚdeviceÚdtyper   r   údim ú  should be divided by num_heads ú.ç      à¿é   ÚbiasT)ÚsuperÚ__init__r   r   Úscaler   r   ÚnnÚLinearÚqkvÚDropoutr    Úprojr!   r"   )Úselfr   r   r    r!   r"   r%   r&   ÚddÚhead_dimÚ	__class__s             €ú\/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/twins.pyr.   zLocallyGroupedAttn.__init__)   sÛ   ø€ ð ¨Ñ/ˆØ�QŠwˆˆwÜ‰ÑÔØ�Y‰ !Ò#Ð] t¨C¨5Ð0PÐQZÐP[Ð[\Ð%]Ó]Ð#àˆŒØ"ˆŒØ˜)Ñ#ˆØ Ñ%ˆŒ
Ü(Ó*ˆŒä—9‘9˜S #¨¡'Ñ;°Ð;¸Ñ;ˆŒÜŸ™ IÓ.ˆŒÜ—I‘I˜c 3Ñ-¨"Ñ-ˆŒ	ÜŸ™ IÓ.ˆŒØˆ�ó    Úsizec           	      óà  — |j                   \  }}}|\  }}|j                  ||||«      }dx}}	| j                  || j                  z  z
  | j                  z  }
| j                  || j                  z  z
  | j                  z  }t        j                  |dd||
|	|f«      }|j                   \  }}}}|| j                  z  || j                  z  }}|j                  ||| j                  || j                  |«      j                  dd«      }| j                  |«      j                  |||z  | j                  | j                  z  d| j                  || j                  z  «      j                  dddddd«      }|j                  d«      \  }}}| j                  r<t        j                  |||| j                  r| j                  j                  nd¬«      }nL|| j                   z  }||j                  d	d
«      z  }|j#                  d
¬«      }| j                  |«      }||z  }|j                  dd«      j                  |||| j                  | j                  |«      }|j                  dd«      j                  ||| j                  z  || j                  z  |«      }|
dkD  s|dkD  r|d d …d |…d |…d d …f   j%                  «       }|j                  |||«      }| j'                  |«      }| j)                  |«      }|S )Nr   é   r+   r   é   é   ç        ©Ú	dropout_péþÿÿÿéÿÿÿÿ©r   )ÚshapeÚviewr"   ÚFÚpadÚreshapeÚ	transposer2   r   ÚpermuteÚunbindr   Úscaled_dot_product_attentionÚtrainingr    Úpr/   ÚsoftmaxÚ
contiguousr4   r!   )r5   Úxr;   ÚBÚNÚCÚHÚWÚpad_lÚpad_tÚpad_rÚpad_bÚ_ÚHpÚWpÚ_hÚ_wr2   ÚqÚkÚvÚattns                         r9   ÚforwardzLocallyGroupedAttn.forwardD   sž  € ð —'‘'‰ˆˆ1ˆaØ‰ˆˆ1Ø�F‰F�1�a˜˜AÓˆØÐˆ�Ø—‘˜1˜tŸw™w™;Ñ&¨$¯'©'Ñ1ˆØ—‘˜1˜tŸw™w™;Ñ&¨$¯'©'Ñ1ˆÜ�E‰E�!�a˜˜E 5¨%°Ð7Ó8ˆØ—w‘w‰ˆˆ2ˆr�1Ø�t—w‘w‘  d§g¡g¡ˆBˆØ�I‰I�a˜˜TŸW™W b¨$¯'©'°1Ó5×?Ñ?ÀÀ1ÓEˆØ�h‰h�q‹k×!Ñ!Øˆr�B‰w˜Ÿ™ $§'¡'Ñ)¨1¨d¯n©n¸aÀ4Ç>Á>Ñ>QóSßSZÑSZÐ[\Ð^_ÐabÐdeÐghÐjkÓSlð 	à—*‘*˜Q“-‰ˆˆ1ˆaà�?Š?Ü×.Ñ.Ø�1�aØ.2¯mªm˜$Ÿ.™.×*Ò*Àô‰Að
 �D—J‘J‘ˆAØ�q—{‘{ 2 rÓ*Ñ*ˆDØ—<‘< B�<Ó'ˆDØ—>‘> $Ó'ˆDØ�q‘ˆAà�K‰K˜˜1Ó×%Ñ% a¨¨R°·±¸$¿'¹'À1ÓEˆØ�K‰K˜˜1Ó×%Ñ% a¨¨d¯g©g©°r¸D¿G¹G±|ÀQÓGˆØ�1Š9˜ š	Ø’!�R�a�R˜˜!˜šQ�,‘×*Ñ*Ó,ˆAØ�I‰I�a˜˜AÓˆØ�I‰I�a‹LˆØ�N‰N˜1ÓˆØˆr:   ©é   r@   r@   r   NN©Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚtorchÚjitÚFinalÚboolÚ__annotations__ÚintÚfloatr.   ÚSize_rf   Ú__classcell__©r8   s   @r9   r   r   #   sp   ø… ñà—	‘	—‘ Ñ%Ó%ð
 Ø!Ø!ØØØñàðð ðð ð	ð
 ðð õð6%˜u÷ %r:   r   c                   óŒ   ‡ — e Zd ZU dZej
                  j                  e   ed<   	 	 	 	 	 	 dde	de	de
de
de	f
ˆ fd„Zd	efd
„Zˆ xZS )ÚGlobalSubSampleAttnzQ GSA: using a  key to summarize the information for a group to be efficient.
    r   r   r   r    r!   Úsr_ratioc                 óx  •— ||dœ}t         ‰
| �  «        ||z  dk(  sJ d|› d|› d�«       ‚|| _        || _        ||z  }	|	dz  | _        t        «       | _        t        j                  ||fddi|¤Ž| _	        t        j                  ||d	z  fddi|¤Ž| _
        t        j                  |«      | _        t        j                  ||fi |¤Ž| _        t        j                  |«      | _        || _        |d
kD  r;t        j                   ||f||dœ|¤Ž| _        t        j$                  |fi |¤Ž| _        y d | _        d | _        y )Nr$   r   r'   r(   r)   r*   r,   Tr=   r   ©Úkernel_sizeÚstride)r-   r.   r   r   r/   r   r   r0   r1   rb   Úkvr3   r    r4   r!   rz   ÚConv2dÚsrÚ	LayerNormÚnorm)r5   r   r   r    r!   rz   r%   r&   r6   r7   r8   s             €r9   r.   zGlobalSubSampleAttn.__init__”   s5  ø€ ð ¨Ñ/ˆÜ‰ÑÔØ�Y‰ !Ò#Ð] t¨C¨5Ð0PÐQZÐP[Ð[\Ð%]Ó]Ð#àˆŒØ"ˆŒØ˜)Ñ#ˆØ Ñ%ˆŒ
Ü(Ó*ˆŒä—‘˜3 Ñ5¨$Ð5°"Ñ5ˆŒÜ—)‘)˜C  q¡Ñ:¨tÐ:°rÑ:ˆŒÜŸ™ IÓ.ˆŒÜ—I‘I˜c 3Ñ-¨"Ñ-ˆŒ	ÜŸ™ IÓ.ˆŒà ˆŒØ�aŠ<Ü—i‘i  SÐV°hÀxÑVÐSUÑVˆDŒGÜŸ™ SÑ/¨BÑ/ˆD�IàˆDŒGØˆD�Ir:   r;   c                 óF  — |j                   \  }}}| j                  |«      j                  ||| j                  || j                  z  «      j	                  dddd«      }| j
                  �i |j	                  ddd«      j                  ||g|¢­Ž }| j                  |«      j                  ||d«      j	                  ddd«      }| j                  |«      }| j                  |«      j                  |dd| j                  || j                  z  «      j	                  ddddd«      }|j                  d«      \  }}	| j                  rPt        j                  j                  j                  |||	| j                  r| j                  j                   nd¬«      }nL|| j"                  z  }||j%                  d	d«      z  }
|
j'                  d¬
«      }
| j                  |
«      }
|
|	z  }|j%                  dd«      j                  |||«      }| j)                  |«      }| j+                  |«      }|S )Nr   r=   r   r+   rD   r>   r@   rA   rC   rE   )rF   rb   rJ   r   rL   r�   rƒ   r   rM   r   rn   r0   Ú
functionalrN   rO   r    rP   r/   rK   rQ   r4   r!   )r5   rS   r;   rT   rU   rV   rb   r   rc   rd   re   s              r9   rf   zGlobalSubSampleAttn.forward¶   sÜ  € Ø—'‘'‰ˆˆ1ˆaØ�F‰F�1‹I×Ñ˜a  D§N¡N°A¸¿¹Ñ4GÓH×PÑPÐQRÐTUÐWXÐZ[Ó\ˆà�7‰7ÐØ*�—	‘	˜!˜Q Ó"×*Ñ*¨1¨aÐ7°$Ò7ˆAØ—‘˜“
×"Ñ" 1 a¨Ó,×4Ñ4°Q¸¸1Ó=ˆAØ—	‘	˜!“ˆAØ�W‰W�Q‹Z×Ñ  2 q¨$¯.©.¸!¸t¿~¹~Ñ:MÓN×VÑVÐWXÐZ[Ð]^Ð`aÐcdÓeˆØ�y‰y˜‹|‰ˆˆ1à�?Š?Ü—‘×#Ñ#×@Ñ@Ø�1�aØ.2¯mªm˜$Ÿ.™.×*Ò*Àð Aó ‰Að
 �D—J‘J‘ˆAØ�q—{‘{ 2 rÓ*Ñ*ˆDØ—<‘< B�<Ó'ˆDØ—>‘> $Ó'ˆDØ�q‘ˆAà�K‰K˜˜1Ó×%Ñ% a¨¨AÓ.ˆØ�I‰I�a‹LˆØ�N‰N˜1Óˆàˆr:   rg   ri   rw   s   @r9   ry   ry   �   sq   ø… ñà—	‘	—‘ Ñ%Ó%ð
 Ø!Ø!ØØØñ àð ð ð ð ð	 ð
 ð ð õ ðD˜u÷ r:   ry   c                   óÌ   ‡ — e Zd Zddddej                  ej
                  ddddf
dedededed	ed
edeej                     deej                     dede
e   fˆ fd„Zdefd„Zˆ xZS )ÚBlockg      @r@   r   Nr   r   Ú	mlp_ratior!   r    Ú	drop_pathÚ	act_layerÚ
norm_layerrz   r"   c                 óè  •— t         ‰| �  «        ||dœ} ||fi |¤Ž| _        |
€t        ||dd ||fi |¤Ž| _        n0|
dk(  rt        |||||	fi |¤Ž| _        nt        |||||
fi |¤Ž| _        |dkD  rt        |«      nt        j                  «       | _
         ||fi |¤Ž| _        t        d|t        ||z  «      ||dœ|¤Ž| _        |dkD  rt        |«      | _        y t        j                  «       | _        y )Nr$   Fr   r@   )Úin_featuresÚhidden_featuresrŠ   Údrop© )r-   r.   Únorm1r   re   ry   r   r   r0   ÚIdentityÚ
drop_path1Únorm2r   rs   ÚmlpÚ
drop_path2)r5   r   r   rˆ   r!   r    r‰   rŠ   r‹   rz   r"   r%   r&   r6   r8   s                 €r9   r.   zBlock.__init__Ö   s  ø€ ô 	‰ÑÔØ¨Ñ/ˆÙ Ñ* rÑ*ˆŒ
Øˆ:Ü! # y°%¸¸yÈ)ÑZÐWYÑZˆD�IØ�1ŠWÜ+¨C°¸IÀyÐRZÑaÐ^`ÑaˆD�Iä*¨3°	¸9ÀiÐQSÑZÐWYÑZˆDŒIØ1:¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒá Ñ* rÑ*ˆŒ
Üð 
ØÜ  i¡Ó0ØØñ	
ð
 ñ
ˆŒð 2;¸R²œ( 9Ó-ˆ�ÄRÇ[Á[Ã]ˆ�r:   r;   c                 óÐ   — || j                  | j                  | j                  |«      |«      «      z   }|| j                  | j	                  | j                  |«      «      «      z   }|S ©N)r“   re   r‘   r–   r•   r”   )r5   rS   r;   s      r9   rf   zBlock.forwardú   sQ   € Ø�—‘ §	¡	¨$¯*©*°Q«-¸Ó >Ó?Ñ?ˆØ�—‘ §¡¨¯©°A«Ó 7Ó8Ñ8ˆØˆr:   )rj   rk   rl   r0   ÚGELUr‚   rs   rt   r   ÚModuler   r.   ru   rf   rv   rw   s   @r9   r‡   r‡   Ô   s¾   ø„ ð  "Ø!Ø!Ø!Ø)+¯©Ø*,¯,©,ØØ $ØØñ"Sàð"Sð ð"Sð ð	"Sð
 ð"Sð ð"Sð ð"Sð ˜BŸI™I‘ð"Sð ˜RŸY™Y™ð"Sð ð"Sð ˜‘õ"SðH˜u÷ r:   r‡   c                   óH   ‡ — e Zd Z	 	 	 	 ddededefˆ fd„Zdefd„Zd„ Zˆ xZS )	ÚPosConvÚin_chansÚ	embed_dimr~   c           	      ó¤   •— ||dœ}t         ‰| �  «        t        j                  t        j                  ||d|dfd|dœ|¤Ž«      | _        || _        y )Nr$   r+   r   T)r,   Úgroups)r-   r.   r0   Ú
Sequentialr€   r4   r~   )r5   r�   rž   r~   r%   r&   r6   r8   s          €r9   r.   zPosConv.__init__  sT   ø€ ð ¨Ñ/ˆÜ‰ÑÔÜ—M‘MÜ�I‰I�h 	¨1¨f°aÐ[¸dÈ9Ñ[ÐXZÑ[ó
ˆŒ	ð ˆ�r:   r;   c                 óú   — |j                   \  }}} |j                  dd«      j                  ||g|¢­Ž }| j                  |«      }| j                  dk(  r||z  }|j                  d«      j                  dd«      }|S )Nr   r=   )rF   rK   rG   r4   r~   Úflatten)r5   rS   r;   rT   rU   rV   Úcnn_feat_tokens          r9   rf   zPosConv.forward  sx   € Ø—'‘'‰ˆˆ1ˆaØ/˜Ÿ™ Q¨Ó*×/Ñ/°°1Ð<°tÒ<ˆØ�I‰I�nÓ%ˆØ�;‰;˜!ÒØ�ÑˆAØ�I‰I�a‹L×"Ñ" 1 aÓ(ˆØˆr:   c                 óD   — t        d«      D �cg c]  }d|z  ‘Œ	 c}S c c}w )Nr>   zproj.%d.weight)Úrange)r5   Úis     r9   Úno_weight_decayzPosConv.no_weight_decay  s    € Ü.3°A«hÖ7¨Ð  1Ó$Ò7Ð7ùÒ7s   Ž)é   r   NN)	rj   rk   rl   rs   r.   ru   rf   r¨   rv   rw   s   @r9   rœ   rœ      sC   ø„ ð
 !ØØØñàðð ðð õ	ð˜uó ö8r:   rœ   c            	       ó”   ‡ — e Zd ZdZ	 	 	 	 	 	 d	deeeeef   f   deeeeef   f   dedefˆ fd„Zdeej                  e
f   fd„Zˆ xZS )
Ú
PatchEmbedz Image to Patch Embedding
    Úimg_sizeÚ
patch_sizer�   rž   c                 óº  •— ||dœ}t         ‰| �  «        t        |«      }t        |«      }|| _        || _        |d   |d   z  dk(  r|d   |d   z  dk(  sJ d|› d|› d�«       ‚|d   |d   z  |d   |d   z  c| _        | _        | j
                  | j                  z  | _        t        j                  ||f||dœ|¤Ž| _
        t        j                  |fi |¤Ž| _        y )Nr$   r   r   z	img_size z! should be divided by patch_size r)   r|   )r-   r.   r   r¬   r­   rW   rX   Únum_patchesr0   r€   r4   r‚   rƒ   )	r5   r¬   r­   r�   rž   r%   r&   r6   r8   s	           €r9   r.   zPatchEmbed.__init__"  sû   ø€ ð ¨Ñ/ˆÜ‰ÑÔÜ˜XÓ&ˆÜ˜zÓ*ˆ
à ˆŒØ$ˆŒØ˜‰{˜Z¨™]Ñ*¨aÒ/°H¸Q±KÀ*ÈQÁ-Ñ4OÐSTÒ4Tð 	QØ˜�zÐ!BÀ:À,ÈaÐPó	QÐTà! !™¨
°1©Ñ5°xÀ±{ÀjÐQRÁmÑ7SˆˆŒ�”ØŸ6™6 D§F¡F™?ˆÔÜ—I‘I˜h¨	Ðc¸zÐR\ÑcÐ`bÑcˆŒ	Ü—L‘L Ñ1¨bÑ1ˆ�	r:   Úreturnc                 óö   — |j                   \  }}}}| j                  |«      j                  d«      j                  dd«      }| j	                  |«      }|| j
                  d   z  || j
                  d   z  f}||fS )Nr=   r   r   )rF   r4   r£   rK   rƒ   r­   )r5   rS   rT   rV   rW   rX   Úout_sizes          r9   rf   zPatchEmbed.forward9  su   € Ø—W‘W‰
ˆˆ1ˆa�à�I‰I�a‹L× Ñ  Ó#×-Ñ-¨a°Ó3ˆØ�I‰I�a‹LˆØ˜Ÿ™¨Ñ+Ñ+¨Q°$·/±/À!Ñ2DÑ-DÐEˆà�(ˆ{Ðr:   )éà   é   r+   r©   NN)rj   rk   rl   rm   r   rs   r   r.   rn   ÚTensorru   rf   rv   rw   s   @r9   r«   r«     s…   ø„ ñð
 58Ø68ØØ ØØñ2à˜C  s¨C x¡Ð0Ñ1ð2ð ˜c 5¨¨c¨¡?Ð2Ñ3ð2ð ð	2ð
 õ2ð.˜E %§,¡,°Ð"5Ñ6÷ r:   r«   c            %       óB  ‡ — e Zd ZdZdddddddd	d
ddddddd eej                  d¬«      eddfdee	e
e	e	f   f   de	de	de	dede
e	df   de
e	df   de
edf   de
e	df   de
e	df   dee
e	df      dedededed ed!eej                     d"ef$ˆ fd#„Zej&                  j(                  d$„ «       Zej&                  j(                  d9d%„«       Zej&                  j(                  d:d&„«       Zej&                  j(                  d'ej                  fd(„«       Zd;de	dee   fd)„Zd*„ Z	 	 	 	 	 d<d+ej6                  d,eee	ee	   f      d-ed.ed/ed0ed'eeej6                     e
ej6                  eej6                     f   f   fd1„Z	 	 	 d=d,ee	ee	   f   d2ed3efd4„Zd5„ Z d9d6efd7„Z!d8„ Z"ˆ xZ#S )>r   zš Twins Vision Transformer (Revisiting Spatial Attention)

    Adapted from PVT (PyramidVisionTransformer) class at https://github.com/whai362/PVT.git
    r³   r>   r+   éè  Úavg©é@   é€   é   é   )r   r=   r>   rh   ©r>   r>   r>   r>   ©r+   r>   é   r+   ©rh   r>   r=   r   Nr@   g�íµ ÷Æ°>)Úepsr¬   r­   r�   Únum_classesÚglobal_poolÚ
embed_dims.r   Ú
mlp_ratiosÚdepthsÚ	sr_ratiosÚwssÚ	drop_rateÚpos_drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_rater‹   Ú	block_clsc                 óÒ  •‡— t         ‰| �  «        ||dœ}|| _        || _        || _        |	| _        || _        |d   x| _        | _        d| _	        t        |«      }|}t        j                  «       | _        t        j                  «       | _        t        t!        |	«      «      D ]w  }| j                  j#                  t%        |‰|||   fi |¤Ž«       | j                  j#                  t        j&                  |¬«      «       ||   }t)        ˆfd„|D «       «      }dŠŒy t        j                  «       | _        g | _        t/        |t1        |	«      «      }d}t        t!        |	«      «      D ]³  }t        j                  t        |	|   «      D �cg c]5  } |d||   ||   ||   |||||z      ||
|   |�|dz  dk(  rdn||   d	œ	|¤Ž‘Œ7 c}«      }| j*                  j#                  |«       | xj,                  t3        d
|› �||   dd|z   z  ¬«      gz  c_        ||	|   z  }Œµ t        j                  |D �cg c]  }t5        ||fi |¤Ž‘Œ c}«      | _         || j                  fi |¤Ž| _        t        j&                  |«      | _        |dkD  r!t        j<                  | j                  |fi |¤Žnt        j>                  «       | _         | jC                  | jD                  «       y c c}w c c}w )Nr$   rD   F)rP   c              3   ó(   •K  — | ]	  }|‰z  –— Œ y ­wr˜   r�   )Ú.0Útr­   s     €r9   ú	<genexpr>z!Twins.__init__.<locals>.<genexpr>q  s   øè ø€ Ò?°˜Q *�_Ñ?ùs   ƒr=   r   r   )	r   r   rˆ   r!   r    r‰   r‹   rz   r"   zblock.)ÚmoduleÚnum_chsÚ	reductionr�   )#r-   r.   rÃ   r�   rÄ   rÇ   rÅ   Únum_featuresÚhead_hidden_sizeÚgrad_checkpointingr   r0   Ú
ModuleListÚpatch_embedsÚ	pos_dropsr¦   ÚlenÚappendr«   r3   ÚtupleÚblocksÚfeature_infor   ÚsumÚdictrœ   Ú	pos_blockrƒ   Ú	head_dropr1   r’   ÚheadÚapplyÚ_init_weights)r5   r¬   r­   r�   rÃ   rÄ   rÅ   r   rÆ   rÇ   rÈ   rÉ   rÊ   rË   rÌ   rÍ   rÎ   r‹   rÏ   r%   r&   r6   Úprev_chsr§   ÚdprÚcurrc   Ú_blockrž   r8   s     `                          €r9   r.   zTwins.__init__H  sË  ù€ ô. 	‰ÑÔØ¨Ñ/ˆØ&ˆÔØ ˆŒØ&ˆÔØˆŒØ$ˆŒØ4>¸r±NÐBˆÔ˜DÔ1Ø"'ˆÔä˜XÓ&ˆØˆÜŸM™M›OˆÔÜŸ™›ˆŒÜ”s˜6“{Ó#ò 	ˆAØ×Ñ×$Ñ$¤Z°¸*ÀhÐPZÐ[\ÑP]Ñ%dÐacÑ%dÔeØ�N‰N×!Ñ!¤"§*¡*¨}Ô"=Ô>Ø! !‘}ˆHÜÓ?°hÔ?Ó?ˆHØ‰Jð	ô —m‘m“oˆŒØˆÔÜ'¨¼¸F»ÓDˆØˆÜ”s˜6“{Ó#ò 	ˆAÜ—]‘]ô ˜V A™YÓ'ö$)ð ñ %.ð %Ø˜q‘MØ# A™,Ø$ Q™-Ø(Ø(Ø˜c A™g™,Ø%Ø" 1™Ø˜ q¨1¡u°¢z‘1°s¸1±vñ%ð ó%ò $)ó *ˆFð �K‰K×Ñ˜vÔ&Ø×Ò¤$°°q°c¨lÀJÈqÁMÐ]^ÐabÐcdÑadÑ]eÔ"fÐ!gÑgÕØ�6˜!‘9Ñ‰Cð	ô" Ÿ™Ð]gÖ'hÐPY¬°	¸9Ñ(KÈÓ(KÒ'hÓiˆŒá˜t×0Ñ0Ñ7°BÑ7ˆŒ	ô Ÿ™ IÓ.ˆŒØGRÐUVÂ”B—I‘I˜d×/Ñ/°ÑCÀÒCÔ\^×\gÑ\gÓ\iˆŒ	ð 	�
‰
�4×%Ñ%Õ&ùò3$)ùò  (is   Æ:K
È2K$c                 ó€   — t        | j                  j                  «       D ��cg c]
  \  }}d|z   ‘Œ c}}«      S c c}}w )Nz
pos_block.)Úsetrå   Únamed_parameters)r5   ÚnrP   s      r9   r¨   zTwins.no_weight_decay”  s1   € ä°·±×1PÑ1PÓ1R×S©¨¨A�L 1Ó$ÓSÓTÐTùÓSs   £:
c                 ó0   — t        d|rddgng d¢¬«      }|S )Nz^patch_embeds.0)z)^(?:blocks|patch_embeds|pos_block)\.(\d+)N©z^norm)iŸ† ))z^blocks\.(\d+)\.(\d+)N)z"^(?:patch_embeds|pos_block)\.(\d+))r   ró   )Ústemrá   )rä   )r5   ÚcoarseÚmatchers      r9   Úgroup_matcherzTwins.group_matcher˜  s.   € äØ#ñ ð EØ#ñòô

ˆð ˆr:   c                 ó   — |rJ d«       ‚y )Nz$gradient checkpointing not supportedr�   )r5   Úenables     r9   Úset_grad_checkpointingzTwins.set_grad_checkpointing§  s   € áÐAÐAÓAˆz�6r:   r°   c                 ó   — | j                   S r˜   )rç   )r5   s    r9   Úget_classifierzTwins.get_classifier«  s   € à�y‰yÐr:   c                 óž  — || _         |�|dv sJ ‚|| _        t        | j                  d«      r | j                  j                  j
                  nd }t        | j                  d«      r | j                  j                  j                  nd }|dkD  r)t        j                  | j                  |||¬«      | _        y t        j                  «       | _        y )N)Ú r¸   Úweightr   r$   )rÃ   rÄ   Úhasattrrç   rÿ   r%   r&   r0   r1   rØ   r’   )r5   rÃ   rÄ   r%   r&   s        r9   Úreset_classifierzTwins.reset_classifier¯  s£   € Ø&ˆÔØÐ"Ø +Ñ-Ð-Ð-Ø*ˆDÔÜ,3°D·I±I¸xÔ,H�—‘×!Ñ!×(Ò(ÈdˆÜ*1°$·)±)¸XÔ*F�—	‘	× Ñ ×&Ò&ÈDˆØ]hÐklÒ]l”B—I‘I˜d×/Ñ/°ÀVÐSXÔYˆ�	Ôrt×r}Ñr}Órˆ�	r:   c                 óp  — t        |t        j                  «      rjt        |j                  d¬«       t        |t        j                  «      r8|j
                  �+t        j                  j                  |j
                  d«       y y y t        |t        j                  «      rUt        j                  j                  |j
                  d«       t        j                  j                  |j                  d«       y t        |t        j                  «      r©|j                  d   |j                  d   z  |j                  z  }||j                  z  }|j                  j                  j                  dt        j                   d|z  «      «       |j
                  �%|j
                  j                  j#                  «        y y y )Ng{®Gáz”?)Ústdr   g      ð?r   g       @)Ú
isinstancer0   r1   r   rÿ   r,   ÚinitÚ	constant_r‚   r€   r}   Úout_channelsr    ÚdataÚnormal_ÚmathÚsqrtÚzero_)r5   ÚmÚfan_outs      r9   ré   zTwins._init_weights¸  s!  € Ü�aœŸ™Ô#Ü˜!Ÿ(™(¨Õ,Ü˜!œRŸY™YÔ'¨A¯F©FÐ,>Ü—‘×!Ñ! !§&¡&¨!Õ,ð -?Ð'ä˜œ2Ÿ<™<Ô(Ü�G‰G×Ñ˜aŸf™f aÔ(Ü�G‰G×Ñ˜aŸh™h¨Õ,Ü˜œ2Ÿ9™9Ô%Ø—m‘m AÑ&¨¯©°qÑ)9Ñ9¸A¿N¹NÑJˆGØ˜Ÿ™Ñ ˆGØ�H‰H�M‰M×!Ñ! !¤T§Y¡Y¨s°W©}Ó%=Ô>Ø�v‰vÐ!Ø—‘—‘×!Ñ!Õ#ð "ð	 &r:   rS   Úindicesrƒ   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc           	      óX  — |dk(  sJ d«       ‚g }t        t        | j                  «      |«      \  }}	|j                  \  }
}}}t	        t        | j                  | j                  | j                  | j                  «      «      D �]  \  }\  }}}} ||«      \  }} ||«      }t	        |«      D ]  \  }} |||«      }|dk(  sŒ |||«      }Œ |t        | j                  «      dz
  k  rL |j                  |
g|¢d‘­Ž j                  dddd«      j                  «       }||v sŒŸ|j                  |«       Œ±||v sŒ¶|r| j                  |«      n|}|j                   |j                  |
g|¢d‘­Ž j                  dddd«      j                  «       «       �Œ |r|S | j                  |«      }||fS )aç   Forward features that returns intermediates.
        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        ÚNCHWz$Output shape for Twins must be NCHW.r   r   rD   r+   r=   )r   rÞ   rá   rF   Ú	enumerateÚziprÜ   rÝ   rå   rÇ   rJ   rL   rR   rß   rƒ   )r5   rS   r  rƒ   r  r  r  ÚintermediatesÚtake_indicesÚ	max_indexrT   r]   ÚheightÚwidthr§   Úembedr�   rá   Úpos_blkr;   ÚjÚblkÚx_feats                          r9   Úforward_intermediateszTwins.forward_intermediatesÇ  s¾  € ð( ˜VÒ#ÐKÐ%KÓKÐ#ØˆÜ"6´s¸4¿;¹;Ó7GÈÓ"QÑˆ�ið
  Ÿg™gÑˆˆ1ˆf�eÜ1:¼3Ø×!Ñ! 4§>¡>°4·;±;ÀÇÁó<Pó 2
ó 	hÑ-ˆAÑ-��t˜V Wñ ˜A“h‰GˆAˆtÙ�Q“ˆAÜ# FÓ+ò )‘��3Ù˜˜4“L�Ø˜“6Ù  4Ó(‘Að)ð
 ”3�t—{‘{Ó# aÑ'Ò'Ø�A—I‘I˜aÐ+ $Ð+¨Ò+×3Ñ3°A°q¸!¸QÓ?×JÑJÓL�Ø˜Ò$Ø!×(Ñ(¨Õ+à˜Ò$á-1˜TŸY™Y qœ\°q�FØ!×(Ñ(¨¨¯©¸Ð)E¸DÐ)EÀ"Ò)E×)MÑ)MÈaÐQRÐTUÐWXÓ)Y×)dÑ)dÓ)fÖgð%	hñ( Ø Ð à�I‰I�a‹Lˆà�-ÐÐr:   Ú
prune_normÚ
prune_headc                 ó¨   — t        t        | j                  «      |«      \  }}|rt        j                  «       | _        |r| j                  dd«       |S )z@ Prune layers not required for specified intermediates.
        r   rþ   )r   rÞ   rá   r0   r’   rƒ   r  )r5   r  r"  r#  r  r  s         r9   Úprune_intermediate_layerszTwins.prune_intermediate_layersþ  sG   € ô #7´s¸4¿;¹;Ó7GÈÓ"QÑˆ�iáÜŸ™›ˆDŒIÙØ×!Ñ! ! RÔ(ØÐr:   c                 óü  — |j                   d   }t        t        | j                  | j                  | j
                  | j                  «      «      D ]™  \  }\  }}}} ||«      \  }} ||«      }t        |«      D ]  \  }	}
 |
||«      }|	dk(  sŒ |||«      }Œ |t        | j                  «      dz
  k  sŒe |j                  |g|¢d‘­Ž j                  dddd«      j                  «       }Œ› | j                  |«      }|S )Nr   r   rD   r+   r=   )rF   r  r  rÜ   rÝ   rá   rå   rÞ   rÇ   rJ   rL   rR   rƒ   )r5   rS   rT   r§   r  r�   rá   r  r;   r  r  s              r9   Úforward_featureszTwins.forward_features  s   € Ø�G‰G�A‰JˆÜ1:Ü�D×%Ñ% t§~¡~°t·{±{ÀDÇNÁNÓSó2Uò 		MÑ-ˆAÑ-��t˜V Wá˜A“h‰GˆAˆtÙ�Q“ˆAÜ# FÓ+ò )‘��3Ù˜˜4“L�Ø˜“6Ù  4Ó(‘Að)ð ”3�t—{‘{Ó# aÑ'Ó'Ø�A—I‘I˜aÐ+ $Ð+¨Ò+×3Ñ3°A°q¸!¸QÓ?×JÑJÓL‘ð		Mð �I‰I�a‹LˆØˆr:   Ú
pre_logitsc                 ó�   — | j                   dk(  r|j                  d¬«      }| j                  |«      }|r|S | j                  |«      S )Nr¸   r   rE   )rÄ   Úmeanræ   rç   )r5   rS   r(  s      r9   Úforward_headzTwins.forward_head  sB   € Ø×Ñ˜uÒ$Ø—‘˜1�“ˆAØ�N‰N˜1ÓˆÙˆqÐ0 D§I¡I¨a£LÐ0r:   c                 óJ   — | j                  |«      }| j                  |«      }|S r˜   )r'  r+  )r5   rS   s     r9   rf   zTwins.forward#  s'   € Ø×!Ñ! !Ó$ˆØ×Ñ˜aÓ ˆØˆr:   ©F)Tr˜   )NFFr  F)r   FT)$rj   rk   rl   rm   r   r0   r‚   r‡   r   rs   r   Ústrrt   r   r   rš   r	   r.   rn   ro   Úignorer¨   r÷   rú   rü   r  ré   rµ   r   rq   r!  r%  r'  r+  rf   rv   rw   s   @r9   r   r   C  sü  ø„ ñð 58ØØØ#Ø$Ø*=Ø)5Ø,8Ø&2Ø)5Ø-1Ø!Ø#%Ø$&Ø$&Ø$&Ù*1°"·,±,ÀDÔ*IØ"ØØñ+J'à˜C  s¨C x¡Ð0Ñ1ðJ'ð ðJ'ð ð	J'ð
 ðJ'ð ðJ'ð ˜c 3˜h™ðJ'ð ˜S #˜X‘ðJ'ð ˜e S˜jÑ)ðJ'ð ˜#˜s˜(‘OðJ'ð ˜S #˜X‘ðJ'ð ˜%  S ™/Ñ*ðJ'ð ðJ'ð !ðJ'ð "ðJ'ð  "ð!J'ð" "ð#J'ð$ ˜RŸY™Y™ð%J'ð& õ'J'ðX ‡Y�Y×ÑñUó ðUð ‡Y�Y×Ñòó ðð ‡Y�Y×ÑòBó ðBð ‡Y�Y×Ñð §	¡	ò ó ðñ@¨Cð @¸hÀs¹mó @ò$ð$ 8<ØØ$Ø$Ø',ñ5 à�|‰|ð5 ð ˜e C¨¨c© NÑ3Ñ4ð5 ð ð	5 ð
 ð5 ð ð5 ð !%ð5 ð 
ˆt�E—L‘LÑ! 5¨¯©°t¸E¿L¹LÑ7IÐ)IÑ#JÐJÑ	Kó5 ðr ./Ø$Ø#ñ	à˜3  S¡	˜>Ñ*ðð ðð ó	ò ñ1¨$ó 1ör:   c                 óh   — |j                  dd«      }t        t        | |fdt        |d¬«      i|¤Ž}|S )NÚout_indicesr>   Úfeature_cfgÚgetter)r1  Úfeature_cls)Úpopr   r   rä   )ÚvariantÚ
pretrainedÚkwargsr1  Úmodels        r9   Ú_create_twinsr:  )  sC   € Ø—*‘*˜]¨AÓ.€KÜ Üˆw˜
ñä [¸hÔGðð ñ€Eð
 €Lr:   c                 ó4   — | ddd dddt         t        dddd	œ|¥S )
Nr·   )r+   r³   r³   gÍÌÌÌÌÌì?ÚbicubicTzpatch_embeds.0.projrç   z
apache-2.0)ÚurlrÃ   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚfixed_input_sizer*  r  Ú
first_convÚ
classifierÚlicenser
   )r=  r8  s     r9   Ú_cfgrF  3  s5   € àØ¨=ÀtØ¨ÈÜ%Ô.BØ+¸6Øñð ðð r:   ztimm/)Ú	hf_hub_id)ztwins_pcpvt_small.in1kztwins_pcpvt_base.in1kztwins_pcpvt_large.in1kztwins_svt_small.in1kztwins_svt_base.in1kztwins_svt_large.in1kr°   c           	      óf   — t        dg d¢g d¢g d¢g d¢g d¢¬«      }t        d	d| it        |fi |¤Ž¤ŽS )
Nr>   ©rº   r»   i@  r½   ©r   r=   r?   rh   ©rh   rh   r>   r>   r¿   rÁ   ©r­   rÅ   r   rÆ   rÇ   rÈ   r7  )Útwins_pcpvt_small©rä   r:  ©r7  r8  Ú
model_argss      r9   rM  rM  I  s<   € äØÒ!4ÂÒYeÚ¢|ô5€Jô Ñb¸ÐbÄtÈJÑGaÐZ`ÑGaÑbÐbr:   c           	      óf   — t        dg d¢g d¢g d¢g d¢g d¢¬«      }t        d	d| it        |fi |¤Ž¤ŽS )
Nr>   rI  rJ  rK  )r+   r>   é   r+   rÁ   rL  r7  )Útwins_pcpvt_baserN  rO  s      r9   rS  rS  Q  s<   € äØÒ!4ÂÒYeÚªô6€Jô Ña¸
ÐaÄdÈ:ÑF`ÐY_ÑF`ÑaÐar:   c           	      óf   — t        dg d¢g d¢g d¢g d¢g d¢¬«      }t        d	d| it        |fi |¤Ž¤ŽS )
Nr>   rI  rJ  rK  )r+   rh   é   r+   rÁ   rL  r7  )Útwins_pcpvt_largerN  rO  s      r9   rV  rV  Y  s<   € äØÒ!4ÂÒYeÚªô6€Jô Ñb¸ÐbÄtÈJÑGaÐZ`ÑGaÑbÐbr:   c           
      ól   — t        dg d¢g d¢g d¢g d¢g d¢g d¢¬«      }t        d
d	| it        |fi |¤Ž¤ŽS )Nr>   r¹   )r=   r>   rh   r´   r¾   )r=   r=   é
   r>   ©é   rZ  rZ  rZ  rÁ   ©r­   rÅ   r   rÆ   rÇ   rÉ   rÈ   r7  )Útwins_svt_smallrN  rO  s      r9   r\  r\  a  s?   € äØÒ!4ÂÒZfÚ¢,º,ôH€Jô Ñ`°zÐ`ÄTÈ*ÑE_ÐX^ÑE_Ñ`Ð`r:   c           
      ól   — t        dg d¢g d¢g d¢g d¢g d¢g d¢¬«      }t        d
d	| it        |fi |¤Ž¤ŽS )Nr>   )é`   éÀ   i€  r©   )r+   rÀ   é   é   r¾   ©r=   r=   rR  r=   rY  rÁ   r[  r7  )Útwins_svt_baserN  rO  s      r9   rc  rc  i  s?   € äØÒ!4ÂÒ[gÚ¢,º,ôH€Jô Ñ_°jÐ_ÄDÈÑD^ÐW]ÑD^Ñ_Ð_r:   c           
      ól   — t        dg d¢g d¢g d¢g d¢g d¢g d¢¬«      }t        d
d	| it        |fi |¤Ž¤ŽS )Nr>   )r»   r¼   r½   i   )r>   rh   r´   é    r¾   rb  rY  rÁ   r[  r7  )Útwins_svt_largerN  rO  s      r9   rf  rf  q  s?   € äØÒ!6Â.Ò]iÚ¢,º,ôH€Jô Ñ`°zÐ`ÄTÈ*ÑE_ÐX^ÑE_Ñ`Ð`r:   r-  )rþ   )9rm   r
  Ú	functoolsr   Útypingr   r   r   r   r   r	   rn   Útorch.nnr0   Útorch.nn.functionalr…   rH   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_features_fxr   Ú	_registryr   r   Úvision_transformerr   Ú__all__rs   ru   rš   r   ry   r‡   rœ   r«   r   r:  rF  Údefault_cfgsrM  rS  rV  r\  rc  rf  r�   r:   r9   ú<module>rt     sÊ  ðñó Ý ß :× :ã Ý ß Ð ç Aß j× jÝ *Ý +Ý 1ß <Ý )àˆ)€àˆc�3ˆh‰€ð ôE˜Ÿ™ó Eó ðEôVB˜"Ÿ)™)ô BôJ)ˆB�I‰Iô )ôX8ˆb�i‰iô 8ô<"�—‘ô "ôJcˆB�I‰Iô cóLó	ñ %Ù"¨WÔ5Ù!¨GÔ4Ù"¨WÔ5Ù ¨7Ô3Ù¨'Ô2Ù ¨7Ô3ñ&ó €ð ñc°Uò có ðcð ñb°Eò bó ðbð ñc°Uò có ðcð ña°5ò aó ðað ñ`°%ò `ó ð`ð ña°5ò aó ñar:   