Ë
    éÓ^j®  ã                   ó	  — d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlm Z m!Z!m"Z" ddl#m$Z$ dgZ%e G d„ dejL                  «      «       Z'd¦d„Z( G d„ dejL                  «      Z) G d„ dejL                  «      Z* G d„ dejL                  «      Z+ G d„ dejL                  «      Z, G d„ dejL                  «      Z- G d„ dejL                  «      Z.d„ Z/d§d„Z0d¨d„Z1 e!i d  e1d!d"¬#«      “d$ e1d!d%¬#«      “d& e1d!d'd(¬)«      “d* e1d!d+¬#«      “d, e1d!d-¬#«      “d. e1d!d/d(¬)«      “d0 e1d!d1¬#«      “d2 e1d!d3¬#«      “d4 e1d!d5d(¬)«      “d6 e1d!d7¬#«      “d8 e1d!d9¬#«      “d: e1d!d;d(¬)«      “d< e1d!d=¬#«      “d> e1d!d?¬#«      “d@ e1d!dAd(¬)«      “dB e1d!dC¬#«      “dD e1d!dE¬#«      “i dF e1d!dGd(¬)«      “dH e1d!dI¬#«      “dJ e1d!dK¬#«      “dL e1d!dMd(¬)«      “dN e1d!dO¬#«      “dP e1d!dQ¬#«      “dR e1d!dSd(¬)«      “dT e1d!dU¬#«      “dV e1d!dW¬#«      “dX e1d!dYd(¬)«      “dZ e1d!d[¬#«      “d\ e1d!d]¬#«      “d^ e1d!d_d(¬)«      “d` e1d!da¬#«      “db e1d!dc¬#«      “dd e1d!ded(¬)«      “df e1d!dg¬#«      “¥ e1d!dh¬#«       e1d!did(¬)«       e1d!dj¬#«       e1d!dk¬#«       e1d!dld(¬)«       e1d!dm¬#«       e1d!dn¬#«       e1d!dod(¬)«      dpœ¥«      Z2e d©dqe.fdr„«       Z3e d©dqe.fds„«       Z4e d©dqe.fdt„«       Z5e d©dqe.fdu„«       Z6e d©dqe.fdv„«       Z7e d©dqe.fdw„«       Z8e d©dqe.fdx„«       Z9e d©dqe.fdy„«       Z:e d©dqe.fdz„«       Z;e d©dqe.fd{„«       Z<e d©dqe.fd|„«       Z=e d©dqe.fd}„«       Z>e d©dqe.fd~„«       Z?e d©dqe.fd„«       Z@e d©dqe.fd€„«       ZAe d©dqe.fd�„«       ZBe d©dqe.fd‚„«       ZCe d©dqe.fdƒ„«       ZDe d©dqe.fd„„«       ZEe d©dqe.fd…„«       ZFe d©dqe.fd†„«       ZGe d©dqe.fd‡„«       ZHe d©dqe.fdˆ„«       ZIe d©dqe.fd‰„«       ZJe d©dqe.fdŠ„«       ZKe d©dqe.fd‹„«       ZLe d©dqe.fdŒ„«       ZMe d©dqe.fd�„«       ZN e"eOi dŽd$“d�d&“d�d,“d‘d.“d’d2“d“d4“d”d8“d•d:“d–d>“d—d@“d˜dD“d™dF“dšdJ“d›dL“dœdP“d�dR“dždV“dXd\d^dbdddŸd d¡d¢d£d¤d¥œ¥«       y)ªa]   Cross-Covariance Image Transformer (XCiT) in PyTorch

Paper:
    - https://arxiv.org/abs/2106.09681

Same as the official implementation, with some minor adaptations, original copyright below
    - https://github.com/facebookresearch/xcit/blob/master/xcit.py

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
é    N)Úpartial)ÚListÚOptionalÚTupleÚUnionÚTypeÚAny©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚDropPathÚtrunc_normal_Ú	to_2tupleÚuse_fused_attnÚMlpé   )Úbuild_model_with_cfg)Úfeature_take_indices)Úregister_notrace_module)Ú
checkpoint)Úregister_modelÚgenerate_default_cfgsÚregister_model_deprecations)Ú	ClassAttnÚXcitc                   óP   ‡ — e Zd ZdZ	 	 	 	 	 d
dededefˆ fd„Zdededefd	„Zˆ xZS )ÚPositionalEncodingFourierzä
    Positional encoding relying on a fourier kernel matching the one used in the "Attention is all you Need" paper.
    Based on the official XCiT code
        - https://github.com/facebookresearch/xcit/blob/master/xcit.py
    Ú
hidden_dimÚdimÚtemperaturec                 óÖ   •— ||dœ}t         ‰| �  «        t        j                  |dz  |fddi|¤Ž| _        dt
        j                  z  | _        || _        || _	        || _
        d| _        y )N©ÚdeviceÚdtypeé   Úkernel_sizer   ç�íµ ÷Æ°>)ÚsuperÚ__init__ÚnnÚConv2dÚtoken_projectionÚmathÚpiÚscaler    r   r   Úeps)Úselfr   r   r    r#   r$   ÚddÚ	__class__s          €ú[/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/xcit.pyr)   z"PositionalEncodingFourier.__init__)   sh   ø€ ð ¨Ñ/ˆÜ‰ÑÔÜ "§	¡	¨*°q©.¸#Ñ SÈ1Ð SÐPRÑ SˆÔØœŸ™‘[ˆŒ
Ø&ˆÔØ$ˆŒØˆŒØˆ�ó    ÚBÚHÚWc           
      óì  — | j                   j                  j                  }| j                   j                  j                  }t	        j
                  d|dz   |¬«      j                  t        j                  «      j                  d«      j                  dd|«      }t	        j
                  d|dz   |¬«      j                  t        j                  «      j                  d|d«      }||d d …dd …d d …f   | j                  z   z  | j                  z  }||d d …d d …dd …f   | j                  z   z  | j                  z  }t	        j
                  | j                  |¬«      j                  t        j                  «      }| j                  dt	        j                  |dd¬«      z  | j                  z  z  }|d d …d d …d d …d f   |z  }	|d d …d d …d d …d f   |z  }
t	        j                  |	d d …d d …d d …dd d…f   j!                  «       |	d d …d d …d d …dd d…f   j#                  «       gd¬	«      j%                  d
«      }	t	        j                  |
d d …d d …d d …dd d…f   j!                  «       |
d d …d d …d d …dd d…f   j#                  «       gd¬	«      j%                  d
«      }
t	        j&                  |
|	fd
¬	«      j)                  dd
dd«      }| j                  |j                  |«      «      }|j                  |ddd«      S )Nr   )r#   éÿÿÿÿr%   Úfloor)Úrounding_moder   é   ©r   é   )r,   Úweightr#   r$   ÚtorchÚarangeÚtoÚfloat32Ú	unsqueezeÚrepeatr0   r/   r   r    ÚdivÚstackÚsinÚcosÚflattenÚcatÚpermute)r1   r6   r7   r8   r#   r$   Úy_embedÚx_embedÚdim_tÚpos_xÚpos_yÚposs               r4   Úforwardz!PositionalEncodingFourier.forward:   s�  € Ø×&Ñ&×-Ñ-×4Ñ4ˆØ×%Ñ%×,Ñ,×2Ñ2ˆÜ—,‘,˜q ! a¡%°Ô7×:Ñ:¼5¿=¹=ÓI×SÑSÐTUÓV×]Ñ]Ð^_ÐabÐdeÓfˆÜ—,‘,˜q ! a¡%°Ô7×:Ñ:¼5¿=¹=ÓI×PÑPÐQRÐTUÐWXÓYˆØ˜W¢Q¨©ªQ YÑ/°$·(±(Ñ:Ñ;¸d¿j¹jÑHˆØ˜W¢Qª¨2©3 YÑ/°$·(±(Ñ:Ñ;¸d¿j¹jÑHˆÜ—‘˜TŸ_™_°VÔ<×?Ñ?ÄÇÁÓNˆØ× Ñ  Q¬¯©°5¸!È7Ô)SÑ%SÐVZ×VeÑVeÑ%eÑfˆØšš1ša ˜Ñ&¨Ñ.ˆØšš1ša ˜Ñ&¨Ñ.ˆÜ—‘˜U¢1¢aª¨A¨D¨q¨D =Ñ1×5Ñ5Ó7¸ºqÂ!ÂQÈÈÈ1È¸}Ñ9M×9QÑ9QÓ9SÐTÐZ[Ô\×dÑdÐefÓgˆÜ—‘˜U¢1¢aª¨A¨D¨q¨D =Ñ1×5Ñ5Ó7¸ºqÂ!ÂQÈÈÈ1È¸}Ñ9M×9QÑ9QÓ9SÐTÐZ[Ô\×dÑdÐefÓgˆÜ�i‰i˜ ˜¨AÔ.×6Ñ6°q¸!¸QÀÓBˆØ×#Ñ# C§F¡F¨5£MÓ2ˆØ�z‰z˜!˜Q  1Ó%Ð%r5   )é    é   i'  NN)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚintÚfloatr)   rT   Ú__classcell__©r3   s   @r4   r   r   !   sT   ø„ ñð !ØØ!&ØØñàðð ðð õ	ð"&˜ð & ð &¨÷ &r5   r   c           
      ó¨   — ||dœ}t         j                  j                  t        j                  | |fd|dddœ|¤Žt        j                  |fi |¤Ž«      S )z3x3 convolution + batch normr"   r?   r   F)r&   ÚstrideÚpaddingÚbias)rA   r*   Ú
Sequentialr+   ÚBatchNorm2d)Ú	in_planesÚ
out_planesr`   r#   r$   r2   s         r4   Úconv3x3rg   L   sT   € à UÑ	+€BÜ�8‰8×ÑÜ
�	‰	�)˜ZÐc°Q¸vÈqÐW\ÑcÐ`bÑcÜ
�‰�zÑ( RÑ(óð r5   c                   ó|   ‡ — e Zd ZdZddddej
                  ddfdeded	ed
edeej                     f
ˆ fd„Z	d„ Z
ˆ xZS )ÚConvPatchEmbedz<Image to Patch Embedding using multiple convolutional layerséà   é   r?   rV   NÚimg_sizeÚ
patch_sizeÚin_chansÚ	embed_dimÚ	act_layerc                 ób  •— ||dœ}t         ‰
| �  «        t        |«      }|d   |z  |d   |z  z  }	|| _        || _        |	| _        |dk(  r|t        j                  j                  t        ||dz  dfi |¤Ž |«       t        |dz  |dz  dfi |¤Ž |«       t        |dz  |dz  dfi |¤Ž |«       t        |dz  |dfi |¤Ž«      | _
        y |dk(  rct        j                  j                  t        ||dz  dfi |¤Ž |«       t        |dz  |dz  dfi |¤Ž |«       t        |dz  |dfi |¤Ž«      | _
        y d‚)	Nr"   r   r   rk   é   r%   r=   z=For convolutional projection, patch size has to be in [8, 16])r(   r)   r   rl   rm   Únum_patchesrA   r*   rc   rg   Úproj)r1   rl   rm   rn   ro   rp   r#   r$   r2   rs   r3   s             €r4   r)   zConvPatchEmbed.__init__X   sT  ø€ ð ¨Ñ/ˆÜ‰ÑÔÜ˜XÓ&ˆØ ‘{ jÑ0°X¸a±[ÀJÑ5NÑOˆØ ˆŒØ$ˆŒØ&ˆÔà˜ÒÜŸ™×+Ñ+Ü˜ )¨q¡.°!Ñ:°rÑ:Ù“Ü˜	 Q™¨	°Q©¸Ñ@¸RÑ@Ù“Ü˜	 Q™¨	°Q©¸Ñ@¸RÑ@Ù“Ü˜	 Q™¨	°1Ñ;¸Ñ;óˆD�Ið ˜1Š_ÜŸ™×+Ñ+Ü˜ )¨q¡.°!Ñ:°rÑ:Ù“Ü˜	 Q™¨	°Q©¸Ñ@¸RÑ@Ù“Ü˜	 Q™¨	°1Ñ;¸Ñ;óˆD�Ið RÐRr5   c                 ó®   — | j                  |«      }|j                  d   |j                  d   }}|j                  d«      j                  dd«      }|||ffS )Nr%   r?   r   )rt   ÚshaperK   Ú	transpose)r1   ÚxÚHpÚWps       r4   rT   zConvPatchEmbed.forward   sP   € Ø�I‰I�a‹LˆØ—‘˜‘˜QŸW™W Q™ZˆBˆØ�I‰I�a‹L×"Ñ" 1 aÓ(ˆØ�2�r�(ˆ{Ðr5   )rW   rX   rY   rZ   r*   ÚGELUr[   r   ÚModuler)   rT   r]   r^   s   @r4   ri   ri   U   sj   ø„ ÙFð  Ø ØØ Ø)+¯©ØØñ%Sàð%Sð ð%Sð ð	%Sð
 ð%Sð ˜BŸI™I‘õ%SöNr5   ri   c            	       ó„   ‡ — e Zd ZdZdej
                  dddfdedee   deej                     defˆ fd„Z
d	ed
efd„Zˆ xZS )ÚLPIa  
    Local Patch Interaction module that allows explicit communication between tokens in 3x3 windows to augment the
    implicit communication performed by the block diagonal scatter attention. Implemented using 2 layers of separable
    3x3 convolutions with GeLU and BatchNorm2d
    Nr?   Úin_featuresÚout_featuresrp   r&   c                 ó8  •— t         ‰	| �  «        ||dœ}|xs |}|dz  }t        j                  j                  ||f|||dœ|¤Ž| _         |«       | _        t        j                  |fi |¤Ž| _        t        j                  j                  ||f|||dœ|¤Ž| _	        y )Nr"   r%   )r&   ra   Úgroups)
r(   r)   rA   r*   r+   Úconv1Úactrd   ÚbnÚconv2)
r1   r   r€   rp   r&   r#   r$   r2   ra   r3   s
            €r4   r)   zLPI.__init__�   s®   ø€ ô 	‰ÑÔØ¨Ñ/ˆØ#Ò2 {ˆØ Ñ"ˆä—X‘X—_‘_Ø˜ðjØ2=ÀwÐWbñjØfhñjˆŒ
á“;ˆŒÜ—.‘. Ñ3°Ñ3ˆŒÜ—X‘X—_‘_Ø˜ðlØ3>ÈÐXdñlØhjñlˆ�
r5   r7   r8   c                 ó@  — |j                   \  }}}|j                  ddd«      j                  ||||«      }| j                  |«      }| j	                  |«      }| j                  |«      }| j                  |«      }|j                  |||«      j                  ddd«      }|S )Nr   r%   r   )rv   rM   Úreshaperƒ   r„   r…   r†   )r1   rx   r7   r8   r6   ÚNÚCs          r4   rT   zLPI.forward¢   sŽ   € Ø—'‘'‰ˆˆ1ˆaØ�I‰I�a˜˜AÓ×&Ñ& q¨!¨Q°Ó2ˆØ�J‰J�q‹MˆØ�H‰H�Q‹KˆØ�G‰G�A‹JˆØ�J‰J�q‹MˆØ�I‰I�a˜˜AÓ×&Ñ& q¨!¨QÓ/ˆØˆr5   )rW   rX   rY   rZ   r*   r{   r[   r   r   r|   r)   rT   r]   r^   s   @r4   r~   r~   †   sl   ø„ ñð +/Ø)+¯©Ø ØØñlàðlð # 3™-ðlð ˜BŸI™I‘ð	lð
 õlð*˜Cð  C÷ r5   r~   c                   óÐ   ‡ — e Zd ZdZdddddej
                  ej                  ddddfdeded	ed
e	dededede
ej                     de
ej                     dee   de	fˆ fd„Zd„ Zˆ xZS )ÚClassAttentionBlockzAClass Attention Layer as in CaiT https://arxiv.org/abs/2103.17239ç      @Fç        ç      ð?Nr   Ú	num_headsÚ	mlp_ratioÚqkv_biasÚ	proj_dropÚ	attn_dropÚ	drop_pathrp   Ú
norm_layerÚetaÚtokens_normc                 óz  •— ||dœ}t         ‰| �  «         |	|fi |¤Ž| _        t        |f||||dœ|¤Ž| _        |dkD  rt        |«      nt        j                  «       | _         |	|fi |¤Ž| _	        t        d|t        ||z  «      ||dœ|¤Ž| _        |dkD  rt        |«      nt        j                  «       | _        |
�jt        j                  |
t        j                   |fi |¤Žz  «      | _        t        j                  |
t        j                   |fi |¤Žz  «      | _        || _        y d\  | _        | _        || _        y )Nr"   ©r�   r’   r”   r“   rŽ   ©r   Úhidden_featuresrp   Údrop)r�   r�   © )r(   r)   Únorm1r   Úattnr   r*   ÚIdentityÚ
drop_path1Únorm2r   r[   ÚmlpÚ
drop_path2Ú	ParameterrA   ÚonesÚgamma1Úgamma2r˜   )r1   r   r�   r‘   r’   r“   r”   r•   rp   r–   r—   r˜   r#   r$   r2   r3   s                  €r4   r)   zClassAttentionBlock.__init__°   s=  ø€ ð  ¨Ñ/ˆÜ‰ÑÔÙ Ñ* rÑ*ˆŒ
ÜØð
àØØØñ
ð ñ
ˆŒ	ð 2;¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒá Ñ* rÑ*ˆŒ
Üð 
ØÜ  i¡Ó0ØØñ	
ð
 ñ
ˆŒð 2;¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒàˆ?ÜŸ,™, s¬U¯Z©Z¸Ñ-B¸rÑ-BÑ'BÓCˆDŒKÜŸ,™, s¬U¯Z©Z¸Ñ-B¸rÑ-BÑ'BÓCˆDŒKð
 'ˆÕð (0Ñ$ˆDŒK˜œð 'ˆÕr5   c                 ó>  — | j                  |«      }t        j                  | j                  |«      |d d …dd …f   gd¬«      }|| j	                  | j
                  |z  «      z   }| j                  r| j                  |«      }n:t        j                  | j                  |d d …dd…f   «      |d d …dd …f   gd¬«      }|}|d d …dd…f   }| j                  | j                  |«      z  }t        j                  ||d d …dd …f   gd¬«      }|| j                  |«      z   }|S )Nr   r>   r   )rŸ   rA   rL   r    r¢   r¨   r˜   r£   r©   r¤   r¥   )r1   rx   Úx_norm1Úx_attnÚx_resÚ	cls_tokens         r4   rT   zClassAttentionBlock.forwardà   s  € Ø—*‘*˜Q“-ˆÜ—‘˜DŸI™I gÓ.°º¸1¹2¸±Ð?ÀQÔGˆØ�—‘ §¡¨fÑ 4Ó5Ñ5ˆà×ÒØ—
‘
˜1“‰Aä—	‘	˜4Ÿ:™: aª¨1¨Q¨3¨¡iÓ0°!²A°q±r°E±(Ð;ÀÔCˆAØˆØ’a˜˜1˜�f‘Iˆ	Ø—K‘K $§(¡(¨9Ó"5Ñ5ˆ	Ü�I‰I�y !¢A q¡r E¡(Ð+°Ô3ˆØ�D—O‘O AÓ&Ñ&ˆØˆr5   )rW   rX   rY   rZ   r*   r{   Ú	LayerNormr[   r\   Úboolr   r|   r   r)   rT   r]   r^   s   @r4   rŒ   rŒ   ­   s¼   ø„ ÙKð  "Ø"Ø!Ø!Ø!Ø)+¯©Ø*,¯,©,Ø#%Ø %ØØñ.'àð.'ð ð.'ð ð	.'ð
 ð.'ð ð.'ð ð.'ð ð.'ð ˜BŸI™I‘ð.'ð ˜RŸY™Y™ð.'ð ˜%‘ð.'ð õ.'ö`r5   rŒ   c                   ó¼   ‡ — e Zd ZU ej                  j
                  e   ed<   	 	 	 	 	 	 	 d
dededede	de	f
ˆ fd„Z
d„ Zej                  j                  d	„ «       Zˆ xZS )ÚXCAÚ
fused_attnr   r�   r’   r”   r“   c                 óž  •— ||dœ}t         ‰	| �  «        || _        t        d¬«      | _        t        j                  t        j                  |ddfi |¤Ž«      | _	        t        j                  ||dz  fd|i|¤Ž| _        t        j                  |«      | _        t        j                  ||fi |¤Ž| _        t        j                  |«      | _        y )Nr"   T)Úexperimentalr   r?   rb   )r(   r)   r�   r   r³   r*   r¦   rA   r§   r    ÚLinearÚqkvÚDropoutr”   rt   r“   )
r1   r   r�   r’   r”   r“   r#   r$   r2   r3   s
            €r4   r)   zXCA.__init__ø   s§   ø€ ð ¨Ñ/ˆÜ‰ÑÔØ"ˆŒÜ(°dÔ;ˆŒÜŸ<™<¬¯
©
°9¸aÀÑ(IÀbÑ(IÓJˆÔÜ—9‘9˜S #¨¡'Ñ?°Ð?¸BÑ?ˆŒÜŸ™ IÓ.ˆŒÜ—I‘I˜c 3Ñ-¨"Ñ-ˆŒ	ÜŸ™ IÓ.ˆ�r5   c                 óô  — |j                   \  }}}| j                  |«      j                  ||d| j                  || j                  z  «      j	                  ddddd«      }|j                  d«      \  }}}| j                  r‘t        j                  j                  j                  |d¬«      | j                  z  }t        j                  j                  j                  |d¬«      }t        j                  j                  j                  |||d¬	«      }n t        j                  j                  j                  |d¬«      }t        j                  j                  j                  |d¬«      }||j                  d
d«      z  | j                  z  }	|	j                  d¬«      }	| j                  |	«      }	|	|z  }|j	                  dddd«      j                  |||«      }| j!                  |«      }| j#                  |«      }|S )Nr?   r%   r   r=   r   r:   r>   r�   )r/   éþÿÿÿ)rv   r·   rˆ   r�   rM   Úunbindr³   rA   r*   Ú
functionalÚ	normalizer    Úscaled_dot_product_attentionrw   Úsoftmaxr”   rt   r“   )
r1   rx   r6   r‰   rŠ   r·   ÚqÚkÚvr    s
             r4   rT   zXCA.forward  s§  € Ø—'‘'‰ˆˆ1ˆaà�h‰h�q‹k×!Ñ! ! Q¨¨4¯>©>¸1ÀÇÁÑ;NÓO×WÑWÐXYÐ[\Ð^_ÐabÐdeÓfˆØ—*‘*˜Q“-‰ˆˆ1ˆaà�?Š?Ü—‘×#Ñ#×-Ñ-¨a°RÐ-Ó8¸4×;KÑ;KÑKˆAÜ—‘×#Ñ#×-Ñ-¨a°RÐ-Ó8ˆAÜ—‘×#Ñ#×@Ñ@ÀÀAÀqÐPSÐ@ÓT‰Aô —‘×#Ñ#×-Ñ-¨a°RÐ-Ó8ˆAÜ—‘×#Ñ#×-Ñ-¨a°RÐ-Ó8ˆAØ˜Ÿ™ B¨Ó+Ñ+¨t×/?Ñ/?Ñ?ˆDØ—<‘< B�<Ó'ˆDØ—>‘> $Ó'ˆDØ�q‘ˆAà�I‰I�a˜˜A˜qÓ!×)Ñ)¨!¨Q°Ó2ˆØ�I‰I�a‹LˆØ�N‰N˜1ÓˆØˆr5   c                 ó   — dhS )Nr    rž   ©r1   s    r4   Úno_weight_decayzXCA.no_weight_decay$  s
   € àˆÐr5   )rr   FrŽ   rŽ   NN)rW   rX   rY   rA   ÚjitÚFinalr°   Ú__annotations__r[   r\   r)   rT   ÚignorerÅ   r]   r^   s   @r4   r²   r²   ñ   s‡   ø… Ø—	‘	—‘ Ñ%Ó%ðð Ø"Ø!Ø!ØØñ/àð/ð ð/ð ð	/ð
 ð/ð õ/ò(ð0 ‡Y�Y×Ññó ôr5   r²   c                   óÊ   ‡ — e Zd Zdddddej                  ej
                  dddf
dededed	ed
ededede	ej                     de	ej                     defˆ fd„Zdedefd„Zˆ xZS )ÚXCABlockr�   FrŽ   r�   Nr   r�   r‘   r’   r“   r”   r•   rp   r–   r—   c                 ó0  •— ||dœ}t         ‰| �  «         |	|fi |¤Ž| _        t        |f||||dœ|¤Ž| _        |dkD  rt        |«      nt        j                  «       | _         |	|fi |¤Ž| _	        t        d||dœ|¤Ž| _        |dkD  rt        |«      nt        j                  «       | _         |	|fi |¤Ž| _        t        d|t        ||z  «      ||dœ|¤Ž| _        |dkD  rt        |«      nt        j                  «       | _        t        j$                  |
t'        j(                  |fi |¤Žz  «      | _        t        j$                  |
t'        j(                  |fi |¤Žz  «      | _        t        j$                  |
t'        j(                  |fi |¤Žz  «      | _        y )Nr"   rš   rŽ   )r   rp   r›   rž   )r(   r)   rŸ   r²   r    r   r*   r¡   r¢   Únorm3r~   Úlocal_mpÚ
drop_path3r£   r   r[   r¤   r¥   r¦   rA   r§   r¨   Úgamma3r©   )r1   r   r�   r‘   r’   r“   r”   r•   rp   r–   r—   r#   r$   r2   r3   s                 €r4   r)   zXCABlock.__init__*  sx  ø€ ð ¨Ñ/ˆÜ‰ÑÔÙ Ñ* rÑ*ˆŒ
ÜØð
àØØØñ
ð ñ
ˆŒ	ð 2;¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒá Ñ* rÑ*ˆŒ
ÜÐG¨°yÑGÀBÑGˆŒØ1:¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒá Ñ* rÑ*ˆŒ
Üð 
ØÜ  i¡Ó0ØØñ	
ð
 ñ
ˆŒð 2;¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒä—l‘l 3¬¯©°CÑ)>¸2Ñ)>Ñ#>Ó?ˆŒÜ—l‘l 3¬¯©°CÑ)>¸2Ñ)>Ñ#>Ó?ˆŒÜ—l‘l 3¬¯©°CÑ)>¸2Ñ)>Ñ#>Ó?ˆ�r5   r7   r8   c           	      ó„  — || j                  | j                  | j                  | j                  |«      «      z  «      z   }|| j	                  | j
                  | j                  | j                  |«      ||«      z  «      z   }|| j                  | j                  | j                  | j                  |«      «      z  «      z   }|S ©N)r¢   r¨   r    rŸ   rÏ   rÐ   rÎ   rÍ   r¥   r©   r¤   r£   )r1   rx   r7   r8   s       r4   rT   zXCABlock.forwardX  s”   € Ø�—‘ §¡¨d¯i©i¸¿
¹
À1»Ó.FÑ FÓGÑGˆð �—‘ §¡¨d¯m©m¸D¿J¹JÀq»MÈ1ÈaÓ.PÑ PÓQÑQˆØ�—‘ §¡¨d¯h©h°t·z±zÀ!³}Ó.EÑ EÓFÑFˆØˆr5   )rW   rX   rY   r*   r{   r¯   r[   r\   r°   r   r|   r)   rT   r]   r^   s   @r4   rË   rË   )  sÁ   ø„ ð
  "Ø"Ø!Ø!Ø!Ø)+¯©Ø*,¯,©,ØØØñ,@àð,@ð ð,@ð ð	,@ð
 ð,@ð ð,@ð ð,@ð ð,@ð ˜BŸI™I‘ð,@ð ˜RŸY™Y™ð,@ð õ,@ð\˜Cð  C÷ r5   rË   c            +       ó  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d-deeeeef   f   dedededededed	ed
ede	dededededede
eej                        de
eej                        dede	dede	f*ˆ fd„Zd„ Zej"                  j$                  d„ «       Zej"                  j$                  d.d„«       Zej"                  j$                  d/d„«       Zej"                  j$                  dej                  fd„«       Zd0dede
e   fd„Z	 	 	 	 	 d1dej0                  d e
eeee   f      d!e	d"e	d#ed$e	deeej0                     eej0                  eej0                     f   f   fd%„Z	 	 	 d2d eeee   f   d&e	d'e	fd(„Zd)„ Zd.d*e	fd+„Zd,„ Zˆ xZS )3r   z 
    Based on timm and DeiT code bases
    https://github.com/rwightman/pytorch-image-models/tree/master/timm
    https://github.com/facebookresearch/deit/
    rl   rm   rn   Únum_classesÚglobal_poolro   Údepthr�   r‘   r’   Ú	drop_rateÚpos_drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_raterp   r–   Úcls_attn_layersÚuse_pos_embedr—   r˜   c                 ó  •— t         ‰| �  «        ||dœ}|dv sJ ‚t        |«      }|d   |z  dk(  r|d   |z  dk(  sJ d«       ‚|xs t        t        j
                  d¬«      }|xs t        j                  }|| _        || _        |x| _	        x| _
        | _        || _        d| _        t        d|||||dœ|¤Ž| _        |}t	        j                   t#        j$                  d	d	|fi |¤Ž«      | _        |rt)        dd
|i|¤Ž| _        nd| _        t	        j,                  |¬«      | _        t	        j0                  t3        |«      D �cg c]  }t5        d|||	|
||||||dœ
|¤Ž‘Œ c}«      | _        t3        |«      D �cg c]  }t9        ||d|› �¬«      ‘Œ c}| _        t	        j0                  t3        |«      D �cg c]  }t=        d|||	|
||||||dœ
|¤Ž‘Œ c}«      | _         ||fi |¤Ž| _         t	        j,                  |«      | _!        |dkD  r!t	        jD                  | j                  |fi |¤Žnt	        jF                  «       | _$        tK        | j&                  d¬«       | jM                  | jN                  «       yc c}w c c}w c c}w )a¦  
        Args:
            img_size (int, tuple): input image size
            patch_size (int): patch size
            in_chans (int): number of input channels
            num_classes (int): number of classes for classification head
            embed_dim (int): embedding dimension
            depth (int): depth of transformer
            num_heads (int): number of attention heads
            mlp_ratio (int): ratio of mlp hidden dim to embedding dim
            qkv_bias (bool): enable bias for qkv if True
            drop_rate (float): dropout rate after positional embedding, and in XCA/CA projection + MLP
            pos_drop_rate: position embedding dropout rate
            proj_drop_rate (float): projection dropout rate
            attn_drop_rate (float): attention dropout rate
            drop_path_rate (float): stochastic depth rate (constant across all layers)
            norm_layer: (nn.Module): normalization layer
            cls_attn_layers: (int) Depth of Class attention layers
            use_pos_embed: (bool) whether to use positional encoding
            eta: (float) layerscale initialization value
            tokens_norm: (bool) Whether to normalize all tokens or just the cls_token in the CA

        Notes:
            - Although `layer_norm` is user specifiable, there are hard-coded `BatchNorm2d`s in the local patch
              interaction (class LPI) and the patch embedding (class ConvPatchEmbed)
        r"   ©Ú ÚavgÚtokenr   z2`patch_size` should divide image dimensions evenlyr'   )r0   F)rl   rm   rn   ro   rp   r   r   N)Úp)
r   r�   r‘   r’   r“   r”   r•   rp   r–   r—   zblocks.)Únum_chsÚ	reductionÚmodule)
r   r�   r‘   r’   r“   r”   rp   r–   r—   r˜   ç{®Gáz”?©Ústdrž   )(r(   r)   r   r   r*   r¯   r{   rÔ   rn   Únum_featuresÚhead_hidden_sizero   rÕ   Úgrad_checkpointingri   Úpatch_embedr¦   rA   Úzerosr®   r   Ú	pos_embedr¸   Úpos_dropÚ
ModuleListÚrangerË   ÚblocksÚdictÚfeature_inforŒ   Úcls_attn_blocksÚnormÚ	head_dropr¶   r¡   Úheadr   ÚapplyÚ_init_weights)r1   rl   rm   rn   rÔ   rÕ   ro   rÖ   r�   r‘   r’   r×   rØ   rÙ   rÚ   rÛ   rp   r–   rÜ   rÝ   r—   r˜   r#   r$   r2   ÚrÚ_Úir3   s                               €r4   r)   zXcit.__init__h  sº  ø€ ôh 	‰ÑÔØ¨Ñ/ˆØÐ2Ñ2Ð2Ð2Ü˜XÓ&ˆØ˜‘˜jÑ(¨AÒ-°H¸Q±KÀ*Ñ4LÐPQÒ4Qð 	AØ@ó	AÐRàÒB¤7¬2¯<©<¸TÔ#Bˆ
ØÒ(¤§¡ˆ	à&ˆÔØ ˆŒØENÐNˆÔÐN˜DÔ1°D´NØ&ˆÔØ"'ˆÔä)ð 
ØØ!ØØØñ
ð ñ
ˆÔð ˆäŸ™¤e§k¡k°!°Q¸	Ñ&HÀRÑ&HÓIˆŒÙÜ6ÑK¸9ÐKÈÑKˆD�Nà!ˆDŒNÜŸ
™
 ]Ô3ˆŒä—m‘mô ˜5“\ö%#ð ô ð ØØ#Ø#Ø!Ø(Ø(Ø(Ø#Ø%Øñð óò%#ó $ˆŒô bgÐglÓamÖnÐ\]œT¨)¸qÈ7ÐSTÐRUÈÖWÒnˆÔä!Ÿ}™}ô ˜?Ó+ö.-ð ô  ð ØØ#Ø#Ø!Ø#Ø(Ø#Ø%ØØ'ñð óò.-ó  .ˆÔñ" ˜yÑ/¨BÑ/ˆŒ	ÜŸ™ IÓ.ˆŒØGRÐUVÂ”B—I‘I˜d×/Ñ/°ÑCÀÒCÔ\^×\gÑ\gÓ\iˆŒ	ô 	�d—n‘n¨#Õ.Ø�
‰
�4×%Ñ%Õ&ùòQ%#ùò oùò.-s   ÅI7Å8I<Æ4Jc                 ó  — t        |t        j                  «      rjt        |j                  d¬«       t        |t        j                  «      r8|j
                  �+t        j                  j                  |j
                  d«       y y y y )Nrç   rè   r   )Ú
isinstancer*   r¶   r   r@   rb   ÚinitÚ	constant_)r1   Úms     r4   rû   zXcit._init_weightsæ  sZ   € Ü�aœŸ™Ô#Ü˜!Ÿ(™(¨Õ,Ü˜!œRŸY™YÔ'¨A¯F©FÐ,>Ü—‘×!Ñ! !§&¡&¨!Õ,ð -?Ð'ð $r5   c                 ó
   — ddhS )Nrï   r®   rž   rÄ   s    r4   rÅ   zXcit.no_weight_decayì  s   € à˜[Ð)Ð)r5   c                 ó"   — t        ddddg¬«      S )Nz ^cls_token|pos_embed|patch_embedz^blocks\.(\d+))z^cls_attn_blocks\.(\d+)N)z^norm)iŸ† )Ústemró   rö   )rô   )r1   Úcoarses     r4   Úgroup_matcherzXcit.group_matcherð  s   € äØ4Ø$Ø?ÐAUÐVô
ð 	
r5   c                 ó   — || _         y rÒ   )rì   )r1   Úenables     r4   Úset_grad_checkpointingzXcit.set_grad_checkpointingø  s
   € à"(ˆÕr5   Úreturnc                 ó   — | j                   S rÒ   )rù   rÄ   s    r4   Úget_classifierzXcit.get_classifierü  s   € à�y‰yÐr5   c                 óž  — || _         |�|dv sJ ‚|| _        t        | j                  d«      r | j                  j                  j
                  nd }t        | j                  d«      r | j                  j                  j                  nd }|dkD  r)t        j                  | j                  |||¬«      | _        y t        j                  «       | _        y )Nrß   r@   r   r"   )rÔ   rÕ   Úhasattrrù   r@   r#   r$   r*   r¶   rê   r¡   )r1   rÔ   rÕ   r#   r$   s        r4   Úreset_classifierzXcit.reset_classifier   s¤   € Ø&ˆÔØÐ"ØÐ"6Ñ6Ð6Ð6Ø*ˆDÔÜ,3°D·I±I¸xÔ,H�—‘×!Ñ!×(Ò(ÈdˆÜ*1°$·)±)¸XÔ*F�—	‘	× Ñ ×&Ò&ÈDˆØ]hÐklÒ]l”B—I‘I˜d×/Ñ/°ÀVÐSXÔYˆ�	Ôrt×r}Ñr}Órˆ�	r5   rx   Úindicesr÷   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc           	      óÈ  — |dv sJ d«       ‚|dk(  }g }t        t        | j                  «      |«      \  }	}
|j                  \  }}}}| j	                  |«      \  }\  }}| j
                  �G| j                  |||«      j                  |d|j                  d   «      j                  ddd«      }||z   }| j                  |«      }t        j                  j                  «       s|s| j                  }n| j                  d|
dz    }t        |«      D ]q  \  }}| j                  r-t        j                  j                  «       st        ||||«      }n
 ||||«      }||	v sŒN|j                  |r| j!                  |«      n|«       Œs |rA|D �cg c]6  }|j                  |||d«      j                  dd	dd«      j#                  «       ‘Œ8 }}|r|S t        j$                  | j&                  j)                  |dd«      |fd¬
«      }| j*                  D ]A  }| j                  r+t        j                  j                  «       st        ||«      }Œ: ||«      }ŒC | j!                  |«      }||fS c c}w )aè   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )ÚNCHWÚNLCz)Output format must be one of NCHW or NLC.r  Nr:   r   r   r%   r?   r>   )r   Úlenró   rv   rí   rï   rˆ   rM   rð   rA   rÆ   Úis_scriptingÚ	enumeraterì   r   Úappendr÷   Ú
contiguousrL   r®   Úexpandrö   )r1   rx   r  r÷   r  r  r  rˆ   ÚintermediatesÚtake_indicesÚ	max_indexr6   rý   ÚheightÚwidthry   rz   Úpos_encodingró   rþ   ÚblkÚys                         r4   Úforward_intermediateszXcit.forward_intermediates	  sA  € ð* ˜_Ñ,ÐYÐ.YÓYÐ,Ø Ñ&ˆØˆÜ"6´s¸4¿;¹;Ó7GÈÓ"QÑˆ�ið  Ÿg™gÑˆˆ1ˆf�eØ×&Ñ& qÓ)‰ˆ‰8ˆB�Ø�>‰>Ð%àŸ>™>¨!¨R°Ó4×<Ñ<¸QÀÀAÇGÁGÈAÁJÓO×WÑWÐXYÐ[\Ð^_Ó`ˆLØ�LÑ ˆAØ�M‰M˜!Óˆä�9‰9×!Ñ!Ô#©:Ø—[‘[‰Fà—[‘[  )¨a¡-Ð0ˆFÜ Ó'ò 	B‰FˆAˆsØ×&Ò&¬u¯y©y×/EÑ/EÔ/GÜ˜s A r¨2Ó.‘á˜˜2˜r“N�Ø�LÒ à×$Ñ$±T T§Y¡Y¨q¤\¸qÕAð	Bñ à`mÖnÐ[\˜QŸY™Y q¨"¨b°"Ó5×=Ñ=¸aÀÀAÀqÓI×TÑTÕVÐnˆMÐnáØ Ð ô �I‰I�t—~‘~×,Ñ,¨Q°°BÓ7¸Ð;ÀÔCˆØ×'Ñ'ò 	ˆCØ×&Ò&¬u¯y©y×/EÑ/EÔ/GÜ˜s AÓ&‘á˜“F‘ð		ð �I‰I�a‹Lˆà�-ÐÐùò os   Æ;IÚ
prune_normÚ
prune_headc                 ó  — t        t        | j                  «      |«      \  }}| j                  d|dz    | _        |rt        j                  «       | _        |r+t        j                  «       | _        | j                  dd«       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   rà   )	r   r  ró   r*   r¡   r÷   rñ   rö   r  )r1   r  r(  r)  r   r!  s         r4   Úprune_intermediate_layerszXcit.prune_intermediate_layersM  sk   € ô #7´s¸4¿;¹;Ó7GÈÓ"QÑˆ�iØ—k‘k . 9¨q¡=Ð1ˆŒÙÜŸ™›ˆDŒIÙÜ#%§=¡=£?ˆDÔ Ø×!Ñ! ! RÔ(ØÐr5   c                 óì  — |j                   d   }| j                  |«      \  }\  }}| j                  �G| j                  |||«      j                  |d|j                   d   «      j	                  ddd«      }||z   }| j                  |«      }| j                  D ]E  }| j                  r-t        j                  j                  «       st        ||||«      }Œ< ||||«      }ŒG t        j                  | j                  j                  |dd«      |fd¬«      }| j                  D ]A  }| j                  r+t        j                  j                  «       st        ||«      }Œ: ||«      }ŒC | j!                  |«      }|S )Nr   r:   r   r%   r>   )rv   rí   rï   rˆ   rM   rð   ró   rì   rA   rÆ   r  r   rL   r®   r  rö   r÷   )r1   rx   r6   ry   rz   r$  r%  s          r4   Úforward_featureszXcit.forward_features^  sL  € Ø�G‰G�A‰Jˆà×&Ñ& qÓ)‰ˆ‰8ˆB�à�>‰>Ð%àŸ>™>¨!¨R°Ó4×<Ñ<¸QÀÀAÇGÁGÈAÁJÓO×WÑWÐXYÐ[\Ð^_Ó`ˆLØ�LÑ ˆAØ�M‰M˜!Óˆà—;‘;ò 	#ˆCØ×&Ò&¬u¯y©y×/EÑ/EÔ/GÜ˜s A r¨2Ó.‘á˜˜2˜r“N‘ð		#ô �I‰I�t—~‘~×,Ñ,¨Q°°BÓ7¸Ð;ÀÔCˆà×'Ñ'ò 	ˆCØ×&Ò&¬u¯y©y×/EÑ/EÔ/GÜ˜s AÓ&‘á˜“F‘ð		ð �I‰I�a‹LˆØˆr5   Ú
pre_logitsc                 óÌ   — | j                   r3| j                   dk(  r|d d …dd …f   j                  d¬«      n|d d …df   }| j                  |«      }|r|S | j                  |«      S )Nrá   r   r>   r   )rÕ   Úmeanrø   rù   )r1   rx   r.  s      r4   Úforward_headzXcit.forward_headz  s_   € Ø×ÒØ(,×(8Ñ(8¸EÒ(A�’!�Q‘R�%‘—‘ !�Ô$ÀqÊÈAÈÁwˆAØ�N‰N˜1ÓˆÙˆqÐ0 D§I¡I¨a£LÐ0r5   c                 óJ   — | j                  |«      }| j                  |«      }|S rÒ   )r-  r1  )r1   rx   s     r4   rT   zXcit.forward€  s'   € Ø×!Ñ! !Ó$ˆØ×Ñ˜aÓ ˆØˆr5   )rj   rk   r?   éè  râ   rV   é   r4  r�   TrŽ   rŽ   rŽ   rŽ   rŽ   NNr%   Tr�   FNN©F)TrÒ   )NFFr  F)r   FT) rW   rX   rY   rZ   r   r[   r   Ústrr\   r°   r   r   r*   r|   r)   rû   rA   rÆ   rÉ   rÅ   r  r  r  r  ÚTensorr   r'  r+  r-  r1  rT   r]   r^   s   @r4   r   r   a  sû  ø„ ñð 58Ø ØØ#Ø&Ø ØØØ!Ø!Ø!Ø#%Ø$&Ø$&Ø$&Ø37Ø48Ø#$Ø"&ØØ %ØØñ1|'à˜C  s¨C x¡Ð0Ñ1ð|'ð ð|'ð ð	|'ð
 ð|'ð ð|'ð ð|'ð ð|'ð ð|'ð ð|'ð ð|'ð ð|'ð !ð|'ð "ð|'ð "ð|'ð  "ð!|'ð"    R§Y¡Y¡Ñ0ð#|'ð$ !  b§i¡i¡Ñ1ð%|'ð& !ð'|'ð(  ð)|'ð* ð+|'ð, õ-|'ò|-ð ‡Y�Y×Ññ*ó ð*ð ‡Y�Y×Ñò
ó ð
ð ‡Y�Y×Ñò)ó ð)ð ‡Y�Y×Ñð §	¡	ò ó ðñ@¨Cð @¸hÀs¹mó @ð 8<ØØ$Ø$Ø',ñB à�|‰|ðB ð ˜e C¨¨c© NÑ3Ñ4ðB ð ð	B ð
 ðB ð ðB ð !%ðB ð 
ˆt�E—L‘LÑ! 5¨¯©°t¸E¿L¹LÑ7IÐ)IÑ#JÐJÑ	KóB ðL ./Ø$Ø#ñ	à˜3  S¡	˜>Ñ*ðð ðð ó	ò"ñ81¨$ó 1ör5   c           	      ó¢  — d| v r| d   } t        |dd «      d u}| D �cg c]  }|j                  d«      sŒ|‘Œ }}|D ],  }|r%| j                  |«      | |j                  dd«      <   Œ*| |= Œ. d| v rÛd|j	                  «       v rÉt        |j                  «      }t        |«      D ]¦  }| j                  d|› d�«      }|j                  d	d
|j                  d
   «      }t        d«      D ]  \  }}	||   | d|› d|	› d�<   Œ | j                  d|› d�d «      }
|
€Œs|
j                  d	d
«      }
t        d«      D ]  \  }}	|
|   | d|› d|	› d�<   Œ Œ¨ | S c c}w )NÚmodelrï   zpos_embeder.z
pos_embed.z!cls_attn_blocks.0.attn.qkv.weightzcls_attn_blocks.0.attn.q.weightzcls_attn_blocks.z.attn.qkv.weightr?   r:   r·   z.attn.z.weightz.attn.qkv.biasz.bias)ÚgetattrÚ
startswithÚpopÚreplaceÚ
state_dictr  rö   rò   rˆ   rv   r  )r>  r9  rÝ   rÁ   Úpos_embed_keysÚnum_ca_blocksrþ   Ú
qkv_weightÚjÚ	subscriptr’   s              r4   Úcheckpoint_filter_fnrD  †  s«  € Ø�*ÑØ Ñ(ˆ
ô ˜E ;°Ó5¸TÐA€MØ!+ÖI˜A¨q¯|©|¸KÕ/H’aÐI€NÐIØò ˆÙØBLÇ.Á.ÐQRÓBSˆJ�q—y‘y °Ó>Ò?à˜1‘ð	ð +¨jÑ8Ð=^Ðbg×brÑbrÓbtÑ=tÜ˜E×1Ñ1Ó2ˆÜ�}Ó%ò 		[ˆAØ#Ÿ™Ð*:¸1¸#Ð=MÐ(NÓOˆJØ#×+Ñ+¨A¨r°:×3CÑ3CÀBÑ3GÓHˆJÜ )¨%Ó 0ò [‘��9ØMWÐXYÉ]�
Ð-¨a¨S°°y°kÀÐIÒJð[à!—~‘~Ð(8¸¸¸>Ð&JÈDÓQˆHØÑ#Ø#×+Ñ+¨A¨rÓ2�Ü$-¨eÓ$4ò [‘L�A�yØOWÐXYÉ{�JÐ!1°!°°F¸9¸+ÀUÐKÒLñ[ð		[ð Ðùò) Js
   �E´Ec                 ór   — |j                  dd«      }t        t        | |ft        t	        |d¬«      dœ|¤Ž}|S )NÚout_indicesr?   Úgetter)rF  Úfeature_cls)Úpretrained_filter_fnÚfeature_cfg)r<  r   r   rD  rô   )ÚvariantÚ
pretrainedÚdefault_cfgÚkwargsrF  r9  s         r4   Ú_create_xcitrO  £  sJ   € Ø—*‘*˜]¨AÓ.€KÜ ÜØØðô 2Ü [¸hÔGñð ñ€Eð €Lr5   c                 ó4   — | ddd dddt         t        dddd	œ|¥S )
Nr3  )r?   rj   rj   r�   ÚbicubicTzpatch_embed.proj.0.0rù   z
apache-2.0)ÚurlrÔ   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚfixed_input_sizer0  ré   Ú
first_convÚ
classifierÚlicenser
   )rR  rN  s     r4   Ú_cfgr[  °  s5   € àØ¨=ÀtØ¨)ÈÜ%Ô.BØ,¸FØñð $*ðð r5   zxcit_nano_12_p16_224.fb_in1kztimm/z<https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p16_224.pth)Ú	hf_hub_idrR  z!xcit_nano_12_p16_224.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p16_224_dist.pthz!xcit_nano_12_p16_384.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p16_384_dist.pth)r?   é€  r]  )r\  rR  rS  zxcit_tiny_12_p16_224.fb_in1kz<https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p16_224.pthz!xcit_tiny_12_p16_224.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p16_224_dist.pthz!xcit_tiny_12_p16_384.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p16_384_dist.pthzxcit_tiny_24_p16_224.fb_in1kz<https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p16_224.pthz!xcit_tiny_24_p16_224.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p16_224_dist.pthz!xcit_tiny_24_p16_384.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p16_384_dist.pthzxcit_small_12_p16_224.fb_in1kz=https://dl.fbaipublicfiles.com/xcit/xcit_small_12_p16_224.pthz"xcit_small_12_p16_224.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p16_224_dist.pthz"xcit_small_12_p16_384.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p16_384_dist.pthzxcit_small_24_p16_224.fb_in1kz=https://dl.fbaipublicfiles.com/xcit/xcit_small_24_p16_224.pthz"xcit_small_24_p16_224.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p16_224_dist.pthz"xcit_small_24_p16_384.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p16_384_dist.pthzxcit_medium_24_p16_224.fb_in1kz>https://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p16_224.pthz#xcit_medium_24_p16_224.fb_dist_in1kzChttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p16_224_dist.pthz#xcit_medium_24_p16_384.fb_dist_in1kzChttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p16_384_dist.pthzxcit_large_24_p16_224.fb_in1kz=https://dl.fbaipublicfiles.com/xcit/xcit_large_24_p16_224.pthz"xcit_large_24_p16_224.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p16_224_dist.pthz"xcit_large_24_p16_384.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p16_384_dist.pthzxcit_nano_12_p8_224.fb_in1kz;https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p8_224.pthz xcit_nano_12_p8_224.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p8_224_dist.pthz xcit_nano_12_p8_384.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p8_384_dist.pthzxcit_tiny_12_p8_224.fb_in1kz;https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p8_224.pthz xcit_tiny_12_p8_224.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p8_224_dist.pthz xcit_tiny_12_p8_384.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p8_384_dist.pthzxcit_tiny_24_p8_224.fb_in1kz;https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p8_224.pthz xcit_tiny_24_p8_224.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p8_224_dist.pthz xcit_tiny_24_p8_384.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p8_384_dist.pthzxcit_small_12_p8_224.fb_in1kz<https://dl.fbaipublicfiles.com/xcit/xcit_small_12_p8_224.pthz!xcit_small_12_p8_224.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p8_224_dist.pthz!xcit_small_12_p8_384.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p8_384_dist.pthzxcit_small_24_p8_224.fb_in1kz<https://dl.fbaipublicfiles.com/xcit/xcit_small_24_p8_224.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p8_224_dist.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p8_384_dist.pthz=https://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p8_224.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p8_224_dist.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p8_384_dist.pthz<https://dl.fbaipublicfiles.com/xcit/xcit_large_24_p8_224.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p8_224_dist.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p8_384_dist.pth)ú!xcit_small_24_p8_224.fb_dist_in1kú!xcit_small_24_p8_384.fb_dist_in1kzxcit_medium_24_p8_224.fb_in1kú"xcit_medium_24_p8_224.fb_dist_in1kú"xcit_medium_24_p8_384.fb_dist_in1kzxcit_large_24_p8_224.fb_in1kú!xcit_large_24_p8_224.fb_dist_in1kú!xcit_large_24_p8_384.fb_dist_in1kr  c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrk   é€   r4  r=   r�   F©rm   ro   rÖ   r�   r—   r˜   rL  )Úxcit_nano_12_p16_224©rô   rO  ©rL  rN  Ú
model_argsr9  s       r4   rg  rg  @  s=   € äØ ¨B¸!ÀÐRWôY€JäÑe¸JÐeÌ$ÈzÑJdÐ]cÑJdÑe€EØ€Lr5   c           	      óX   — t        ddddddd¬«      }t        d
d	| it        |fi |¤Ž¤Ž}|S )Nrk   re  r4  r=   r�   Fr]  )rm   ro   rÖ   r�   r—   r˜   rl   rL  )Úxcit_nano_12_p16_384rh  ri  s       r4   rl  rl  H  s@   € äØ ¨B¸!ÀÐRWÐbeôg€JäÑe¸JÐeÌ$ÈzÑJdÐ]cÑJdÑe€EØ€Lr5   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrk   éÀ   r4  r=   r�   Trf  rL  )Úxcit_tiny_12_p16_224rh  ri  s       r4   ro  ro  P  ó=   € äØ ¨B¸!ÀÐRVôX€JäÑe¸JÐeÌ$ÈzÑJdÐ]cÑJdÑe€EØ€Lr5   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrk   rn  r4  r=   r�   Trf  rL  )Úxcit_tiny_12_p16_384rh  ri  s       r4   rr  rr  X  rp  r5   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrk   r]  r4  rr   r�   Trf  rL  )Úxcit_small_12_p16_224rh  ri  s       r4   rt  rt  `  ó>   € äØ ¨B¸!ÀÐRVôX€JäÑf¸ZÐfÌ4ÐPZÑKeÐ^dÑKeÑf€EØ€Lr5   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrk   r]  r4  rr   r�   Trf  rL  )Úxcit_small_12_p16_384rh  ri  s       r4   rw  rw  h  ru  r5   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrk   rn  é   r=   çñhãˆµøä>Trf  rL  )Úxcit_tiny_24_p16_224rh  ri  s       r4   r{  r{  p  ó=   € äØ ¨B¸!ÀÐSWôY€JäÑe¸JÐeÌ$ÈzÑJdÐ]cÑJdÑe€EØ€Lr5   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrk   rn  ry  r=   rz  Trf  rL  )Úxcit_tiny_24_p16_384rh  ri  s       r4   r~  r~  x  r|  r5   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrk   r]  ry  rr   rz  Trf  rL  )Úxcit_small_24_p16_224rh  ri  s       r4   r€  r€  €  ó>   € äØ ¨B¸!ÀÐSWôY€JäÑf¸ZÐfÌ4ÐPZÑKeÐ^dÑKeÑf€EØ€Lr5   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrk   r]  ry  rr   rz  Trf  rL  )Úxcit_small_24_p16_384rh  ri  s       r4   rƒ  rƒ  ˆ  r�  r5   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrk   é   ry  rr   rz  Trf  rL  )Úxcit_medium_24_p16_224rh  ri  s       r4   r†  r†  �  ó>   € äØ ¨B¸!ÀÐSWôY€JäÑg¸jÐgÌDÐQ[ÑLfÐ_eÑLfÑg€EØ€Lr5   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrk   r…  ry  rr   rz  Trf  rL  )Úxcit_medium_24_p16_384rh  ri  s       r4   r‰  r‰  ˜  r‡  r5   c           	      óV   — t        dddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	Nrk   rV   ry  rz  Trf  rL  )Úxcit_large_24_p16_224rh  ri  s       r4   r‹  r‹     ó>   € äØ ¨B¸"À$ÐTXôZ€JäÑf¸ZÐfÌ4ÐPZÑKeÐ^dÑKeÑf€EØ€Lr5   c           	      óV   — t        dddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	Nrk   rV   ry  rz  Trf  rL  )Úxcit_large_24_p16_384rh  ri  s       r4   rŽ  rŽ  ¨  rŒ  r5   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrr   re  r4  r=   r�   Frf  rL  )Úxcit_nano_12_p8_224rh  ri  s       r4   r�  r�  ±  ó=   € äØ ¨2¸ÀÐQVôX€JäÑd¸:ÐdÌÈjÑIcÐ\bÑIcÑd€EØ€Lr5   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrr   re  r4  r=   r�   Frf  rL  )Úxcit_nano_12_p8_384rh  ri  s       r4   r“  r“  ¹  r‘  r5   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrr   rn  r4  r=   r�   Trf  rL  )Úxcit_tiny_12_p8_224rh  ri  s       r4   r•  r•  Á  ó=   € äØ ¨2¸ÀÐQUôW€JäÑd¸:ÐdÌÈjÑIcÐ\bÑIcÑd€EØ€Lr5   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrr   rn  r4  r=   r�   Trf  rL  )Úxcit_tiny_12_p8_384rh  ri  s       r4   r˜  r˜  É  r–  r5   c           	      óV   — t        dddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	Nrr   r]  r4  r�   Trf  rL  )Úxcit_small_12_p8_224rh  ri  s       r4   rš  rš  Ñ  ó=   € äØ ¨2¸ÀÐQUôW€JäÑe¸JÐeÌ$ÈzÑJdÐ]cÑJdÑe€EØ€Lr5   c           	      óV   — t        dddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	Nrr   r]  r4  r�   Trf  rL  )Úxcit_small_12_p8_384rh  ri  s       r4   r�  r�  Ù  r›  r5   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrr   rn  ry  r=   rz  Trf  rL  )Úxcit_tiny_24_p8_224rh  ri  s       r4   rŸ  rŸ  á  ó=   € äØ ¨2¸ÀÐRVôX€JäÑd¸:ÐdÌÈjÑIcÐ\bÑIcÑd€EØ€Lr5   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrr   rn  ry  r=   rz  Trf  rL  )Úxcit_tiny_24_p8_384rh  ri  s       r4   r¢  r¢  é  r   r5   c           	      óV   — t        dddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	Nrr   r]  ry  rz  Trf  rL  )Úxcit_small_24_p8_224rh  ri  s       r4   r¤  r¤  ñ  ó=   € äØ ¨2¸ÀÐRVôX€JäÑe¸JÐeÌ$ÈzÑJdÐ]cÑJdÑe€EØ€Lr5   c           	      óV   — t        dddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	Nrr   r]  ry  rz  Trf  rL  )Úxcit_small_24_p8_384rh  ri  s       r4   r§  r§  ù  r¥  r5   c           	      óV   — t        dddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	Nrr   r…  ry  rz  Trf  rL  )Úxcit_medium_24_p8_224rh  ri  s       r4   r©  r©    ó>   € äØ ¨2¸ÀÐRVôX€JäÑf¸ZÐfÌ4ÐPZÑKeÐ^dÑKeÑf€EØ€Lr5   c           	      óV   — t        dddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	Nrr   r…  ry  rz  Trf  rL  )Úxcit_medium_24_p8_384rh  ri  s       r4   r¬  r¬  	  rª  r5   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrr   rV   ry  rk   rz  Trf  rL  )Úxcit_large_24_p8_224rh  ri  s       r4   r®  r®    ó=   € äØ ¨2¸ÀÐSWôY€JäÑe¸JÐeÌ$ÈzÑJdÐ]cÑJdÑe€EØ€Lr5   c           	      óV   — t        dddddd¬«      }t        d	d| it        |fi |¤Ž¤Ž}|S )
Nrr   rV   ry  rk   rz  Trf  rL  )Úxcit_large_24_p8_384rh  ri  s       r4   r±  r±    r¯  r5   Úxcit_nano_12_p16_224_distÚxcit_nano_12_p16_384_distÚxcit_tiny_12_p16_224_distÚxcit_tiny_12_p16_384_distÚxcit_tiny_24_p16_224_distÚxcit_tiny_24_p16_384_distÚxcit_small_12_p16_224_distÚxcit_small_12_p16_384_distÚxcit_small_24_p16_224_distÚxcit_small_24_p16_384_distÚxcit_medium_24_p16_224_distÚxcit_medium_24_p16_384_distÚxcit_large_24_p16_224_distÚxcit_large_24_p16_384_distÚxcit_nano_12_p8_224_distÚxcit_nano_12_p8_384_distÚxcit_tiny_12_p8_224_distr^  r_  r`  ra  rb  rc  )Úxcit_tiny_12_p8_384_distÚxcit_tiny_24_p8_224_distÚxcit_tiny_24_p8_384_distÚxcit_small_12_p8_224_distÚxcit_small_12_p8_384_distÚxcit_small_24_p8_224_distÚxcit_small_24_p8_384_distÚxcit_medium_24_p8_224_distÚxcit_medium_24_p8_384_distÚxcit_large_24_p8_224_distÚxcit_large_24_p8_384_dist)r   NN)FN)rà   r5  )PrZ   r-   Ú	functoolsr   Útypingr   r   r   r   r   r	   rA   Útorch.nnr*   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_features_fxr   Ú_manipulater   Ú	_registryr   r   r   Úcaitr   Ú__all__r|   r   rg   ri   r~   rŒ   r²   rË   r   rD  rO  r[  Údefault_cfgsrg  rl  ro  rr  rt  rw  r{  r~  r€  rƒ  r†  r‰  r‹  rŽ  r�  r“  r•  r˜  rš  r�  rŸ  r¢  r¤  r§  r©  r¬  r®  r±  rW   rž   r5   r4   ú<module>rÛ     s8	  ðñ	ó Ý ß :× :ã Ý ç Aß OÕ OÝ *Ý +Ý 1Ý #ß YÑ YÝ àˆ(€ð ô'& §	¡	ó '&ó ð'&óTô.�R—Y‘Yô .ôb$ˆ"�)‰)ô $ôNA˜"Ÿ)™)ô AôH5ˆ"�)‰)ô 5ôp5ˆr�y‰yô 5ôpbˆ2�9‰9ô bòJ	ó:
óñ %ð B&à"¡DØØJô%LðB&ð
 (©ØØOô*QðB&ð (©ØØOÐ\iô*kðB&ð #¡DØØJô%LðB&ð (©ØØOô*QðB&ð" (©ØØOÐ\iô*kð#B&ð( #¡DØØJô%Lð)B&ð. (©ØØOô*Qð/B&ð4 (©ØØOÐ\iô*kð5B&ð: $¡TØØKô&Mð;B&ð@ )©$ØØPô+RðAB&ðF )©$ØØPÐ]jô+lðGB&ðL $¡TØØKô&MðMB&ðR )©$ØØPô+RðSB&ðX )©$ØØPÐ]jô+lðYB&ð^ %¡dØØLô'Nð_B&ðd *©4ØØQô,SñeB&ðj *©4ØØQÐ^kô,mðkB&ðp $¡TØØKô&MðqB&ðv )©$ØØPô+RðwB&ð| )©$ØØPÐ]jô+lð}B&ðF "¡4ØØIô$KðGB&ðL '©ØØNô)PðMB&ðR '©ØØNÐ[hô)jðSB&ðX "¡4ØØIô$KðYB&ð^ '©ØØNô)Pð_B&ðd '©ØØNÐ[hô)jðeB&ðj "¡4ØØIô$KðkB&ðp '©ØØNô)PðqB&ðv '©ØØNÐ[hô)jðwB&ð| #¡DØØJô%Lð}B&ðB (©ØØOô*QðCB&ðH (©ØØOÐ\iô*kðIB&ðN #¡DØØJô%LñOB&ñT *.ØØOô*Qñ *.ØØOÐ\iô*kñ &*ØØKô&Mñ +/ØØPô+Rñ +/ØØPÐ]jô+lñ %)ØØJô%Lñ *.ØØOô*Qñ *.ØØOÐ\iô*kòB&ó B€ðJ ñ¸ò ó ðð ñ¸ò ó ðð ñ¸ò ó ðð ñ¸ò ó ðð ñ¸ò ó ðð ñ¸ò ó ðð ñ¸ò ó ðð ñ¸ò ó ðð ñ¸ò ó ðð ñ¸ò ó ðð ñ¸$ò ó ðð ñ¸$ò ó ðð ñ¸ò ó ðð ñ¸ò ó ðð ñ°tò ó ðð ñ°tò ó ðð ñ°tò ó ðð ñ°tò ó ðð ñ¸ò ó ðð ñ¸ò ó ðð ñ°tò ó ðð ñ°tò ó ðð ñ¸ò ó ðð ñ¸ò ó ðð ñ¸ò ó ðð ñ¸ò ó ðð ñ¸ò ó ðð ñ¸ò ó ðñ ˜Hð  'àÐ!Dð 'ð  Ð!Dð 'ð  Ð!Dð	 'ð
  Ð!Dð 'ð  Ð!Dð 'ð  Ð!Dð 'ð !Ð"Fð 'ð !Ð"Fð 'ð !Ð"Fð 'ð !Ð"Fð 'ð "Ð#Hð 'ð "Ð#Hð 'ð !Ð"Fð 'ð !Ð"Fð 'ð$ Ð Bð% 'ð& Ð Bð' 'ð( Ð Bð) 'ð* !CØ BØ BØ!DØ!DØ!DØ!DØ"FØ"FØ!DØ!Dò? 'õ  r5   