Ë
    éÓ^j‚Z  ã                   ó\  — d Z ddlmZ ddlmZmZmZmZmZm	Z	 ddl
Z
ddlmZ ddlmZmZ ddlmZmZmZmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZmZ g d¢Z  G d„ dejB                  «      Z" G d„ dejB                  «      Z# G d„ dejB                  «      Z$ G d„ dejB                  «      Z% G d„ dejB                  «      Z&d5d„Z'd6d„Z(d7d„Z) e e)ddd¬«       e)dd¬«       e)dd d¬«       e)dd!¬«       e)dd"¬«       e)dd#d¬«       e)dd$¬«       e)dd%¬«       e)dd&¬«       e)dd'd(¬«      d)œ
«      Z*ed6d*e&fd+„«       Z+ed6d*e&fd,„«       Z,ed6d*e&fd-„«       Z-ed6d*e&fd.„«       Z.ed6d*e&fd/„«       Z/ed6d*e&fd0„«       Z0ed6d*e&fd1„«       Z1ed6d*e&fd2„«       Z2ed6d*e&fd3„«       Z3ed6d*e&fd4„«       Z4y)8a3   Class-Attention in Image Transformers (CaiT)

Paper: 'Going deeper with Image Transformers' - https://arxiv.org/abs/2103.17239

Original code and weights from https://github.com/facebookresearch/deit, copyright below

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
é    )Úpartial)ÚListÚOptionalÚTupleÚUnionÚTypeÚAnyN©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Ú
PatchEmbedÚMlpÚDropPathÚtrunc_normal_Úuse_fused_attné   )Úbuild_model_with_cfg)Úfeature_take_indices)Ú
checkpointÚcheckpoint_seq)Úregister_modelÚgenerate_default_cfgs)ÚCaitÚ	ClassAttnÚLayerScaleBlockClassAttnÚLayerScaleBlockÚTalkingHeadAttnc                   ó‚   ‡ — e Zd ZU ej                  j
                  e   ed<   	 	 	 	 	 	 d	dededede	de	f
ˆ fd„Z
d„ Zˆ xZS )
r   Ú
fused_attnÚdimÚ	num_headsÚqkv_biasÚ	attn_dropÚ	proj_dropc                 óÊ  •— t         ‰
| �  «        ||dœ}|| _        ||z  }	|	dz  | _        t	        «       | _        t        j                  ||fd|i|¤Ž| _        t        j                  ||fd|i|¤Ž| _	        t        j                  ||fd|i|¤Ž| _
        t        j                  |«      | _        t        j                  ||fi |¤Ž| _        t        j                  |«      | _        y )N©ÚdeviceÚdtypeç      à¿Úbias)ÚsuperÚ__init__r!   Úscaler   r   ÚnnÚLinearÚqÚkÚvÚDropoutr#   Úprojr$   ©Úselfr    r!   r"   r#   r$   r'   r(   ÚddÚhead_dimÚ	__class__s             €ú[/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/cait.pyr,   zClassAttn.__init__    sÌ   ø€ ô 	‰ÑÔØ¨Ñ/ˆØ"ˆŒØ˜)Ñ#ˆØ Ñ%ˆŒ
Ü(Ó*ˆŒä—‘˜3 Ñ9¨(Ð9°bÑ9ˆŒÜ—‘˜3 Ñ9¨(Ð9°bÑ9ˆŒÜ—‘˜3 Ñ9¨(Ð9°bÑ9ˆŒÜŸ™ IÓ.ˆŒÜ—I‘I˜c 3Ñ-¨"Ñ-ˆŒ	ÜŸ™ IÓ.ˆ�ó    c                 óô  — |j                   \  }}}| j                  |d d …df   «      j                  d«      j                  |d| j                  || j                  z  «      j                  dddd«      }| j                  |«      j                  ||| j                  || j                  z  «      j                  dddd«      }| j                  |«      j                  ||| j                  || j                  z  «      j                  dddd«      }| j                  rPt        j                  j                  j                  |||| j                  r| j                  j                  nd¬«      }nL|| j                   z  }||j#                  dd«      z  }	|	j%                  d¬	«      }	| j                  |	«      }	|	|z  }|j#                  dd«      j                  |d|«      }| j'                  |«      }| j)                  |«      }|S )
Nr   r   é   é   ç        )Ú	dropout_péþÿÿÿéÿÿÿÿ©r    )Úshaper0   Ú	unsqueezeÚreshaper!   Úpermuter1   r2   r   Útorchr.   Ú
functionalÚscaled_dot_product_attentionÚtrainingr#   Úpr-   Ú	transposeÚsoftmaxr4   r$   )
r6   ÚxÚBÚNÚCr0   r1   r2   Úx_clsÚattns
             r:   ÚforwardzClassAttn.forward8   s²  € Ø—'‘'‰ˆˆ1ˆaØ�F‰F�1’Q˜�T‘7‹O×%Ñ% aÓ(×0Ñ0°°A°t·~±~ÀqÈDÏNÉNÑGZÓ[×cÑcÐdeÐghÐjkÐmnÓoˆØ�F‰F�1‹I×Ñ˜a  D§N¡N°A¸¿¹Ñ4GÓH×PÑPÐQRÐTUÐWXÐZ[Ó\ˆØ�F‰F�1‹I×Ñ˜a  D§N¡N°A¸¿¹Ñ4GÓH×PÑPÐQRÐTUÐWXÐZ[Ó\ˆà�?Š?Ü—H‘H×'Ñ'×DÑDØ�1�aØ.2¯mªm˜$Ÿ.™.×*Ò*Àð Eó ‰Eð
 �D—J‘J‘ˆAØ�q—{‘{ 2 rÓ*Ñ*ˆDØ—<‘< B�<Ó'ˆDØ—>‘> $Ó'ˆDØ˜1‘HˆEà—‘  1Ó%×-Ñ-¨a°°AÓ6ˆØ—	‘	˜%Ó ˆØ—‘˜uÓ%ˆàˆr;   ©é   Fr?   r?   NN)Ú__name__Ú
__module__Ú__qualname__rH   ÚjitÚFinalÚboolÚ__annotations__ÚintÚfloatr,   rU   Ú__classcell__©r9   s   @r:   r   r      sf   ø… ð —	‘	—‘ Ñ%Ó%ð
 Ø"Ø!Ø!ØØñ/àð/ð ð/ð ð	/ð
 ð/ð õ/ö0r;   r   c                   ó   ‡ — e Zd Zdddddej                  ej
                  eedddfdedede	d	e
d
e	de	de	deej                     deej                     deej                     deej                     de	fˆ fd„Zd„ Zˆ xZS )r   ç      @Fr?   ç-Cëâ6?Nr    r!   Ú	mlp_ratior"   r$   r#   Ú	drop_pathÚ	act_layerÚ
norm_layerÚ
attn_blockÚ	mlp_blockÚinit_valuesc                 óà  •— t         ‰| �  «        ||dœ} |	|fi |¤Ž| _         |
|f||||dœ|¤Ž| _        |dkD  rt	        |«      nt        j                  «       | _         |	|fi |¤Ž| _        t        ||z  «      } |d||||dœ|¤Ž| _
        t        j                  |t        j                  |fi |¤Žz  «      | _        t        j                  |t        j                  |fi |¤Žz  «      | _        y ©Nr&   )r!   r"   r#   r$   r?   )Úin_featuresÚhidden_featuresrh   Údrop© ©r+   r,   Únorm1rT   r   r.   ÚIdentityrg   Únorm2r_   ÚmlpÚ	ParameterrH   ÚonesÚgamma_1Úgamma_2©r6   r    r!   rf   r"   r$   r#   rg   rh   ri   rj   rk   rl   r'   r(   r7   Úmlp_hidden_dimr9   s                    €r:   r,   z!LayerScaleBlockClassAttn.__init__T   óÿ   ø€ ô" 	‰ÑÔØ¨Ñ/ˆÙ Ñ* rÑ*ˆŒ
ÙØð
àØØØñ
ð ñ
ˆŒ	ð 1:¸B²œ )Ô,ÄBÇKÁKÃMˆŒÙ Ñ* rÑ*ˆŒ
Ü˜S 9™_Ó-ˆÙð 
ØØ*ØØñ	
ð
 ñ
ˆŒô —|‘| K´%·*±*¸SÑ2GÀBÑ2GÑ$GÓHˆŒÜ—|‘| K´%·*±*¸SÑ2GÀBÑ2GÑ$GÓHˆ�r;   c           	      ó4  — t        j                  ||fd¬«      }|| j                  | j                  | j	                  | j                  |«      «      z  «      z   }|| j                  | j                  | j                  | j                  |«      «      z  «      z   }|S )Nr   rC   )	rH   Úcatrg   rz   rT   rt   r{   rw   rv   )r6   rO   rS   Úus       r:   rU   z LayerScaleBlockClassAttn.forward}   sv   € Ü�I‰I�u˜a�j aÔ(ˆØ˜Ÿ™ t§|¡|°d·i±iÀÇ
Á
È1ÃÓ6NÑ'NÓOÑOˆØ˜Ÿ™ t§|¡|°d·h±h¸t¿z¹zÈ%Ó?PÓ6QÑ'QÓRÑRˆØˆr;   )rX   rY   rZ   r.   ÚGELUÚ	LayerNormr   r   r_   r`   r]   r   ÚModuler,   rU   ra   rb   s   @r:   r   r   Q   sß   ø„ ð  "Ø"Ø!Ø!Ø!Ø)+¯©Ø*,¯,©,Ø*3Ø),Ø!%ØØñ'Iàð'Ið ð'Ið ð	'Ið
 ð'Ið ð'Ið ð'Ið ð'Ið ˜BŸI™I‘ð'Ið ˜RŸY™Y™ð'Ið ˜RŸY™Y™ð'Ið ˜BŸI™I‘ð'Ið õ'IöRr;   r   c                   óH   ‡ — e Zd Z	 	 	 	 	 	 ddededededef
ˆ fd„Zd„ Zˆ xZS )	r   r    r!   r"   r#   r$   c                 óª  •— t         ‰
| �  «        ||dœ}|| _        ||z  }	|	dz  | _        t	        j
                  ||dz  fd|i|¤Ž| _        t	        j                  |«      | _        t	        j
                  ||fi |¤Ž| _	        t	        j
                  ||fi |¤Ž| _
        t	        j
                  ||fi |¤Ž| _        t	        j                  |«      | _        y )Nr&   r)   r>   r*   )r+   r,   r!   r-   r.   r/   Úqkvr3   r#   r4   Úproj_lÚproj_wr$   r5   s             €r:   r,   zTalkingHeadAttn.__init__‡   s¼   ø€ ô 	‰ÑÔØ¨Ñ/ˆà"ˆŒà˜)Ñ#ˆà Ñ%ˆŒ
ä—9‘9˜S #¨¡'Ñ?°Ð?¸BÑ?ˆŒÜŸ™ IÓ.ˆŒä—I‘I˜c 3Ñ-¨"Ñ-ˆŒ	ä—i‘i 	¨9Ñ;¸Ñ;ˆŒÜ—i‘i 	¨9Ñ;¸Ñ;ˆŒäŸ™ IÓ.ˆ�r;   c                 óÎ  — |j                   \  }}}| j                  |«      j                  ||d| j                  || j                  z  «      j	                  ddddd«      }|d   | j
                  z  |d   |d   }}}||j                  dd«      z  }	| j                  |	j	                  dddd«      «      j	                  dddd«      }	|	j                  d¬«      }	| j                  |	j	                  dddd«      «      j	                  dddd«      }	| j                  |	«      }	|	|z  j                  dd«      j                  |||«      }| j                  |«      }| j                  |«      }|S )	Nr>   r=   r   r   é   rA   rB   rC   )rD   r‡   rF   r!   rG   r-   rM   rˆ   rN   r‰   r#   r4   r$   )
r6   rO   rP   rQ   rR   r‡   r0   r1   r2   rT   s
             r:   rU   zTalkingHeadAttn.forward¤   sL  € Ø—'‘'‰ˆˆ1ˆaØ�h‰h�q‹k×!Ñ! ! Q¨¨4¯>©>¸1ÀÇÁÑ;NÓO×WÑWÐXYÐ[\Ð^_ÐabÐdeÓfˆØ�a‘&˜4Ÿ:™:Ñ% s¨1¡v¨s°1©vˆaˆ1ˆà�1—;‘;˜r 2Ó&Ñ&ˆà�{‰{˜4Ÿ<™<¨¨1¨a°Ó3Ó4×<Ñ<¸QÀÀ1ÀaÓHˆà�|‰| ˆ|Ó#ˆà�{‰{˜4Ÿ<™<¨¨1¨a°Ó3Ó4×<Ñ<¸QÀÀ1ÀaÓHˆØ�~‰~˜dÓ#ˆà�A‰X× Ñ   AÓ&×.Ñ.¨q°!°QÓ7ˆØ�I‰I�a‹LˆØ�N‰N˜1ÓˆØˆr;   rV   )	rX   rY   rZ   r_   r]   r`   r,   rU   ra   rb   s   @r:   r   r   „   sQ   ø„ ð Ø"Ø!Ø!ØØñ/àð/ð ð/ð ð	/ð
 ð/ð õ/ö:r;   r   c                   ó   ‡ — e Zd Zdddddej                  ej
                  eedddfdedede	d	e
d
e	de	de	deej                     deej                     deej                     deej                     de	fˆ fd„Zd„ Zˆ xZS )r   rd   Fr?   re   Nr    r!   rf   r"   r$   r#   rg   rh   ri   rj   rk   rl   c                 óà  •— t         ‰| �  «        ||dœ} |	|fi |¤Ž| _         |
|f||||dœ|¤Ž| _        |dkD  rt	        |«      nt        j                  «       | _         |	|fi |¤Ž| _        t        ||z  «      } |d||||dœ|¤Ž| _
        t        j                  |t        j                  |fi |¤Žz  «      | _        t        j                  |t        j                  |fi |¤Žz  «      | _        y rn   rs   r|   s                    €r:   r,   zLayerScaleBlock.__init__»   r~   r;   c           	      ó  — || j                  | j                  | j                  | j                  |«      «      z  «      z   }|| j                  | j                  | j                  | j                  |«      «      z  «      z   }|S ©N)rg   rz   rT   rt   r{   rw   rv   ©r6   rO   s     r:   rU   zLayerScaleBlock.forwardä   sa   € Ø�—‘˜tŸ|™|¨d¯i©i¸¿
¹
À1»Ó.FÑFÓGÑGˆØ�—‘˜tŸ|™|¨d¯h©h°t·z±zÀ!³}Ó.EÑEÓFÑFˆØˆr;   )rX   rY   rZ   r.   r‚   rƒ   r   r   r_   r`   r]   r   r„   r,   rU   ra   rb   s   @r:   r   r   ¸   sß   ø„ ð  "Ø"Ø!Ø!Ø!Ø)+¯©Ø*,¯,©,Ø*9Ø),Ø!%ØØñ'Iàð'Ið ð'Ið ð	'Ið
 ð'Ið ð'Ið ð'Ið ð'Ið ˜BŸI™I‘ð'Ið ˜RŸY™Y™ð'Ið ˜RŸY™Y™ð'Ið ˜BŸI™I‘ð'Ið õ'IöRr;   r   c            7       ó  ‡ — e Zd Zdddddddddd	d
d
d
d
d
eee eej                  d¬«      ej                  e
edeeddddfdedededededededededededededededeej$                     d eej$                     d!eej$                     d"eej$                     d#eej$                     d$eej$                     d%eej$                     d&ed'eej$                     d(eej$                     d)ed*ef6ˆ fd+„Zd,„ Zej,                  j.                  d-„ «       Zej,                  j.                  dAd.„«       Zej,                  j.                  dBd/„«       Zej,                  j.                  d0ej$                  fd1„«       ZdCdedee   fd2„Z	 	 	 	 	 dDd3ej<                  d4eeee e   f      d5ed6ed7ed8ed0ee ej<                     e!ej<                  e ej<                     f   f   fd9„Z"	 	 	 dEd4eee e   f   d:ed;efd<„Z#d=„ Z$dBd>efd?„Z%d@„ Z&ˆ xZ'S )Fr   éà   é   r>   éè  Útokené   é   rd   Tr?   ç�íµ ÷Æ°>)Úepsre   r=   NÚimg_sizeÚ
patch_sizeÚin_chansÚnum_classesÚglobal_poolÚ	embed_dimÚdepthr!   rf   r"   Ú	drop_rateÚpos_drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_rateÚblock_layersÚblock_layers_tokenÚpatch_layerri   rh   rj   rk   rl   Úattn_block_token_onlyÚmlp_block_token_onlyÚdepth_token_onlyÚmlp_ratio_token_onlyc                 óL  •— t         ‰$| �  «        ||dœ}|dv sJ ‚|| _        || _        || _        |x| _        x| _        | _        d| _         |d||||dœ|¤Ž| _	        | j                  j                  }t        | j                  d«      r| j                  j                  «       n|} t        j                  t        j                   dd|fi |¤Ž«      | _        t        j                  t        j                   d||fi |¤Ž«      | _        t        j&                  |¬«      | _        t+        |«      D �!cg c]  }!|‘Œ }"}!t        j,                  t+        |«      D �!cg c]  }! |d|||	|
|||"|!   |||||dœ|¤Ž‘Œ c}!Ž | _        t+        |«      D �!cg c]  }!t1        || d	|!› �¬
«      ‘Œ c}!| _        t        j4                  t+        |«      D �#cg c]  }# |d||||
|||||dœ	|¤Ž‘Œ c}#«      | _         ||fi |¤Ž| _        t        j&                  |«      | _        |dkD  rt        j<                  ||fi |¤Žnt        j>                  «       | _         tC        | j$                  d¬«       tC        | j"                  d¬«       | jE                  | jF                  «       y c c}!w c c}!w c c}!w c c}#w )Nr&   ©Ú r•   ÚavgF)rš   r›   rœ   rŸ   Ú
feat_ratior   )rL   )r    r!   rf   r"   r$   r#   rg   ri   rh   rj   rk   rl   úblocks.)Únum_chsÚ	reductionÚmodule)	r    r!   rf   r"   ri   rh   rj   rk   rl   r   ç{®Gáz”?©Ústdrr   )$r+   r,   r�   rœ   rž   Únum_featuresÚhead_hidden_sizerŸ   Úgrad_checkpointingÚpatch_embedÚnum_patchesÚhasattrr±   r.   rx   rH   ÚzerosÚ	cls_tokenÚ	pos_embedr3   Úpos_dropÚrangeÚ
SequentialÚblocksÚdictÚfeature_infoÚ
ModuleListÚblocks_token_onlyÚnormÚ	head_dropr/   ru   Úheadr   ÚapplyÚ_init_weights)%r6   rš   r›   rœ   r�   rž   rŸ   r    r!   rf   r"   r¡   r¢   r£   r¤   r¥   r¦   r§   r¨   ri   rh   rj   rk   rl   r©   rª   r«   r¬   r'   r(   r7   r½   ÚrÚiÚdprÚ_r9   s%                                       €r:   r,   zCait.__init__í   s¶  ø€ ô@ 	‰ÑÔØ¨Ñ/ˆØÐ2Ñ2Ð2Ð2à&ˆÔØ ˆŒØ&ˆÔØENÐNˆÔÐN˜DÔ1°D´NØ"'ˆÔá&ð 
ØØ!ØØñ	
ð
 ñ
ˆÔð ×&Ñ&×2Ñ2ˆÜ-4°T×5EÑ5EÀ|Ô-TˆD×Ñ×'Ñ'Ô)ÐZdˆäŸ™¤e§k¡k°!°Q¸	Ñ&HÀRÑ&HÓIˆŒÜŸ™¤e§k¡k°!°[À)Ñ&RÈrÑ&RÓSˆŒÜŸ
™
 ]Ô3ˆŒä',¨U£|Ö4 !Š~Ð4ˆÐ4Ü—m‘mô ˜“<ö&!ð ñ '3ð '
ØØØØØ$Ø$Ø˜!‘fØ!ØØ!ØØ#ñ'
ð ó'
ò &!ð "ˆŒô bgÐglÓamÖnÐ\]œT¨)¸qÈ7ÐSTÐRUÈÖWÒnˆÔä!#§¡ô Ð)Ó*ö0,ð ñ 1Cð 1
ØØØ*ØØ!ØØ,Ø*Ø#ñ1
ð ó1
ò 0,ó "-ˆÔñ ˜yÑ/¨BÑ/ˆŒ	äŸ™ IÓ.ˆŒØ?JÈQº”B—I‘I˜i¨Ñ;¸Ò;ÔTV×T_ÑT_ÓTaˆŒ	ä�d—n‘n¨#Õ.Ü�d—n‘n¨#Õ.Ø�
‰
�4×%Ñ%Õ&ùòM 5ùò&!ùò oùò0,s   Ä.	JÅJÆ
JÇJ!c                 óê  — t        |t        j                  «      rjt        |j                  d¬«       t        |t        j                  «      r8|j
                  �+t        j                  j                  |j
                  d«       y y y t        |t        j                  «      rUt        j                  j                  |j
                  d«       t        j                  j                  |j                  d«       y y )Nr¶   r·   r   ç      ð?)	Ú
isinstancer.   r/   r   Úweightr*   ÚinitÚ	constant_rƒ   )r6   Úms     r:   rÎ   zCait._init_weightsM  s˜   € Ü�aœŸ™Ô#Ü˜!Ÿ(™(¨Õ,Ü˜!œRŸY™YÔ'¨A¯F©FÐ,>Ü—‘×!Ñ! !§&¡&¨!Õ,ð -?Ð'ä˜œ2Ÿ<™<Ô(Ü�G‰G×Ñ˜aŸf™f aÔ(Ü�G‰G×Ñ˜aŸh™h¨Õ,ð )r;   c                 ó
   — ddhS )NrÁ   rÀ   rr   ©r6   s    r:   Úno_weight_decayzCait.no_weight_decayV  s   € à˜[Ð)Ð)r;   c                 ó   — || _         y r�   )r»   )r6   Úenables     r:   Úset_grad_checkpointingzCait.set_grad_checkpointingZ  s
   € à"(ˆÕr;   c                 ó   ‡ — ˆ fd„}|S )Nc                 óâ  •— t        dD �cg c]  }| j                  |«      ‘Œ c}«      ry| j                  d«      r t        | j                  d«      d   «      dz   S | j                  d«      rNt	        ‰j
                  «      t	        ‰j                  «      z
  dz   }t        | j                  d«      d   «      |z   S | j                  d«      rt	        ‰j
                  «      S t        d«      S c c}w )	N)rÀ   rÁ   r¼   r   r²   ú.r   zblocks_token_only.znorm.Úinf)ÚanyÚ
startswithr_   ÚsplitÚlenrÅ   rÉ   r`   )ÚnameÚnÚ	to_offsetr6   s      €r:   Ú_matcherz$Cait.group_matcher.<locals>._matcher`  sÄ   ø€ ÜÐ0YÖZ¨1�D—O‘O AÕ&ÒZÔ[ØØ—‘ Ô+Ü˜4Ÿ:™: c›?¨1Ñ-Ó.°Ñ2Ð2Ø—‘Ð!5Ô6ä §¡Ó,¬s°4×3IÑ3IÓ/JÑJÈQÑN�	Ü˜4Ÿ:™: c›?¨1Ñ-Ó.°Ñ:Ð:Ø—‘ Ô)Ü˜4Ÿ;™;Ó'Ð'ä˜U“|Ð#ùò [s   ‹C,rr   )r6   Úcoarserë   s   `  r:   Úgroup_matcherzCait.group_matcher^  s   ø€ ô	$ð ˆr;   Úreturnc                 ó   — | j                   S r�   )rÌ   rÛ   s    r:   Úget_classifierzCait.get_classifiero  s   € à�y‰yÐr;   c                 ó¸   — || _         |�|dv sJ ‚|| _        |dkD  r&t        j                  | j                  |«      | _        y t        j
                  «       | _        y )Nr®   r   )r�   rž   r.   r/   r¹   ru   rÌ   )r6   r�   rž   s      r:   Úreset_classifierzCait.reset_classifiers  sU   € Ø&ˆÔØÐ"ØÐ"6Ñ6Ð6Ð6Ø*ˆDÔØALÈqÂ”B—I‘I˜d×/Ñ/°Ó=ˆ�	ÔVX×VaÑVaÓVcˆ�	r;   rO   ÚindicesrÊ   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc           	      ó6  — |dv sJ d«       ‚|dk(  }g }t        t        | j                  «      |«      \  }	}
|j                  \  }}}}| j	                  |«      }|| j
                  z   }| j                  |«      }t        j                  j                  «       s|s| j                  }n| j                  d|
dz    }t        |«      D ]m  \  }}| j                  r+t        j                  j                  «       st        ||«      }n ||«      }||	v sŒJ|j                  |r| j                  |«      n|«       Œo |ra| j                  j                  ||f«      \  }}|D �cg c]6  }|j!                  |||d«      j#                  dddd	«      j%                  «       ‘Œ8 }}|r|S | j&                  j)                  |j                  d   dd«      }t        | j*                  «      D ]  \  }} |||«      }Œ t        j,                  ||fd¬
«      }| j                  |«      }||fS c c}w )aÖ   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        )ÚNCHWÚNLCz)Output format must be one of NCHW or NLC.rø   Nr   rB   r   r>   r=   rC   )r   rç   rÅ   rD   r¼   rÁ   rÂ   rH   r[   Úis_scriptingÚ	enumerater»   r   ÚappendrÊ   Údynamic_feat_sizerF   rG   Ú
contiguousrÀ   ÚexpandrÉ   r€   )r6   rO   ró   rÊ   rô   rõ   rö   rF   ÚintermediatesÚtake_indicesÚ	max_indexrP   rÒ   ÚheightÚwidthrÅ   rÐ   ÚblkÚHÚWÚyÚ
cls_tokenss                         r:   Úforward_intermediateszCait.forward_intermediatesz  s   € ð& ˜_Ñ,ÐYÐ.YÓYÐ,Ø Ñ&ˆØˆÜ"6´s¸4¿;¹;Ó7GÈÓ"QÑˆ�ið  Ÿg™gÑˆˆ1ˆf�eØ×Ñ˜QÓˆØ�—‘ÑˆØ�M‰M˜!Óˆä�9‰9×!Ñ!Ô#©:Ø—[‘[‰Fà—[‘[  )¨a¡-Ð0ˆFÜ Ó'ò 	B‰FˆAˆsØ×&Ò&¬u¯y©y×/EÑ/EÔ/GÜ˜s AÓ&‘á˜“F�Ø�LÒ à×$Ñ$±T T§Y¡Y¨q¤\¸qÕAð	Bñ à×#Ñ#×5Ñ5°v¸u°oÓF‰DˆAˆqØ^kÖlÐYZ˜QŸY™Y q¨!¨Q°Ó3×;Ñ;¸A¸qÀ!ÀQÓG×RÑRÕTÐlˆMÐláØ Ð ð —^‘^×*Ñ*¨1¯7©7°1©:°r¸2Ó>ˆ
Ü × 6Ñ 6Ó7ò 	,‰FˆAˆsÙ˜Q 
Ó+‰Jð	,ä�I‰I�z 1�o¨1Ô-ˆØ�I‰I�a‹Lˆà�-ÐÐùò ms   Å;HÚ
prune_normÚ
prune_headc                 ó  — t        t        | j                  «      |«      \  }}| j                  d|dz    | _        |rt        j                  «       | _        |r+t        j                  «       | _        | j                  dd«       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r¯   )	r   rç   rÅ   r.   ru   rÊ   rÈ   rÉ   rò   )r6   ró   r  r  r  r  s         r:   Úprune_intermediate_layerszCait.prune_intermediate_layers·  sk   € ô #7´s¸4¿;¹;Ó7GÈÓ"QÑˆ�iØ—k‘k . 9¨q¡=Ð1ˆŒÙÜŸ™›ˆDŒIÙÜ%'§]¡]£_ˆDÔ"Ø×!Ñ! ! RÔ(ØÐr;   c                 ó   — | j                  |«      }|| j                  z   }| j                  |«      }| j                  r5t        j
                  j                  «       st        | j                  |«      }n| j                  |«      }| j                  j                  |j                  d   dd«      }t        | j                  «      D ]  \  }} |||«      }Œ t	        j                  ||fd¬«      }| j                  |«      }|S )Nr   rB   r   rC   )r¼   rÁ   rÂ   r»   rH   r[   rú   r   rÅ   rÀ   rÿ   rD   rû   rÉ   r€   rÊ   )r6   rO   r	  rÐ   r  s        r:   Úforward_featureszCait.forward_featuresÈ  sÓ   € Ø×Ñ˜QÓˆØ�—‘ÑˆØ�M‰M˜!ÓˆØ×"Ò"¬5¯9©9×+AÑ+AÔ+CÜ˜tŸ{™{¨AÓ.‰Aà—‘˜A“ˆAØ—^‘^×*Ñ*¨1¯7©7°1©:°r¸2Ó>ˆ
Ü × 6Ñ 6Ó7ò 	,‰FˆAˆsÙ˜Q 
Ó+‰Jð	,ä�I‰I�z 1�o¨1Ô-ˆØ�I‰I�a‹LˆØˆr;   Ú
pre_logitsc                 óÌ   — | j                   r3| j                   dk(  r|d d …dd …f   j                  d¬«      n|d d …df   }| j                  |«      }|r|S | j                  |«      S )Nr°   r   rC   r   )rž   ÚmeanrË   rÌ   )r6   rO   r  s      r:   Úforward_headzCait.forward_head×  s_   € Ø×ÒØ(,×(8Ñ(8¸EÒ(A�’!�Q‘R�%‘—‘ !�Ô$ÀqÊÈAÈÁwˆAØ�N‰N˜1ÓˆÙˆqÐ0 D§I¡I¨a£LÐ0r;   c                 óJ   — | j                  |«      }| j                  |«      }|S r�   )r  r  r�   s     r:   rU   zCait.forwardÝ  s'   € Ø×!Ñ! !Ó$ˆØ×Ñ˜aÓ ˆØˆr;   )T©Fr�   )NFFrø   F)r   FT)(rX   rY   rZ   r   r   r   r   r.   rƒ   r‚   r   r   r   r_   Ústrr`   r]   r   r„   r,   rÎ   rH   r[   ÚignorerÜ   rß   rí   rð   r   rò   ÚTensorr   r   r   r
  r  r  r  rU   ra   rb   s   @r:   r   r   ê   sm  ø„ ð
  Ø ØØ#Ø&Ø ØØØ!Ø!Ø!Ø#%Ø$&Ø$&Ø$&Ø,;Ø2JØ+5Ù*1°"·,±,ÀDÔ*IØ)+¯©Ø*9Ø),Ø!%Ø5>Ø47Ø$%Ø*-ØØñ=^'àð^'ð ð^'ð ð	^'ð
 ð^'ð ð^'ð ð^'ð ð^'ð ð^'ð ð^'ð ð^'ð ð^'ð !ð^'ð "ð^'ð "ð^'ð  "ð!^'ð" ˜rŸy™y™/ð#^'ð$ !% R§Y¡Y¡ð%^'ð& ˜bŸi™i™ð'^'ð( ˜RŸY™Y™ð)^'ð* ˜BŸI™I‘ð+^'ð, ˜RŸY™Y™ð-^'ð. ˜BŸI™I‘ð/^'ð0 ð1^'ð2 $(¨¯	©	¡?ð3^'ð4 #' r§y¡y¡/ð5^'ð6 "ð7^'ð8 #(õ9^'ò@-ð ‡Y�Y×Ññ*ó ð*ð ‡Y�Y×Ñò)ó ð)ð ‡Y�Y×Ñòó ðð  ‡Y�Y×Ñð §	¡	ò ó ðñd¨Cð d¸hÀs¹mó dð 8<ØØ$Ø$Ø',ñ; à�|‰|ð; ð ˜e C¨¨c© NÑ3Ñ4ð; ð ð	; ð
 ð; ð ð; ð !%ð; ð 
ˆt�E—L‘LÑ! 5¨¯©°t¸E¿L¹LÑ7IÐ)IÑ#JÐJÑ	Kó; ð~ ./Ø$Ø#ñ	à˜3  S¡	˜>Ñ*ðð ðð ó	ò"ñ1¨$ó 1ör;   r   c                 óv   — d| v r| d   } i }| j                  «       D ]  \  }}|||j                  dd«      <   Œ |S )NÚmodelzmodule.r¯   )ÚitemsÚreplace)Ú
state_dictr  Úcheckpoint_no_moduler1   r2   s        r:   Úcheckpoint_filter_fnr   ã  sR   € Ø�*ÑØ Ñ(ˆ
ØÐØ× Ñ Ó"ò ;‰ˆˆ1Ø9:Ð˜QŸY™Y y°"Ó5Ò6ð;àÐr;   c                 ór   — |j                  dd«      }t        t        | |ft        t	        |d¬«      dœ|¤Ž}|S )NÚout_indicesr>   Úgetter)r"  Úfeature_cls)Úpretrained_filter_fnÚfeature_cfg)Úpopr   r   r   rÆ   )ÚvariantÚ
pretrainedÚkwargsr"  r  s        r:   Ú_create_caitr+  ì  sJ   € Ø—*‘*˜]¨AÓ.€KÜ ÜØØðô 2Ü [¸hÔGñð ñ€Eð €Lr;   c                 ó4   — | ddd dddt         t        dddd	œ|¥S )
Nr”   )r>   é€  r-  rÔ   ÚbicubicTzpatch_embed.projrÌ   z
apache-2.0)Úurlr�   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚfixed_input_sizer  r¸   Ú
first_convÚ
classifierÚlicenser
   )r/  r*  s     r:   Ú_cfgr8  ù  s5   € àØ¨=ÀtØ¨)ÈÜ%Ô.BØ(¸Øñð ðð r;   ztimm/z1https://dl.fbaipublicfiles.com/deit/XXS24_224.pth)r>   r’   r’   )Ú	hf_hub_idr/  r0  z1https://dl.fbaipublicfiles.com/deit/XXS24_384.pth)r9  r/  z1https://dl.fbaipublicfiles.com/deit/XXS36_224.pthz1https://dl.fbaipublicfiles.com/deit/XXS36_384.pthz0https://dl.fbaipublicfiles.com/deit/XS24_384.pthz/https://dl.fbaipublicfiles.com/deit/S24_224.pthz/https://dl.fbaipublicfiles.com/deit/S24_384.pthz/https://dl.fbaipublicfiles.com/deit/S36_384.pthz/https://dl.fbaipublicfiles.com/deit/M36_384.pthz/https://dl.fbaipublicfiles.com/deit/M48_448.pth)r>   éÀ  r:  )
zcait_xxs24_224.fb_dist_in1kzcait_xxs24_384.fb_dist_in1kzcait_xxs36_224.fb_dist_in1kzcait_xxs36_384.fb_dist_in1kzcait_xs24_384.fb_dist_in1kzcait_s24_224.fb_dist_in1kzcait_s24_384.fb_dist_in1kzcait_s36_384.fb_dist_in1kzcait_m36_384.fb_dist_in1kzcait_m48_448.fb_dist_in1krî   c           	      óT   — t        ddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	Nr“   éÀ   é   r‹   çñhãˆµøä>©r›   rŸ   r    r!   rl   r)  )Úcait_xxs24_224©rÆ   r+  ©r)  r*  Ú
model_argsr  s       r:   r@  r@  5  ó7   € ä ¨s¸"ÈÐW[Ô\€JÜÑ_°jÐ_ÄDÈÑD^ÐW]ÑD^Ñ_€EØ€Lr;   c           	      óT   — t        ddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	Nr“   r<  r=  r‹   r>  r?  r)  )Úcait_xxs24_384rA  rB  s       r:   rF  rF  <  rD  r;   c           	      óT   — t        ddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	Nr“   r<  é$   r‹   r>  r?  r)  )Úcait_xxs36_224rA  rB  s       r:   rI  rI  C  rD  r;   c           	      óT   — t        ddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	Nr“   r<  rH  r‹   r>  r?  r)  )Úcait_xxs36_384rA  rB  s       r:   rK  rK  J  rD  r;   c           	      óT   — t        ddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	Nr“   i   r=  é   r>  r?  r)  )Úcait_xs24_384rA  rB  s       r:   rN  rN  Q  s7   € ä ¨s¸"ÈÐW[Ô\€JÜÑ^°ZÐ^Ä4È
ÑC]ÐV\ÑC]Ñ^€EØ€Lr;   c           	      óT   — t        ddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	Nr“   r-  r=  rW   r>  r?  r)  )Úcait_s24_224rA  rB  s       r:   rP  rP  X  ó7   € ä ¨s¸"ÈÐW[Ô\€JÜÑ]°JÐ]Ä$ÀzÑB\ÐU[ÑB\Ñ]€EØ€Lr;   c           	      óT   — t        ddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	Nr“   r-  r=  rW   r>  r?  r)  )Úcait_s24_384rA  rB  s       r:   rS  rS  _  rQ  r;   c           	      óT   — t        ddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )	Nr“   r-  rH  rW   r˜   r?  r)  )Úcait_s36_384rA  rB  s       r:   rU  rU  f  rQ  r;   c           	      óT   — t        ddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )Nr“   r–   rH  r˜   r?  r)  )Úcait_m36_384rA  rB  s       r:   rW  rW  m  ó7   € ä ¨s¸"ÈÐX\Ô]€JÜÑ]°JÐ]Ä$ÀzÑB\ÐU[ÑB\Ñ]€EØ€Lr;   c           	      óT   — t        ddddd¬«      }t        dd| it        |fi |¤Ž¤Ž}|S )Nr“   r–   é0   r˜   r?  r)  )Úcait_m48_448rA  rB  s       r:   r[  r[  t  rX  r;   r�   r  )r¯   )5Ú__doc__Ú	functoolsr   Útypingr   r   r   r   r   r	   rH   Útorch.nnr.   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_manipulater   r   Ú	_registryr   r   Ú__all__r„   r   r   r   r   r   r   r+  r8  Údefault_cfgsr@  rF  rI  rK  rN  rP  rS  rU  rW  r[  rr   r;   r:   ú<module>rh     sf  ðñõ ß :× :ã Ý ç Aß PÕ PÝ *Ý +ß 3ß <â
a€ô3�—	‘	ô 3ôl0˜rŸy™yô 0ôf1�b—i‘iô 1ôh/�b—i‘iô /ôdvˆ2�9‰9ô vór ó
ó	ñ %Ù#'ØØ?Ø ô$ñ
 $(ØØ?ô$ñ $(ØØ?Ø ô$ñ
 $(ØØ?ô$ñ #'ØØ>ô#ñ "&ØØ=Ø ô"ñ
 "&ØØ=ô"ñ "&ØØ=ô"ñ "&ØØ=ô"ñ "&ØØ=Ø ô"ñQ-&ó -€ð` ñ°$ò ó ðð ñ°$ò ó ðð ñ°$ò ó ðð ñ°$ò ó ðð ñ°ò ó ðð ñ°ò ó ðð ñ°ò ó ðð ñ°ò ó ðð ñ°ò ó ðð ñ°ò ó ñr;   