Ë
    éÓ^j+A  ã                   óä  — d Z 	 ddlmZmZmZmZ ddlZddlmZ ddl	m
Z
mZ ddlmZmZmZmZmZmZmZ ddlmZ ddlmZ dd	lmZmZ d
gZe G d„ dej8                  «      «       Z G d„ dej8                  «      Z G d„ dej8                  «      Z G d„ d
ej8                  «      Z dd„Z!dd„Z" e e"d¬«       e"d¬«       e"d¬«      dœ«      Z#edde fd„«       Z$edde fd„«       Z%edde fd„«       Z&y)a   ConViT Model

@article{d2021convit,
  title={ConViT: Improving Vision Transformers with Soft Convolutional Inductive Biases},
  author={d'Ascoli, St{'e}phane and Touvron, Hugo and Leavitt, Matthew and Morcos, Ari and Biroli, Giulio and Sagun, Levent},
  journal={arXiv preprint arXiv:2103.10697},
  year={2021}
}

Paper link: https://arxiv.org/abs/2103.10697
Original code: https://github.com/facebookresearch/convit, original copyright below

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
é    )ÚOptionalÚUnionÚTypeÚAnyN©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚDropPathÚcalculate_drop_path_ratesÚtrunc_normal_Ú
PatchEmbedÚMlpÚ	LayerNormÚHybridEmbedé   )Úbuild_model_with_cfg)Úregister_notrace_module)Úregister_modelÚgenerate_default_cfgsÚConVitc                   ó†   ‡ — e Zd Z	 	 	 	 	 	 	 ddedededededefˆ fd„Zd„ Zd	„ Zdd
„Z	d„ Z
dedej                  fd„Zˆ xZS )ÚGPSAÚdimÚ	num_headsÚqkv_biasÚ	attn_dropÚ	proj_dropÚlocality_strengthc	                 ón  •— ||dœ}	t         ‰| �  «        || _        || _        ||z  }
|
dz  | _        || _        t        j                  ||dz  fd|i|	¤Ž| _        t        j                  ||fd|i|	¤Ž| _	        t        j                  |«      | _        t        j                  ||fi |	¤Ž| _        t        j                  d|fi |	¤Ž| _        t        j                  |«      | _        t        j                  t!        j"                  | j                  fi |	¤Ž«      | _        t!        j&                  di |	¤Ž| _        y )N©ÚdeviceÚdtypeç      à¿é   Úbiasé   )r   r   r   r&   )ÚsuperÚ__init__r   r   Úscaler   ÚnnÚLinearÚqkÚvÚDropoutr   ÚprojÚpos_projr   Ú	ParameterÚtorchÚonesÚgating_paramÚzerosÚrel_indices)Úselfr   r   r   r   r   r   r!   r"   ÚddÚhead_dimÚ	__class__s              €ú]/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/convit.pyr(   zGPSA.__init__)   s  ø€ ð ¨Ñ/ˆÜ‰ÑÔØ"ˆŒØˆŒØ˜)Ñ#ˆØ Ñ%ˆŒ
Ø!2ˆÔä—)‘)˜C  q¡Ñ>¨xÐ>¸2Ñ>ˆŒÜ—‘˜3 Ñ9¨(Ð9°bÑ9ˆŒäŸ™ IÓ.ˆŒÜ—I‘I˜c 3Ñ-¨"Ñ-ˆŒ	ÜŸ	™	 ! YÑ5°"Ñ5ˆŒÜŸ™ IÓ.ˆŒÜŸL™L¬¯©°D·N±NÑ)IÀbÑ)IÓJˆÔÜ).¯©Ñ)FÀ2Ñ)FˆÕó    c                 óì  — |j                   \  }}}| j                  �| j                  j                   d   |k7  r| j                  |«      | _        | j                  |«      }| j	                  |«      j                  ||| j                  || j                  z  «      j                  dddd«      }||z  j                  dd«      j                  |||«      }| j                  |«      }| j                  |«      }|S )Nr   r   r$   r&   )Úshaper6   Úget_rel_indicesÚget_attentionr-   Úreshaper   ÚpermuteÚ	transposer/   r   )r7   ÚxÚBÚNÚCÚattnr-   s          r;   ÚforwardzGPSA.forwardF   sÝ   € Ø—'‘'‰ˆˆ1ˆaØ×ÑÐ# t×'7Ñ'7×'=Ñ'=¸aÑ'@ÀAÒ'EØ#×3Ñ3°AÓ6ˆDÔØ×!Ñ! !Ó$ˆØ�F‰F�1‹I×Ñ˜a  D§N¡N°A¸¿¹Ñ4GÓH×PÑPÐQRÐTUÐWXÐZ[Ó\ˆØ�A‰X× Ñ   AÓ&×.Ñ.¨q°!°QÓ7ˆØ�I‰I�a‹LˆØ�N‰N˜1ÓˆØˆr<   c                 óò  — |j                   \  }}}| j                  |«      j                  ||d| j                  || j                  z  «      j	                  ddddd«      }|d   |d   }}| j
                  j                  |ddd«      }| j                  |«      j	                  dddd«      }||j                  dd«      z  | j                  z  }	|	j                  d¬«      }	|j                  d¬«      }| j                  j                  dddd«      }
d	t        j                  |
«      z
  |	z  t        j                  |
«      |z  z   }||j                  d¬«      j!                  d«      z  }| j#                  |«      }|S )
Nr$   r   r&   r   é   éÿÿÿÿéþÿÿÿ©r   ç      ð?)r>   r,   rA   r   rB   r6   Úexpandr0   rC   r)   Úsoftmaxr4   Úviewr2   ÚsigmoidÚsumÚ	unsqueezer   )r7   rD   rE   rF   rG   r,   ÚqÚkÚ	pos_scoreÚpatch_scoreÚgatingrH   s               r;   r@   zGPSA.get_attentionQ   sa  € Ø—'‘'‰ˆˆ1ˆaØ�W‰W�Q‹Z×Ñ  1 a¨¯©¸¸d¿n¹nÑ9LÓM×UÑUÐVWÐYZÐ\]Ð_`ÐbcÓdˆØ�!‰u�b˜‘eˆ1ˆØ×$Ñ$×+Ñ+¨A¨r°2°rÓ:ˆ	Ø—M‘M )Ó,×4Ñ4°Q¸¸1¸aÓ@ˆ	Ø˜1Ÿ;™; r¨2Ó.Ñ.°$·*±*Ñ<ˆØ!×)Ñ)¨bÐ)Ó1ˆØ×%Ñ%¨"Ð%Ó-ˆ	à×"Ñ"×'Ñ'¨¨2¨q°!Ó4ˆØ”U—]‘] 6Ó*Ñ*¨kÑ9¼E¿M¹MÈ&Ó<QÐT]Ñ<]Ñ]ˆØ�—‘˜R�Ó ×*Ñ*¨2Ó.Ñ.ˆØ�~‰~˜dÓ#ˆØˆr<   c                 óô   — | j                  |«      j                  d«      }| j                  j                  «       d d …d d …df   dz  }t	        j
                  d||f«      |j                  d«      z  }|r||fS |S )Nr   rL   ç      à?ú	nm,hnm->h)r@   Úmeanr6   Úsqueezer2   ÚeinsumÚsize)r7   rD   Ú
return_mapÚattn_mapÚ	distancesÚdists         r;   Úget_attention_mapzGPSA.get_attention_mapa   sw   € Ø×%Ñ% aÓ(×-Ñ-¨aÓ0ˆØ×$Ñ$×,Ñ,Ó.ªq²!°R¨xÑ8¸BÑ>ˆ	Ü�|‰|˜K¨)°XÐ)>Ó?À)Ç.Á.ÐQRÓBSÑSˆÙØ˜�>Ð!àˆKr<   c                 ó²  — | j                   j                  j                  j                  t	        j
                  | j                  «      «       d}t        | j                  dz  «      }|dz  dk(  r|dz
  dz  n|dz  }t        |«      D ]›  }t        |«      D ]‹  }|||z  z   }d| j                  j                  j                  |df<   d||z
  z  |z  | j                  j                  j                  |df<   d||z
  z  |z  | j                  j                  j                  |df<   Œ� Œ� | j                  j                  xj                  | j                  z  c_        y )Nr   r\   r$   r   rL   )r-   ÚweightÚdataÚcopy_r2   Úeyer   Úintr   Úranger0   r   )r7   Úlocality_distanceÚkernel_sizeÚcenterÚh1Úh2Úpositions          r;   Ú
local_initzGPSA.local_initj   s;  € Ø�‰�‰×Ñ× Ñ ¤§¡¨4¯8©8Ó!4Ô5ØÐä˜$Ÿ.™.¨BÑ.Ó/ˆØ*5¸©/¸QÒ*>�+ ‘/ QÒ&ÀKÐSTÑDTˆÜ˜Ó$ò 	_ˆBÜ˜KÓ(ò _�Ø ¨bÑ 0Ñ0�Ø9;�—‘×$Ñ$×)Ñ)¨(°A¨+Ñ6Ø9:¸bÀ6¹kÑ9JÐM^Ñ9^�—‘×$Ñ$×)Ñ)¨(°A¨+Ñ6Ø9:¸bÀ6¹kÑ9JÐM^Ñ9^�—‘×$Ñ$×)Ñ)¨(°A¨+Ò6ñ	_ð	_ð 	�‰×Ñ×!Ò! T×%;Ñ%;Ñ;Ö!r<   Únum_patchesÚreturnc                 ó   — t        |dz  «      }t        j                  d||d«      }t        j                  |t        j                  ¬«      j                  dd«      t        j                  |t        j                  ¬«      j                  dd«      z
  }|j                  ||«      }|j                  |d¬«      j                  |d¬«      }|dz  |dz  z   }|j                  d«      |d d …d d …d d …df<   |j                  d«      |d d …d d …d d …df<   |j                  d«      |d d …d d …d d …df<   | j                  j                  j                  }| j                  j                  j                  }	|j                  ||	¬	«      S )
Nr\   r   r&   ©r"   rL   r   rN   r$   r    )rl   r2   r5   ÚarangeÚfloat32rR   ÚrepeatÚrepeat_interleaverU   r,   rh   r!   r"   Úto)
r7   ru   Úimg_sizer6   ÚindÚindxÚindyÚinddr!   r"   s
             r;   r?   zGPSA.get_rel_indicesx   sB  € Ü�{ bÑ(Ó)ˆÜ—k‘k ! [°+¸qÓAˆä—‘˜X¬U¯]©]Ô;×@Ñ@ÀÀBÓGÜ—,‘,˜x¬u¯}©}Ô=×BÑBÀ2ÀqÓIñJð 	ð �z‰z˜( HÓ-ˆØ×$Ñ$ X°1Ð$Ó5×GÑGÈÐVWÐGÓXˆØ�q‰y˜4 1™9Ñ$ˆØ"&§.¡.°Ó"3ˆ’A’qš!˜Q�JÑØ"&§.¡.°Ó"3ˆ’A’qš!˜Q�JÑØ"&§.¡.°Ó"3ˆ’A’qš!˜Q�JÑØ—‘—‘×&Ñ&ˆØ—‘—‘×$Ñ$ˆØ�~‰~ V°5ˆ~Ó9Ð9r<   )é   Fç        r„   rO   NN©F)Ú__name__Ú
__module__Ú__qualname__rl   ÚboolÚfloatr(   rI   r@   rf   rt   r2   ÚTensorr?   Ú__classcell__©r:   s   @r;   r   r   '   s‹   ø„ ð
 Ø"Ø!Ø!Ø')ØØñGàðGð ðGð ð	Gð
 ðGð ðGð  %õGò:	òó ò<ð:¨3ð :°5·<±<÷ :r<   r   c                   óP   ‡ — e Zd Z	 	 	 	 	 	 d	dededededef
ˆ fd„Zd
d„Zd„ Zˆ xZ	S )ÚMHSAr   r   r   r   r   c                 ó:  •— ||dœ}t         ‰
| �  «        || _        ||z  }	|	dz  | _        t	        j
                  ||dz  fd|i|¤Ž| _        t	        j                  |«      | _        t	        j
                  ||fi |¤Ž| _	        t	        j                  |«      | _
        y )Nr    r#   r&   r%   )r'   r(   r   r)   r*   r+   Úqkvr.   r   r/   r   )r7   r   r   r   r   r   r!   r"   r8   r9   r:   s             €r;   r(   zMHSA.__init__‹   sŽ   ø€ ð ¨Ñ/ˆÜ‰ÑÔØ"ˆŒØ˜)Ñ#ˆØ Ñ%ˆŒ
ä—9‘9˜S #¨¡'Ñ?°Ð?¸BÑ?ˆŒÜŸ™ IÓ.ˆŒÜ—I‘I˜c 3Ñ-¨"Ñ-ˆŒ	ÜŸ™ IÓ.ˆ�r<   c                 ór  — |j                   \  }}}| j                  |«      j                  ||d| j                  || j                  z  «      j	                  ddddd«      }|d   |d   |d   }	}}||j                  dd«      z  | j                  z  }
|
j                  d¬«      j                  d«      }
t        |d	z  «      }t        j                  |t        j                  ¬
«      j                  dd«      t        j                  |t        j                  ¬
«      j                  dd«      z
  }|j                  ||«      }|j                  |d¬«      j                  |d¬«      }|dz  |dz  z   }|d	z  }|j!                  |
j"                  |
j$                  «      }t        j&                  d||
f«      |z  }|r||
fS |S )Nr&   r$   r   r   rK   rM   rL   rN   r\   rx   r]   )r>   r‘   rA   r   rB   rC   r)   rQ   r^   rl   r2   ry   rz   rR   r{   r|   r}   r!   r"   r`   )r7   rD   rb   rE   rF   rG   r‘   rV   rW   r-   rc   r~   r   r€   r�   r‚   rd   re   s                     r;   rf   zMHSA.get_attention_map    s˜  € Ø—'‘'‰ˆˆ1ˆaØ�h‰h�q‹k×!Ñ! ! Q¨¨4¯>©>¸1ÀÇÁÑ;NÓO×WÑWÐXYÐ[\Ð^_ÐabÐdeÓfˆØ�a‘&˜#˜a™& # a¡&ˆaˆ1ˆØ˜Ÿ™ B¨Ó+Ñ+¨t¯z©zÑ9ˆØ×#Ñ#¨Ð#Ó+×0Ñ0°Ó3ˆä�q˜B‘w“<ˆä—‘˜X¬U¯]©]Ô;×@Ñ@ÀÀBÓGÜ—,‘,˜x¬u¯}©}Ô=×BÑBÀ2ÀqÓIñJð 	ð �z‰z˜( HÓ-ˆØ×$Ñ$ X°1Ð$Ó5×GÑGÈÐVWÐGÓXˆØ�q‰y˜4 1™9Ñ$ˆØ˜B‘Jˆ	Ø—L‘L §¡°(·.±.ÓAˆ	ä�|‰|˜K¨)°XÐ)>Ó?À!ÑCˆÙØ˜�>Ð!àˆKr<   c                 ó  — |j                   \  }}}| j                  |«      j                  ||d| j                  || j                  z  «      j	                  ddddd«      }|j                  d«      \  }}}||j                  dd«      z  | j                  z  }	|	j                  d¬«      }	| j                  |	«      }	|	|z  j                  dd«      j                  |||«      }| j                  |«      }| j                  |«      }|S )	Nr&   r$   r   r   rK   rM   rL   rN   )r>   r‘   rA   r   rB   ÚunbindrC   r)   rQ   r   r/   r   )
r7   rD   rE   rF   rG   r‘   rV   rW   r-   rH   s
             r;   rI   zMHSA.forward¸   sê   € Ø—'‘'‰ˆˆ1ˆaØ�h‰h�q‹k×!Ñ! ! Q¨¨4¯>©>¸1ÀÇÁÑ;NÓO×WÑWÐXYÐ[\Ð^_ÐabÐdeÓfˆØ—*‘*˜Q“-‰ˆˆ1ˆaà�A—K‘K  BÓ'Ñ'¨4¯:©:Ñ5ˆØ�|‰| ˆ|Ó#ˆØ�~‰~˜dÓ#ˆà�A‰X× Ñ   AÓ&×.Ñ.¨q°!°QÓ7ˆØ�I‰I�a‹LˆØ�N‰N˜1ÓˆØˆr<   )rƒ   Fr„   r„   NNr…   )
r†   r‡   rˆ   rl   r‰   rŠ   r(   rf   rI   rŒ   r�   s   @r;   r�   r�   Š   sV   ø„ ð Ø"Ø!Ø!ØØñ/àð/ð ð/ð ð	/ð
 ð/ð õ/ó*ö0r<   r�   c                   ó²   ‡ — e Zd Zdddddej                  eddddfdeded	ed
edededede	ej                     de	ej                     dedefˆ fd„Zd„ Zˆ xZS )ÚBlockç      @Fr„   TrO   Nr   r   Ú	mlp_ratior   r   r   Ú	drop_pathÚ	act_layerÚ
norm_layerÚuse_gpsar   c           	      ó~  •— ||dœ}t         ‰| �  «         |	|fi |¤Ž| _        |
| _        | j                  rt	        |f|||||dœ|¤Ž| _        nt        |f||||dœ|¤Ž| _        |dkD  rt        |«      nt        j                  «       | _
         |	|fi |¤Ž| _        t        ||z  «      }t        d||||dœ|¤Ž| _        y )Nr    )r   r   r   r   r   )r   r   r   r   r„   )Úin_featuresÚhidden_featuresrš   Údrop© )r'   r(   Únorm1rœ   r   rH   r�   r
   r*   ÚIdentityr™   Únorm2rl   r   Úmlp)r7   r   r   r˜   r   r   r   r™   rš   r›   rœ   r   r!   r"   r8   Úmlp_hidden_dimr:   s                   €r;   r(   zBlock.__init__É   sô   ø€ ð  ¨Ñ/ˆÜ‰ÑÔÙ Ñ* rÑ*ˆŒ
Ø ˆŒØ�=Š=ÜØðà#Ø!Ø#Ø#Ø"3ñð ñˆD�Iô Øðà#Ø!Ø#Ø#ñð ñˆDŒIð 1:¸B²œ )Ô,ÄBÇKÁKÃMˆŒÙ Ñ* rÑ*ˆŒ
Ü˜S 9™_Ó-ˆÜð 
ØØ*ØØñ	
ð
 ñ
ˆ�r<   c                 óÎ   — || j                  | j                  | j                  |«      «      «      z   }|| j                  | j                  | j	                  |«      «      «      z   }|S ©N)r™   rH   r¢   r¥   r¤   ©r7   rD   s     r;   rI   zBlock.forwardû   sO   € Ø�—‘˜tŸy™y¨¯©°A«Ó7Ó8Ñ8ˆØ�—‘˜tŸx™x¨¯
©
°1«Ó6Ó7Ñ7ˆØˆr<   )r†   r‡   rˆ   r*   ÚGELUr   rl   rŠ   r‰   r   ÚModuler(   rI   rŒ   r�   s   @r;   r–   r–   Ç   s±   ø„ ð  "Ø"Ø!Ø!Ø!Ø)+¯©Ø*3Ø!Ø')ØØñ0
àð0
ð ð0
ð ð	0
ð
 ð0
ð ð0
ð ð0
ð ð0
ð ˜BŸI™I‘ð0
ð ˜RŸY™Y™ð0
ð ð0
ð  %õ0
ödr<   r–   c            )       óø  ‡ — e Zd ZdZddddddddd	d
ddddddedddddfdededededededededededededededede	e
   deej                     d ed!ed"ef(ˆ fd#„Zd$„ Zej"                  j$                  d%„ «       Zej"                  j$                  d/d&„«       Zej"                  j$                  d0d'„«       Zej"                  j$                  d(ej                  fd)„«       Zd1dede	e   fd*„Zd+„ Zd/d,efd-„Zd.„ Zˆ xZS )2r   zI Vision Transformer with support for patch or hybrid CNN input stage
    éà   é   r&   éè  Útokeni   é   r—   Fr„   NrO   Tr~   Ú
patch_sizeÚin_chansÚnum_classesÚglobal_poolÚ	embed_dimÚdepthr   r˜   r   Ú	drop_rateÚpos_drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_rateÚhybrid_backboner›   Úlocal_up_to_layerr   Úuse_pos_embedc                 óø  •— t         ‰| �  «        ||dœ}|dv sJ ‚||z  }|| _        || _        || _        || _        |x| _        x| _        | _        || _	        || _
        |�t        |f|||dœ|¤Ž| _        nt        d||||dœ|¤Ž| _        | j                  j                  }|| _        t        j                   t#        j$                  dd|fi |¤Ž«      | _        t        j(                  |¬«      | _        | j                  rGt        j                   t#        j$                  d||fi |¤Ž«      | _        t/        | j,                  d¬«       t1        ||«      }t        j2                  t5        |«      D �cg c]  }t7        d|||	|
||||   |||k  |d	œ
|¤Ž‘Œ  c}«      | _         ||fi |¤Ž| _        t=        |d
d¬«      g| _        t        j(                  |«      | _         |d
kD  rt        jB                  ||fi |¤Žnt        jD                  «       | _#        t/        | j&                  d¬«       | jI                  | jJ                  «       | jM                  «       D ]"  \  }}tO        |d«      sŒ|jQ                  «        Œ$ y c c}w )Nr    )Ú Úavgr°   )r~   r³   r¶   )r~   r²   r³   r¶   r   )Úpç{®Gáz”?©Ústd)
r   r   r˜   r   r   r   r™   r›   rœ   r   r   Úhead)Únum_chsÚ	reductionÚmodulert   r¡   ))r'   r(   r´   r³   rµ   r¾   Únum_featuresÚhead_hidden_sizer¶   r   r¿   r   Úpatch_embedr   ru   r*   r1   r2   r5   Ú	cls_tokenr.   Úpos_dropÚ	pos_embedr   r   Ú
ModuleListrm   r–   ÚblocksÚnormÚdictÚfeature_infoÚ	head_dropr+   r£   rÇ   ÚapplyÚ_init_weightsÚnamed_modulesÚhasattrrt   )r7   r~   r²   r³   r´   rµ   r¶   r·   r   r˜   r   r¸   r¹   rº   r»   r¼   r½   r›   r¾   r   r¿   r!   r"   r8   ru   ÚdprÚiÚnÚmr:   s                                €r;   r(   zConVit.__init__  s„  ø€ ô2 	‰ÑÔØ¨Ñ/ˆØÐ2Ñ2Ð2Ð2Ø�YÑˆ	Ø&ˆÔØ ˆŒØ&ˆÔØ!2ˆÔØENÐNˆÔÐN˜DÔ1°D´NØ!2ˆÔØ*ˆÔàÐ&Ü*Øð à!Ø!Ø#ñ	 ð
 ñ ˆDÕô  *ð  Ø!Ø%Ø!Ø#ñ	 ð
 ñ ˆDÔð ×&Ñ&×2Ñ2ˆØ&ˆÔäŸ™¤e§k¡k°!°Q¸	Ñ&HÀRÑ&HÓIˆŒÜŸ
™
 ]Ô3ˆŒà×ÒÜŸ\™\¬%¯+©+°a¸ÀiÑ*VÐSUÑ*VÓWˆDŒNÜ˜$Ÿ.™.¨cÕ2ä'¨¸Ó>ˆÜ—m‘mô ˜U“|ö%%ð ô ð ØØ#Ø#Ø!Ø(Ø(Ø˜a™&Ø%ØÐ.Ñ.Ø"3ñð óò%%ó &ˆŒñ ˜yÑ/¨BÑ/ˆŒ	ô "¨)¸qÈÔPÐQˆÔÜŸ™ IÓ.ˆŒØ?JÈQº”B—I‘I˜i¨Ñ;¸Ò;ÔTV×T_ÑT_ÓTaˆŒ	ä�d—n‘n¨#Õ.Ø�
‰
�4×%Ñ%Ô&Ø×&Ñ&Ó(ò 	‰DˆAˆqÜ�q˜,Õ'Ø—‘•ñ	ùò/%%s   Å/#I7c                 óê  — t        |t        j                  «      rjt        |j                  d¬«       t        |t        j                  «      r8|j
                  �+t        j                  j                  |j
                  d«       y y y t        |t        j                  «      rUt        j                  j                  |j
                  d«       t        j                  j                  |j                  d«       y y )NrÄ   rÅ   r   rO   )	Ú
isinstancer*   r+   r   rh   r%   ÚinitÚ	constant_r   )r7   rÞ   s     r;   rØ   zConVit._init_weights`  s˜   € Ü�aœŸ™Ô#Ü˜!Ÿ(™(¨Õ,Ü˜!œRŸY™YÔ'¨A¯F©FÐ,>Ü—‘×!Ñ! !§&¡&¨!Õ,ð -?Ð'ä˜œ2Ÿ<™<Ô(Ü�G‰G×Ñ˜aŸf™f aÔ(Ü�G‰G×Ñ˜aŸh™h¨Õ,ð )r<   c                 ó
   — ddhS )NrÐ   rÎ   r¡   ©r7   s    r;   Úno_weight_decayzConVit.no_weight_decayi  s   € à˜[Ð)Ð)r<   c                 ó    — t        dddg¬«      S )Nz ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)iŸ† )ÚstemrÒ   )rÔ   )r7   Úcoarses     r;   Úgroup_matcherzConVit.group_matcherm  s   € äØ4Ø-Ð/CÐDô
ð 	
r<   c                 ó   — |rJ d«       ‚y )Nz$gradient checkpointing not supportedr¡   )r7   Úenables     r;   Úset_grad_checkpointingzConVit.set_grad_checkpointingt  s   € áÐAÐAÓAˆz�6r<   rv   c                 ó   — | j                   S r¨   )rÇ   rä   s    r;   Úget_classifierzConVit.get_classifierx  s   € à�y‰yÐr<   c                 ó¸   — || _         |�|dv sJ ‚|| _        |dkD  r&t        j                  | j                  |«      | _        y t        j
                  «       | _        y )N)rÁ   r°   rÂ   r   )r´   rµ   r*   r+   r¶   r£   rÇ   )r7   r´   rµ   s      r;   Úreset_classifierzConVit.reset_classifier|  sS   € Ø&ˆÔØÐ"ØÐ"6Ñ6Ð6Ð6Ø*ˆDÔØ>IÈAºo”B—I‘I˜dŸn™n¨kÓ:ˆ�	ÔSU×S^ÑS^ÓS`ˆ�	r<   c                 ó�  — | j                  |«      }| j                  r|| j                  z   }| j                  |«      }| j                  j                  |j                  d   dd«      }t        | j                  «      D ]5  \  }}|| j                  k(  rt        j                  ||fd¬«      } ||«      }Œ7 | j                  |«      }|S )Nr   rL   r   rN   )rÍ   r¿   rÐ   rÏ   rÎ   rP   r>   Ú	enumeraterÒ   r¾   r2   ÚcatrÓ   )r7   rD   Ú
cls_tokensÚuÚblks        r;   Úforward_featureszConVit.forward_featuresƒ  s²   € Ø×Ñ˜QÓˆØ×ÒØ�D—N‘NÑ"ˆAØ�M‰M˜!ÓˆØ—^‘^×*Ñ*¨1¯7©7°1©:°r¸2Ó>ˆ
Ü §¡Ó,ò 	‰FˆAˆsØ�D×*Ñ*Ò*Ü—I‘I˜z¨1˜o°1Ô5�Ù�A“‰Að	ð �I‰I�a‹LˆØˆr<   Ú
pre_logitsc                 óÌ   — | j                   r3| j                   dk(  r|d d …dd …f   j                  d¬«      n|d d …df   }| j                  |«      }|r|S | j                  |«      S )NrÂ   r   rN   r   )rµ   r^   rÖ   rÇ   )r7   rD   rø   s      r;   Úforward_headzConVit.forward_head�  s_   € Ø×ÒØ(,×(8Ñ(8¸EÒ(A�’!�Q‘R�%‘—‘ !�Ô$ÀqÊÈAÈÁwˆAØ�N‰N˜1ÓˆÙˆqÐ0 D§I¡I¨a£LÐ0r<   c                 óJ   — | j                  |«      }| j                  |«      }|S r¨   )r÷   rú   r©   s     r;   rI   zConVit.forward–  s'   € Ø×!Ñ! !Ó$ˆØ×Ñ˜aÓ ˆØˆr<   r…   )Tr¨   )r†   r‡   rˆ   Ú__doc__r   rl   ÚstrrŠ   r‰   r   r   r   r*   r«   r(   rØ   r2   ÚjitÚignorerå   ré   rì   rî   rð   r÷   rú   rI   rŒ   r�   s   @r;   r   r     só  ø„ ñð
  Ø ØØ#Ø&Ø ØØØ!Ø"Ø!Ø#%Ø$&Ø$&Ø$&Ø-1Ø*3Ø%&Ø')Ø"&ØØñ/YàðYð ðYð ð	Yð
 ðYð ðYð ðYð ðYð ðYð ðYð ðYð ðYð !ðYð "ðYð "ðYð  "ð!Yð" & c™]ð#Yð$ ˜RŸY™Y™ð%Yð&  #ð'Yð(  %ð)Yð*  õ+Yòv-ð ‡Y�Y×Ññ*ó ð*ð ‡Y�Y×Ñò
ó ð
ð ‡Y�Y×ÑòBó ðBð ‡Y�Y×Ñð §	¡	ò ó ðña¨Cð a¸hÀs¹mó aòñ1¨$ó 1ör<   c                 ó`   — |j                  dd «      rt        d«      ‚t        t        | |fi |¤ŽS )NÚfeatures_onlyz<features_only not implemented for Vision Transformer models.)ÚgetÚRuntimeErrorr   r   )ÚvariantÚ
pretrainedÚkwargss      r;   Ú_create_convitr  œ  s0   € Ø‡z�z�/ 4Ô(ÜÐYÓZÐZä¤¨°ÑF¸vÑFÐFr<   c                 ó0   — | ddd t         t        dddddœ
|¥S )Nr¯   )r&   r­   r­   Tzpatch_embed.projrÇ   z
apache-2.0)
Úurlr´   Ú
input_sizeÚ	pool_sizer^   rÆ   Úfixed_input_sizeÚ
first_convÚ
classifierÚlicenser   )r	  r  s     r;   Ú_cfgr  £  s0   € àØ¨=ÀtÜ%Ô.BÐX\Ø(¸È<ñ	ð
 ðð r<   ztimm/)Ú	hf_hub_id)zconvit_tiny.fb_in1kzconvit_small.fb_in1kzconvit_base.fb_in1krv   c           	      óT   — t        dddd¬«      }t        dd| dœt        |fi |¤Ž¤Ž}|S )	Né
   rO   é0   rK   ©r¾   r   r¶   r   Úconvit_tiny©r  r  r¡   ©rÔ   r  ©r  r  Ú
model_argsÚmodels       r;   r  r  µ  s;   € äØ°¸rÈQôP€JäÐf =¸ZÑfÌ4ÐPZÑKeÐ^dÑKeÑf€EØ€Lr<   c           	      óT   — t        dddd¬«      }t        dd| dœt        |fi |¤Ž¤Ž}|S )	Nr  rO   r  é	   r  Úconvit_smallr  r¡   r  r  s       r;   r  r  ½  s;   € äØ°¸rÈQôP€JäÐg >¸jÑgÌDÐQ[ÑLfÐ_eÑLfÑg€EØ€Lr<   c           	      óT   — t        dddd¬«      }t        dd| dœt        |fi |¤Ž¤Ž}|S )	Nr  rO   r  r®   r  Úconvit_baser  r¡   r  r  s       r;   r   r   Å  s;   € äØ°¸rÈRôQ€JäÐf =¸ZÑfÌ4ÐPZÑKeÐ^dÑKeÑf€EØ€Lr<   r…   )rÁ   )'rü   Útypingr   r   r   r   r2   Útorch.nnr*   Ú	timm.datar   r	   Útimm.layersr
   r   r   r   r   r   r   Ú_builderr   Ú_features_fxr   Ú	_registryr   r   Ú__all__r«   r   r�   r–   r   r  r  Údefault_cfgsr  r  r   r¡   r<   r;   ú<module>r*     s  ðñð(÷ .Ó -ã Ý ç Aß s× sÑ sÝ *Ý 1ß <ð ˆ*€ð ô_:ˆ2�9‰9ó _:ó ð_:ôD:ˆ2�9‰9ô :ôz7ˆB�I‰Iô 7ôtXˆR�Y‰Yô XóvGóñ %á¨'Ô2Ù ¨7Ô3Ù¨'Ô2ñ	&ó €ð ñ¨vò ó ðð ñ°ò ó ðð ñ¨vò ó ñr<   