Ë
    éÓ^j[N  ã            	       ó¨  — d Z ddlmZmZmZmZmZmZmZm	Z	 ddl
Z
ddlmZ ddlmc mZ ddlmZmZ ddlmZmZmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lm Z m!Z! dgZ" G d„ dejF                  «      Z$ G d„ dejJ                  «      Z& G d„ dejJ                  «      Z' G d„ dejF                  «      Z( G d„ dejF                  «      Z) G d„ dejF                  «      Z* G d„ dejF                  «      Z+ G d„ dejF                  «      Z, G d„ dejF                  «      Z-dee.e
j^                  f   dejF                  dee.e
j^                  f   fd „Z0d0d!e.d"edee.ef   fd#„Z1 e! e1d$¬%«       e1d$¬%«       e1d$¬%«       e1d$d&¬'«      d(œ«      Z2d1d)e.d*e3d"ede-fd+„Z4e d1d*e3d"ede-fd,„«       Z5e d1d*e3d"ede-fd-„«       Z6e d1d*e3d"ede-fd.„«       Z7e d1d*e3d"ede-fd/„«       Z8y)2aÇ  SHViT
SHViT: Single-Head Vision Transformer with Memory Efficient Macro Design
Code: https://github.com/ysj9909/SHViT
Paper: https://arxiv.org/abs/2401.16456

@inproceedings{yun2024shvit,
  author={Yun, Seokju and Ro, Youngmin},
  title={SHViT: Single-Head Vision Transformer with Memory Efficient Macro Design},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  pages={5756--5767},
  year={2024}
}
é    )ÚAnyÚDictÚListÚOptionalÚSetÚTupleÚTypeÚUnionN©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Ú
GroupNorm1ÚSqueezeExciteÚSelectAdaptivePool2dÚ	LayerTypeÚtrunc_normal_é   )Úbuild_model_with_cfg)Úfeature_take_indices)Úcheckpoint_seq)Úregister_modelÚgenerate_default_cfgsÚSHViTc                   ó¸   ‡ — e Zd Zdej                  fˆ fd„Zdej                  dej                  fd„Z ej                  «       dej                  fd„«       Z
ˆ xZS )ÚResidualÚmc                 ó0   •— t         ‰| �  «        || _        y ©N)ÚsuperÚ__init__r   )Úselfr   Ú	__class__s     €ú\/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/shvit.pyr    zResidual.__init__   s   ø€ Ü‰ÑÔØˆ�ó    ÚxÚreturnc                 ó*   — || j                  |«      z   S r   )r   ©r!   r%   s     r#   ÚforwardzResidual.forward#   s   € Ø�4—6‘6˜!“9‰}Ðr$   c                 óÔ  — t        | j                  t        «      rÍ| j                  j                  «       }|j                  |j
                  k(  sJ ‚t        j                  |j                  j                  d   |j                  j                  d   dd«      }t        j                  |g d¢«      }|xj                  |j                  |j                  j                  «      z  c_        |S | S )Nr   r   )r   r   r   r   )Ú
isinstancer   Ú
Conv2dNormÚfuseÚgroupsÚin_channelsÚtorchÚonesÚweightÚshapeÚFÚpadÚtoÚdevice)r!   r   Úidentitys      r#   r-   zResidual.fuse&   sœ   € ä�d—f‘fœjÔ)Ø—‘—‘“ˆAØ—8‘8˜qŸ}™}Ò,Ð-Ð,Ü—z‘z !§(¡(§.¡.°Ñ"3°Q·X±X·^±^ÀAÑ5FÈÈ1ÓMˆHÜ—u‘u˜X¢yÓ1ˆHØ�HŠH˜Ÿ™ A§H¡H§O¡OÓ4Ñ4�HØˆHàˆKr$   )Ú__name__Ú
__module__Ú__qualname__ÚnnÚModuler    r0   ÚTensorr)   Úno_gradr-   Ú__classcell__©r"   s   @r#   r   r      sU   ø„ ð˜"Ÿ)™)õ ð˜Ÿ™ð ¨%¯,©,ó ð €U‡]�]ƒ_ð	�b—i‘iò 	ó ô	r$   r   c                   óŽ   ‡ — e Zd Z	 	 	 	 	 	 d
dedededededefˆ fd„Z ej                  «       dej                  fd	„«       Z	ˆ xZ
S )r,   r/   Úout_channelsÚkernel_sizeÚstrideÚpaddingÚbn_weight_initc	           
      ó¤  •— ||dœ}
t         ‰| �  «        | j                  dt        j                  |||||fddi|
¤|	¤Ž«       | j                  dt        j
                  |fi |
¤Ž«       t        j                  j                  | j                  j                  |«       t        j                  j                  | j                  j                  d«       y )N©r7   ÚdtypeÚcÚbiasFÚbnr   )r   r    Ú
add_moduler<   ÚConv2dÚBatchNorm2dÚinitÚ	constant_rM   r2   rL   )r!   r/   rC   rD   rE   rF   rG   r7   rJ   ÚkwargsÚddr"   s              €r#   r    zConv2dNorm.__init__4   s¯   ø€ ð ¨Ñ/ˆÜ‰ÑÔØ�‰˜œRŸY™YØ˜ {°F¸GñaØJOðaØSUðaØY_ñaô 	bà�‰˜œbŸn™n¨\Ñ@¸RÑ@ÔAÜ
�‰×Ñ˜$Ÿ'™'Ÿ.™.¨.Ô9Ü
�‰×Ñ˜$Ÿ'™'Ÿ,™,¨Õ*r$   r&   c                 óŒ  — | j                   j                  «       \  }}|j                  |j                  |j                  z   dz  z  }|j                  |d d …d d d f   z  }|j
                  |j                  |j                  z  |j                  |j                  z   dz  z  z
  }t        j                  |j                  d«      | j                  j                  z  |j                  d«      |j                  dd  | j                  j                  | j                  j                  | j                  j                  | j                  j                  |j                  j                   |j                  j"                  ¬«	      }|j                  j$                  j'                  |«       |j
                  j$                  j'                  |«       |S )Nç      à?r   r   é   )	r/   rC   rD   rE   rF   Údilationr.   r7   rJ   )Ú_modulesÚvaluesr2   Úrunning_varÚepsrL   Úrunning_meanr<   rO   ÚsizerK   r.   r3   rE   rF   rX   r7   rJ   ÚdataÚcopy_)r!   rK   rM   ÚwÚbr   s         r#   r-   zConv2dNorm.fuseH   s:  € à—‘×$Ñ$Ó&‰ˆˆ2Ø�I‰I˜Ÿ™¨"¯&©&Ñ0°SÑ8Ñ8ˆØ�H‰H�qš˜D $¨Ð,Ñ-Ñ-ˆØ�G‰G�b—o‘o¨¯	©	Ñ1°R·^±^ÀbÇfÁfÑ5LÐQTÑ4TÑTÑTˆÜ�I‰IØŸ™˜q›	 D§F¡F§M¡MÑ1ØŸ™ ›ØŸ™  ˜Ø—6‘6—=‘=Ø—F‘F—N‘NØ—V‘V—_‘_Ø—6‘6—=‘=Ø—8‘8—?‘?Ø—(‘(—.‘.ô

ˆð 	
�‰�‰×Ñ˜AÔØ	�‰�‰×Ñ˜!ÔØˆr$   )r   r   r   r   NN)r9   r:   r;   Úintr    r0   r?   r<   rO   r-   r@   rA   s   @r#   r,   r,   3   s}   ø„ ð
  !ØØØ"#ØØñ+àð+ð ð+ð ð	+ð
 ð+ð ð+ð  õ+ð( €U‡]�]ƒ_ð�b—i‘iò ó ôr$   r,   c            	       ó‚   ‡ — e Zd Z	 	 	 	 ddedededefˆ fd„Z ej                  «       de	j                  fd„«       Zˆ xZS )	Ú
NormLinearÚin_featuresÚout_featuresrL   Ústdc                 óz  •— ||dœ}t         ‰| �  «        | j                  dt        j                  |fi |¤Ž«       | j                  dt        j
                  ||fd|i|¤Ž«       t        | j                  j                  |¬«       |r5t        j                  j                  | j                  j                  d«       y y )NrI   rM   ÚlrL   )rh   r   )r   r    rN   r<   ÚBatchNorm1dÚLinearr   rj   r2   rQ   rR   rL   )	r!   rf   rg   rL   rh   r7   rJ   rT   r"   s	           €r#   r    zNormLinear.__init___   s�   ø€ ð ¨Ñ/ˆÜ‰ÑÔØ�‰˜œbŸn™n¨[Ñ?¸BÑ?Ô@Ø�‰˜œRŸY™Y {°LÑRÀtÐRÈrÑRÔSÜ�d—f‘f—m‘m¨Õ-ÙÜ�G‰G×Ñ˜dŸf™fŸk™k¨1Õ-ð r$   r&   c                 ó–  — | j                   j                  «       \  }}|j                  |j                  |j                  z   dz  z  }|j
                  | j                  j                  | j                  j                  z  |j                  |j                  z   dz  z  z
  }|j                  |d d d …f   z  }|j
                  €$|| j                  j                  j                  z  }n<|j                  |d d …d f   z  j                  d«      | j                  j
                  z   }t        j                  |j                  d«      |j                  d«      |j                  j                  |j                  j                  ¬«      }|j                  j                   j#                  |«       |j
                  j                   j#                  |«       |S )NrV   éÿÿÿÿr   r   rI   )rY   rZ   r2   r[   r\   rL   rM   r]   rj   ÚTÚviewr<   rl   r^   r7   rJ   r_   r`   )r!   rM   rj   ra   rb   r   s         r#   r-   zNormLinear.fusep   s@  € à—‘×$Ñ$Ó&‰ˆˆAØ�I‰I˜Ÿ™¨"¯&©&Ñ0°SÑ8Ñ8ˆØ�G‰G�d—g‘g×*Ñ*¨T¯W©W¯^©^Ñ;¸r¿~¹~ÐPR×PVÑPVÑ?VÐ[^Ñ>^Ñ^Ñ^ˆØ�H‰H�q˜šq˜‘zÑ!ˆØ�6‰6ˆ>Ø�D—F‘F—M‘M—O‘OÑ#‰Aà—‘˜Aša ˜g™JÑ&×,Ñ,¨RÓ0°4·6±6·;±;Ñ>ˆAÜ�I‰I�a—f‘f˜Q“i §¡¨£°1·8±8·?±?È!Ï(É(Ï.É.ÔYˆØ	�‰�‰×Ñ˜AÔØ	�‰�‰×Ñ˜!ÔØˆr$   )Tg{®Gáz”?NN)r9   r:   r;   rc   ÚboolÚfloatr    r0   r?   r<   rl   r-   r@   rA   s   @r#   re   re   ^   sc   ø„ ð
 ØØØñ.àð.ð ð.ð ð	.ð
 õ.ð" €U‡]�]ƒ_ð�b—i‘iò ó ôr$   re   c                   óš   ‡ — e Zd Zej                  ddfdededeej                     fˆ fd„Zde	j                  de	j                  fd„Zˆ xZS )	ÚPatchMergingNÚdimÚout_dimÚ	act_layerc                 ó  •— ||dœ}t         ‰| �  «        t        |dz  «      }t        ||fi |¤Ž| _         |«       | _        t        ||dddfd|i|¤Ž| _         |«       | _        t        |dfi |¤Ž| _	        t        ||fi |¤Ž| _
        y )NrI   é   é   rW   r   r.   g      Ð?)r   r    rc   r,   Úconv1Úact1Úconv2Úact2r   ÚseÚconv3)	r!   ru   rv   rw   r7   rJ   rT   Úhid_dimr"   s	           €r#   r    zPatchMerging.__init__�   s�   ø€ ð ¨Ñ/ˆÜ‰ÑÔÜ�c˜A‘g“,ˆÜ  WÑ3°Ñ3ˆŒ
Ù“KˆŒ	Ü ¨°!°Q¸ÑPÀ'ÐPÈRÑPˆŒ
Ù“KˆŒ	Ü ¨Ñ4°Ñ4ˆŒÜ ¨Ñ7°BÑ7ˆ�
r$   r%   r&   c                 óÒ   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j	                  |«      }| j                  |«      }|S r   )r{   r|   r}   r~   r   r€   r(   s     r#   r)   zPatchMerging.forward“   sU   € Ø�J‰J�q‹MˆØ�I‰I�a‹LˆØ�J‰J�q‹MˆØ�I‰I�a‹LˆØ�G‰G�A‹JˆØ�J‰J�q‹MˆØˆr$   ©r9   r:   r;   r<   ÚReLUrc   r	   r=   r    r0   r>   r)   r@   rA   s   @r#   rt   rt   €   sV   ø„ ð
 *,¯©ØØñ8àð8ð ð8ð ˜BŸI™I‘õ	8ð$˜Ÿ™ð ¨%¯,©,÷ r$   rt   c                   óš   ‡ — e Zd Zej                  ddfdededeej                     fˆ fd„Zde	j                  de	j                  fd„Zˆ xZS )	ÚFFNNru   Ú	embed_dimrw   c                 ó�   •— ||dœ}t         ‰| �  «        t        ||fi |¤Ž| _         |«       | _        t        ||fddi|¤Ž| _        y )NrI   rG   r   )r   r    r,   Úpw1ÚactÚpw2)r!   ru   r‡   rw   r7   rJ   rT   r"   s          €r#   r    zFFN.__init__ž   sN   ø€ ð ¨Ñ/ˆÜ‰ÑÔÜ˜c 9Ñ3°Ñ3ˆŒÙ“;ˆŒÜ˜i¨ÑE¸QÐEÀ"ÑEˆ�r$   r%   r&   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r   )r‰   rŠ   r‹   r(   s     r#   r)   zFFN.forward¬   s.   € Ø�H‰H�Q‹KˆØ�H‰H�Q‹KˆØ�H‰H�Q‹KˆØˆr$   rƒ   rA   s   @r#   r†   r†   �   sZ   ø„ ð
 *,¯©ØØñFàðFð ðFð ˜BŸI™I‘õ	Fð˜Ÿ™ð ¨%¯,©,÷ r$   r†   c                   óÂ   ‡ — e Zd ZdZeej                  ddfdedededeej                     deej                     f
ˆ fd„Z
d	ej                  d
ej                  fd„Zˆ xZS )ÚSHSAzSingle-Head Self-AttentionNru   Úqk_dimÚpdimÚ
norm_layerrw   c                 ó  •— ||dœ}t         ‰	| �  «        |dz  | _        || _        || _        || _         ||fi |¤Ž| _        t        ||dz  |z   fi |¤Ž| _        t        j                   |«       t        ||fddi|¤Ž«      | _        y )NrI   g      à¿rW   rG   r   )r   r    Úscaler�   ru   r�   Úpre_normr,   Úqkvr<   Ú
SequentialÚproj)
r!   ru   r�   r�   r‘   rw   r7   rJ   rT   r"   s
            €r#   r    zSHSA.__init__µ   sŽ   ø€ ð ¨Ñ/ˆÜ‰ÑÔØ˜t‘^ˆŒ
ØˆŒØˆŒØˆŒ	á" 4Ñ.¨2Ñ.ˆŒä˜d F¨Q¡J°Ñ$5Ñ<¸Ñ<ˆŒÜ—M‘M¡)£+¬z¸#¸sÑ/[ÐSTÐ/[ÐXZÑ/[Ó\ˆ�	r$   r%   r&   c                 óâ  — |j                   \  }}}}t        j                  || j                  | j                  | j                  z
  gd¬«      \  }}| j                  |«      }| j                  |«      }t        j                  || j                  | j                  | j                  gd¬«      \  }	}
}|	j                  d«      |
j                  d«      |j                  d«      }}
}	|	j                  dd«      |
z  | j                  z  }|j                  d¬«      }||j                  dd«      z  j                  || j                  ||«      }| j                  t        j                  ||gd¬«      «      }|S )Nr   )ru   rW   éþÿÿÿrn   )r3   r0   Úsplitr�   ru   r”   r•   r�   ÚflattenÚ	transposer“   ÚsoftmaxÚreshaper—   Úcat)r!   r%   ÚBÚ_ÚHÚWÚx1Úx2r•   ÚqÚkÚvÚattns                r#   r)   zSHSA.forwardË   s$  € Ø—W‘W‰
ˆˆ1ˆa�Ü—‘˜Q §¡¨D¯H©H°t·y±yÑ,@Ð AÈÔK‰ˆˆBØ�]‰]˜2ÓˆØ�h‰h�r‹lˆÜ—+‘+˜c D§K¡K°·±¸d¿i¹iÐ#HÈaÔP‰ˆˆ1ˆaØ—)‘)˜A“, §	¡	¨!£¨a¯i©i¸«lˆaˆ1ˆà—‘˜B Ó# aÑ'¨4¯:©:Ñ5ˆØ�|‰| ˆ|Ó#ˆØ�$—.‘.  RÓ(Ñ(×1Ñ1°!°T·Y±YÀÀ1ÓEˆØ�I‰I”e—i‘i  R °Ô2Ó3ˆØˆr$   )r9   r:   r;   Ú__doc__r   r<   r„   rc   r	   r=   r    r0   r>   r)   r@   rA   s   @r#   rŽ   rŽ   ³   s~   ø„ Ù$ð +5Ø)+¯©ØØñ]àð]ð ð]ð ð	]ð
 ˜RŸY™Y™ð]ð ˜BŸI™I‘õ]ð,˜Ÿ™ð ¨%¯,©,÷ r$   rŽ   c                   óÂ   ‡ — e Zd Zeej
                  ddfdededededeej                     deej                     fˆ fd„Z
d	ej                  d
ej                  fd„Zˆ xZS )Ú
BasicBlockNru   r�   r�   Útyper‘   rw   c	           	      ó6  •— ||dœ}	t         ‰
| �  «        t        t        ||dddf|ddœ|	¤Ž«      | _        |dk(  rt        t        |||||fi |	¤Ž«      | _        nt        j                  «       | _        t        t        |t        |dz  «      fi |	¤Ž«      | _        y )NrI   rz   r   r   )r.   rG   ÚsrW   )r   r    r   r,   ÚconvrŽ   Úmixerr<   ÚIdentityr†   rc   Úffn)r!   ru   r�   r�   r­   r‘   rw   r7   rJ   rT   r"   s             €r#   r    zBasicBlock.__init__Û   s”   ø€ ð ¨Ñ/ˆÜ‰ÑÔÜœZ¨¨S°!°Q¸Ð^À#ÐVWÑ^Ð[]Ñ^Ó_ˆŒ	Ø�3Š;Ü!¤$ s¨F°D¸*ÀiÑ"VÐSUÑ"VÓWˆD�JäŸ™›ˆDŒJÜœC ¤S¨¨q©£\Ñ8°RÑ8Ó9ˆ�r$   r%   r&   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r   )r°   r±   r³   r(   s     r#   r)   zBasicBlock.forwardï   s.   € Ø�I‰I�a‹LˆØ�J‰J�q‹MˆØ�H‰H�Q‹KˆØˆr$   ©r9   r:   r;   r   r<   r„   rc   Ústrr	   r=   r    r0   r>   r)   r@   rA   s   @r#   r¬   r¬   Ú   s   ø„ ð +5Ø)+¯©ØØñ:àð:ð ð:ð ð	:ð
 ð:ð ˜RŸY™Y™ð:ð ˜BŸI™I‘õ:ð(˜Ÿ™ð ¨%¯,©,÷ r$   r¬   c                   óÊ   ‡ — e Zd Zeej
                  ddfdededededededeej                     d	eej                     fˆ fd
„Z
dej                  dej                  fd„Zˆ xZS )Ú
StageBlockNÚprev_dimru   r�   r�   r­   Údepthr‘   rw   c                 ó,  •— |	|
dœ}t         ‰| �  «        d| _        ||k7  r™t        j                  t        t        ||dddfd|i|¤Ž«      t        t        |t        |dz  «      |fi |¤Ž«      t        |||fi |¤Žt        t        ||dddfd|i|¤Ž«      t        t        |t        |dz  «      |fi |¤Ž«      «      nt        j                  «       | _        t        j                  t        |«      D �cg c]  }t        ||||||fi |¤Ž‘Œ c}Ž | _        y c c}w )NrI   Frz   r   r.   rW   )r   r    Úgrad_checkpointingr<   r–   r   r,   r†   rc   rt   r²   Ú
downsampleÚranger¬   Úblocks)r!   r¹   ru   r�   r�   r­   rº   r‘   rw   r7   rJ   rT   r¡   r"   s                €r#   r    zStageBlock.__init__÷   s  ø€ ð ¨Ñ/ˆÜ‰ÑÔØ"'ˆÔð ˜Š_ô Ÿ-™-Ü”Z ¨(°A°q¸!ÑSÀHÐSÐPRÑSÓTÜ”S˜¤3 x°!¡|Ó#4°iÑFÀ2ÑFÓGÜ˜ 3¨	Ñ8°RÑ8Ü”Z  S¨!¨Q°ÑD¸#ÐDÀÑDÓEÜ”S˜œc #¨¡'›l¨IÑ<¸Ñ<Ó=ô
ô #%§+¡+£-ð 	Œô —m‘mÜV[Ð\aÓVbö&
ØQRŒJ�s˜F D¨$°
¸IÑLÈÓLò&
ð ˆ�ùò &
s   Ã/Dr%   r&   c                 óÎ   — | j                  |«      }| j                  r6t        j                  j	                  «       st        | j                  |«      }|S | j                  |«      }|S r   )r½   r¼   r0   ÚjitÚis_scriptingr   r¿   r(   s     r#   r)   zStageBlock.forward  sS   € Ø�O‰O˜AÓˆØ×"Ò"¬5¯9©9×+AÑ+AÔ+CÜ˜tŸ{™{¨AÓ.ˆAð ˆð —‘˜A“ˆAØˆr$   rµ   rA   s   @r#   r¸   r¸   ö   s“   ø„ ð +5Ø)+¯©ØØñàðð ðð ð	ð
 ðð ðð ðð ˜RŸY™Y™ðð ˜BŸI™I‘õð8˜Ÿ™ð ¨%¯,©,÷ r$   r¸   c                   óÔ  ‡ — e Zd Zddddddddd	eej
                  d
d
fdedededeeeef   deeeef   deeeef   deeeef   deeeef   de	de
ej                     de
ej                     fˆ fd„Zej                  j                  defd„«       Zej                  j                  d-dedeeef   fd„«       Zej                  j                  d.d„«       Zej                  j                  dej                  fd„«       Zd/dedefd„Z	 	 	 	 	 d0dej2                  deeeee   f      d ed!ed"ed#edeeej2                     eej2                  eej2                     f   f   fd$„Z	 	 	 d1deeee   f   d%ed&efd'„Zdej2                  dej2                  fd(„Zd-dej2                  d)edej2                  fd*„Z dej2                  dej2                  fd+„Z! ejD                  «       d,„ «       Z#ˆ xZ$S )2r   rz   éè  Úavg)é€   é   i€  )é    é@   é`   )é   rË   rË   )r   rW   rz   )r¯   r¯   r¯   ç        NÚin_chansÚnum_classesÚglobal_poolr‡   Úpartial_dimr�   rº   ÚtypesÚ	drop_rater‘   rw   c                 ó°  •— t         ‰| �  «        ||dœ}|| _        || _        |	| _        g | _        |d   }t        j                  t        ||dz  dddfi |¤Ž |«       t        |dz  |dz  dddfi |¤Ž |«       t        |dz  |dz  dddfi |¤Ž |«       t        |dz  |dddfi |¤Ž«      | _	        g }|}t        t        |«      «      D ]i  }|j                  t        d|||   ||   ||   ||   ||   |
|dœ|¤Ž«       ||   }| j
                  j                  t        |d|dz   z  d	|› �¬
«      «       Œk t        j                  |Ž | _        |d   x| _        | _        t%        |¬«      | _        |rt        j(                  d«      nt        j*                  «       | _        |dkD  rt/        | j"                  |fi |¤Ž| _        y t        j*                  «       | _        y )NrI   r   é   rz   rW   r   ry   )r¹   ru   r�   r�   r­   rº   r‘   rw   zstages.)Únum_chsÚ	reductionÚmodulern   ©Ú	pool_type© )r   r    rÎ   rÍ   rÒ   Úfeature_infor<   r–   r,   Úpatch_embedr¾   ÚlenÚappendr¸   ÚdictÚstagesÚnum_featuresÚhead_hidden_sizer   rÏ   ÚFlattenr²   r›   re   Úhead)r!   rÍ   rÎ   rÏ   r‡   rÐ   r�   rº   rÑ   rÒ   r‘   rw   r7   rJ   rT   Ústem_chsrà   Úprev_chsÚir"   s                      €r#   r    zSHViT.__init__  sû  ø€ ô  	‰ÑÔØ¨Ñ/ˆØ&ˆÔØ ˆŒØ"ˆŒØˆÔð ˜Q‘<ˆÜŸ=™=Ü�x ¨Q¡°°1°aÑ>¸2Ñ>Ù‹KÜ�x 1‘} h°!¡m°Q¸¸1ÑCÀÑCÙ‹KÜ�x 1‘} h°!¡m°Q¸¸1ÑCÀÑCÙ‹KÜ�x 1‘} h°°1°aÑ>¸2Ñ>ó
ˆÔð ˆØˆÜ”s˜9“~Ó&ò 	gˆAØ�M‰Mœ*ð 
Ø!Ø˜a‘LØ˜a‘yØ  ‘^Ø˜1‘XØ˜A‘hØ%Ø#ñ
ð ñ
ô 
ð ! ‘|ˆHØ×Ñ×$Ñ$¤T°(ÀaÈ!ÈAÉ#ÁhÐY`ÐabÐ`cÐWdÔ%eÕfð	gô —m‘m VÐ,ˆŒð 5>¸b±MÐAˆÔ˜DÔ1Ü/¸+ÔFˆÔÙ(3”r—z‘z !”}¼¿¹»ˆŒØLWÐZ[ÊO”J˜t×4Ñ4°kÑHÀRÑHˆ�	Ôac×alÑalÓanˆ�	r$   r&   c                 ó   — t        «       S r   )Úset©r!   s    r#   Úno_weight_decayzSHViT.no_weight_decayY  s	   € ä‹uˆr$   Úcoarsec                 ó,   — t        d|rdnddg¬«      }|S )Nz^patch_embedz^stages\.(\d+))z^stages\.(\d+).downsample)r   )z^stages\.(\d+)\.blocks\.(\d+)N)Ústemr¿   )rß   )r!   rì   Úmatchers      r#   Úgroup_matcherzSHViT.group_matcher]  s'   € äØ Ù(.Ñ$Ø4Ø8ð5ô
ˆð ˆr$   c                 ó4   — | j                   D ]	  }||_        Œ y r   )rà   r¼   )r!   Úenabler¯   s      r#   Úset_grad_checkpointingzSHViT.set_grad_checkpointingh  s   € à—‘ò 	*ˆAØ#)ˆAÕ ñ	*r$   c                 ó.   — | j                   j                  S r   )rä   rj   rê   s    r#   Úget_classifierzSHViT.get_classifierm  s   € à�y‰y�{‰{Ðr$   c                 ó  — || _         t        |¬«      | _        |rt        j                  d«      nt        j
                  «       | _        |dkD  rt        | j                  |«      | _	        y t        j
                  «       | _	        y )NrØ   r   r   )
rÎ   r   rÏ   r<   rã   r²   r›   re   râ   rä   )r!   rÎ   rÏ   s      r#   Úreset_classifierzSHViT.reset_classifierq  s[   € Ø&ˆÔä/¸+ÔFˆÔÙ(3”r—z‘z !”}¼¿¹»ˆŒØFQÐTUÂo”J˜t×4Ñ4°kÓBˆ�	Ô[]×[fÑ[fÓ[hˆ�	r$   r%   ÚindicesÚnormÚ
stop_earlyÚ
output_fmtÚintermediates_onlyc                 ór  — |dv sJ d«       ‚g }t        t        | j                  «      |«      \  }}	| j                  |«      }t        j
                  j                  «       s|s| j                  }
n| j                  d|	dz    }
t        |
«      D ]#  \  }} ||«      }||v sŒ|j                  |«       Œ% |r|S ||fS )aï   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )ÚNCHWzOutput shape must be NCHW.Nr   )	r   rÝ   rà   rÜ   r0   rÁ   rÂ   Ú	enumeraterÞ   )r!   r%   rø   rù   rú   rû   rü   ÚintermediatesÚtake_indicesÚ	max_indexrà   Úfeat_idxÚstages                r#   Úforward_intermediateszSHViT.forward_intermediatesx  sÅ   € ð* ˜YÑ&ÐDÐ(DÓDÐ&ØˆÜ"6´s¸4¿;¹;Ó7GÈÓ"QÑˆ�ið ×Ñ˜QÓˆÜ�9‰9×!Ñ!Ô#©:Ø—[‘[‰Fà—[‘[  )¨a¡-Ð0ˆFä(¨Ó0ò 	(‰OˆH�eÙ�a“ˆAØ˜<Ò'Ø×$Ñ$ QÕ'ð	(ñ
 Ø Ð à�-ÐÐr$   Ú
prune_normÚ
prune_headc                 ó    — t        t        | j                  «      |«      \  }}| j                  d|dz    | _        |r| j                  dd«       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   Ú )r   rÝ   rà   r÷   )r!   rø   r  r  r  r  s         r#   Úprune_intermediate_layerszSHViT.prune_intermediate_layers¢  sM   € ô #7´s¸4¿;¹;Ó7GÈÓ"QÑˆ�iØ—k‘k . 9¨q¡=Ð1ˆŒÙØ×!Ñ! ! RÔ(ØÐr$   c                 óJ   — | j                  |«      }| j                  |«      }|S r   )rÜ   rà   r(   s     r#   Úforward_featureszSHViT.forward_features°  s$   € Ø×Ñ˜QÓˆØ�K‰K˜‹NˆØˆr$   Ú
pre_logitsc                 óæ   — | j                  |«      }| j                  |«      }| j                  dkD  r,t        j                  || j                  | j
                  ¬«      }|r|S | j                  |«      S )NrÌ   )ÚpÚtraining)rÏ   r›   rÒ   r4   Údropoutr  rä   )r!   r%   r  s      r#   Úforward_headzSHViT.forward_headµ  sZ   € Ø×Ñ˜QÓˆØ�L‰L˜‹OˆØ�>‰>˜BÒÜ—	‘	˜!˜tŸ~™~¸¿¹ÔFˆAÙˆqÐ0 D§I¡I¨a£LÐ0r$   c                 óJ   — | j                  |«      }| j                  |«      }|S r   )r  r  r(   s     r#   r)   zSHViT.forward¼  s'   € Ø×!Ñ! !Ó$ˆØ×Ñ˜aÓ ˆØˆr$   c                 ó    ‡— ˆfd„Š ‰| «       y )Nc                 óª   •— | j                  «       D ]?  \  }}t        |d«      r&|j                  «       }t        | ||«        ‰|«       Œ8 ‰|«       ŒA y )Nr-   )Únamed_childrenÚhasattrr-   Úsetattr)ÚnetÚ
child_nameÚchildÚfusedÚfuse_childrens       €r#   r  z!SHViT.fuse.<locals>.fuse_childrenÃ  sQ   ø€ Ø%(×%7Ñ%7Ó%9ò )Ñ!�
˜EÜ˜5 &Ô)Ø!ŸJ™J›L�EÜ˜C ¨UÔ3Ù! %Õ(á! %Õ(ñ)r$   rÚ   )r!   r  s    @r#   r-   z
SHViT.fuseÁ  s   ø€ ô	)ñ 	�dÕr$   ©F)T)rÅ   )NFFrþ   F)r   FT)%r9   r:   r;   r   r<   r„   rc   r¶   r   rr   r	   r=   r    r0   rÁ   Úignorer   rë   rq   r   r   rð   ró   rõ   r÷   r>   r   r
   r   r  r
  r  r  r)   r?   r-   r@   rA   s   @r#   r   r     só  ø„ ð Ø#Ø$Ø.=Ø0<Ø+7Ø*3Ø*9Ø!Ø*4Ø)+¯©ØØñ:oàð:oð ð:oð ð	:oð
 ˜S # s˜]Ñ+ð:oð ˜s C¨˜}Ñ-ð:oð ˜#˜s C˜-Ñ(ð:oð ˜˜c 3˜Ñ'ð:oð ˜˜c 3˜Ñ'ð:oð ð:oð ˜RŸY™Y™ð:oð ˜BŸI™I‘õ:oðx ‡Y�Y×Ñð ò ó ðð ‡Y�Y×Ññ Dð °T¸#¸s¸(±^ò ó ðð ‡Y�Y×Ñò*ó ð*ð ‡Y�Y×Ñð §	¡	ò ó ðñi¨Cð i¸có ið 8<ØØ$Ø$Ø',ñ( à�|‰|ð( ð ˜e C¨¨c© NÑ3Ñ4ð( ð ð	( ð
 ð( ð ð( ð !%ð( ð 
ˆt�E—L‘LÑ! 5¨¯©°t¸E¿L¹LÑ7IÐ)IÑ#JÐJÑ	Kó( ðX ./Ø$Ø#ñ	à˜3  S¡	˜>Ñ*ðð ðð ó	ð %§,¡,ð °5·<±<ó ñ
1˜eŸl™lð 1¸ð 1ÈÏÉó 1ð˜Ÿ™ð ¨%¯,©,ó ð
 €U‡]�]ƒ_ñ
ó ô
r$   Ú
state_dictÚmodelr&   c                 ó*   — | j                  d| «      } | S )Nr!  )Úget)r   r!  s     r#   Úcheckpoint_filter_fnr$  Ï  s   € Ø—‘ ¨Ó4€Jð: Ðr$   ÚurlrS   c                 ó8   — | dddddt         t        dddd	d
ddœ|¥S )NrÄ   )rz   éà   r'  )ry   ry   g      ì?Úbicubiczpatch_embed.0.czhead.lÚmitzarXiv:2401.16456zHSHViT: Single-Head Vision Transformer with Memory Efficient Macro Designz https://github.com/ysj9909/SHViT)r%  rÎ   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚmeanrh   Ú
first_convÚ
classifierÚlicenseÚ	paper_idsÚ
paper_nameÚ
origin_urlr   )r%  rS   s     r#   Ú_cfgr5  ð  s<   € à 4°}ÐSYØ¨IÜ%Ô.BØ'°xØØ'Ø`Ø8ñ
ð ð
ð 
r$   ztimm/)Ú	hf_hub_id)rz   rÇ   rÇ   )r6  r*  )zshvit_s1.in1kzshvit_s2.in1kzshvit_s3.in1kzshvit_s4.in1kÚvariantÚ
pretrainedc                 óN   — t        t        | |ft        t        dd¬«      dœ|¤Ž}|S )N)r   r   rW   T)Úout_indicesÚflatten_sequential)Úpretrained_filter_fnÚfeature_cfg)r   r   r$  rß   )r7  r8  rS   r!  s       r#   Ú_create_shvitr>    s6   € Ü Üˆw˜
ðä1Ü YÀ4ÔHñð ñ	€Eð €Lr$   c           	      óN   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤ŽS )N)rÆ   r'  i@  ©rW   ry   é   )rÈ   é0   éD   ©rç   r¯   r¯   ©r‡   rº   rÐ   rÑ   r8  )Úshvit_s1©rß   r>  ©r8  rS   Ú
model_argss      r#   rF  rF    ó5   € äØ!¨ÀÐTcôe€JäÑY°
ÐY¼dÀ:Ñ>XÐQWÑ>XÑYÐYr$   c           	      óN   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤ŽS )N)rÆ   i4  éÀ  r@  )rÈ   éB   rÊ   rD  rE  r8  )Úshvit_s2rG  rH  s      r#   rN  rN  $  rJ  r$   c           	      óN   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤ŽS )N)éÀ   i`  rL  )rz   rA  rA  )rB  éK   rÊ   rD  rE  r8  )Úshvit_s3rG  rH  s      r#   rR  rR  +  rJ  r$   c           	      óN   — t        dddd¬«      }t        dd| it        |fi |¤Ž¤ŽS )N)r'  iP  rL  )ry   é   é   )rB  éH   rÊ   rD  rE  r8  )Úshvit_s4rG  rH  s      r#   rW  rW  2  rJ  r$   )r	  r  )9rª   Útypingr   r   r   r   r   r   r	   r
   r0   Útorch.nnr<   Útorch.nn.functionalÚ
functionalr4   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_manipulater   Ú	_registryr   r   Ú__all__r=   r   r–   r,   re   rt   r†   rŽ   r¬   r¸   r   r¶   r>   r$  r5  Údefault_cfgsrq   r>  rF  rN  rR  rW  rÚ   r$   r#   ú<module>rd     sV  ðñ÷ F× EÓ Eã Ý ß Ð ç Aß aÕ aÝ *Ý +Ý 'ß <àˆ)€ôˆr�y‰yô ô*(�—‘ô (ôV�—‘ô ôD�2—9‘9ô ô:ˆ"�)‰)ô ô,$ˆ2�9‰9ô $ôN�—‘ô ô8#�—‘ô #ôLpˆB�I‰Iô pðf T¨#¨u¯|©|Ð*;Ñ%<ð ÀRÇYÁYð ÐSWÐX[Ð]b×]iÑ]iÐXiÑSjó ñBˆcð  #ð ¨$¨s°C¨x©.ó ñ %ÙØôñ Øôñ Øôñ Øà ôñ&ó €ñ*˜3ð ¨Dð ÀCð ÈEó ð ñZ˜ð Z°ð Z¸ò Zó ðZð ñZ˜ð Z°ð Z¸ò Zó ðZð ñZ˜ð Z°ð Z¸ò Zó ðZð ñZ˜ð Z°ð Z¸ò Zó ñZr$   