Ë
    éÓ^jÓ~  ã                   óž  — d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZmZmZmZ ddlmZ dd	l m!Z! dd
l"m#Z# ddl$m%Z%m&Z& dgZ'dddddœZ(dddddœZ)dddddœZ* G d„ dejV                  «      Z, G d„ dej                  jV                  «      Z- G d„ dej                  jV                  «      Z. G d „ d!ej                  jV                  «      Z/ G d"„ d#ejV                  «      Z0 G d$„ d%ejV                  «      Z1 G d&„ d'ejV                  «      Z2 G d(„ d)ejf                  «      Z4 G d*„ d+ejV                  «      Z5 G d,„ dejV                  «      Z6d7d-„Z7 e% e7d.¬/«       e7d.¬/«       e7d.¬/«       e7d.¬/«      d0œ«      Z8d8d1„Z9e&d8d2e6fd3„«       Z:e&d8d2e6fd4„«       Z;e&d8d2e6fd5„«       Z<e&d8d2e6fd6„«       Z=y)9aJ   EfficientFormer-V2

@article{
    li2022rethinking,
    title={Rethinking Vision Transformers for MobileNet Size and Speed},
    author={Li, Yanyu and Hu, Ju and Wen, Yang and Evangelidis, Georgios and Salahi, Kamyar and Wang, Yanzhi and Tulyakov, Sergey and Ren, Jian},
    journal={arXiv preprint arXiv:2212.08059},
    year={2022}
}

Significantly refactored and cleaned up for timm from original at: https://github.com/snap-research/EfficientFormer

Original code licensed Apache 2.0, Copyright (c) 2022 Snap Inc.

Modifications and timm support by / Copyright 2023, Ross Wightman
é    N)Úpartial)ÚDictÚListÚOptionalÚTupleÚTypeÚUnion©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)Úcreate_conv2dÚcreate_norm_layerÚget_act_layerÚget_norm_layerÚConvNormActÚLayerScale2dÚDropPathÚcalculate_drop_path_ratesÚtrunc_normal_Ú	to_2tupleÚ	to_ntupleÚndgridé   )Úbuild_model_with_cfg)Úfeature_take_indices)Úcheckpoint_seq)Úgenerate_default_cfgsÚregister_modelÚEfficientFormerV2)é(   éP   éÀ   é€  )é    é@   é�   i   )r$   é0   éx   éà   )r$   r'   é`   é°   )ÚLÚS2ÚS1ÚS0)é   r0   é   é
   )é   r3   é   é   )é   r6   é	   é   )é   r9   r8   r3   )r3   r3   )r3   r3   r3   r3   r6   r6   r6   r6   r6   r6   r6   r3   r3   r3   r3   )
r3   r3   r3   r6   r6   r6   r6   r3   r3   r3   )r3   r3   )r3   r3   r6   r6   r6   r6   r6   r6   r3   r3   r3   r3   )r3   r3   r6   r6   r6   r6   r3   r3   )r3   r3   )	r3   r3   r6   r6   r6   r6   r3   r3   r3   )r3   r3   r6   r6   r3   r3   )r3   r3   )r3   r6   r6   r6   r3   r3   )r3   r6   r6   r3   c                   ót   ‡ — e Zd Z	 	 	 	 	 	 	 	 	 	 ddededededeeef   dededed	ed
ee   fˆ fd„Z	d„ Z
ˆ xZS )ÚConvNormÚin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingÚdilationÚgroupsÚbiasÚ
norm_layerÚnorm_kwargsc           	      ó’   •— ||dœ}|
xs i }
t         ‰| �  «        t        |||f|||||dœ|¤Ž| _        t	        |	|fi |
¤|¤Ž| _        y )N©ÚdeviceÚdtype)r?   r@   rA   rB   rC   )ÚsuperÚ__init__r   Úconvr   Úbn)Úselfr<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rH   rI   ÚddÚ	__class__s                 €úi/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/efficientformer_v2.pyrK   zConvNorm.__init__F   ss   ø€ ð ¨Ñ/ˆØ!Ò' RˆÜ‰ÑÔÜ!ØØØð

ð ØØØØñ

ð ñ

ˆŒ	ô $ J°ÑRÀÐRÈrÑRˆ�ó    c                 óJ   — | j                  |«      }| j                  |«      }|S ©N)rL   rM   ©rN   Úxs     rQ   ÚforwardzConvNorm.forwarde   s!   € Ø�I‰I�a‹LˆØ�G‰G�A‹JˆØˆrR   )
r   r   Ú r   r   TÚbatchnorm2dNNN)Ú__name__Ú
__module__Ú__qualname__Úintr	   ÚstrÚboolr   r   rK   rW   Ú__classcell__©rP   s   @rQ   r;   r;   E   s¦   ø„ ð
  !ØØ')ØØØØ+Ø*.ØØñSàðSð ðSð ð	Sð
 ðSð ˜3 ˜8‘_ðSð ðSð ðSð ðSð ðSð " $™õSö>rR   r;   c                   óT  ‡ — e Zd ZU eeej                  f   ed<   dddddej                  dddf	de
d	e
d
e
de
dee
ee
e
f   f   deej                     dee
   fˆ fd„Z ej"                  «       dˆ fd„	«       Zdd„Zdd„Zdd„Zdd„Zdej.                  dej                  fd„Zd„ Zˆ xZS )ÚAttention2dÚattention_bias_cacher#   r$   r5   r3   é   NÚdimÚkey_dimÚ	num_headsÚ
attn_ratioÚ
resolutionÚ	act_layerr?   c
           	      ó˜  •— ||	dœ}
t         ‰| �  «        || _        |dz  | _        || _        t        |«      }|�bt        |D �cg c]  }t        j                  ||z  «      ‘Œ c}«      }t        ||fd||dœ|
¤Ž| _
        t        j                  |d¬«      | _        nd | _
        d | _        || _        | j                  d   | j                  d   z  | _        t!        ||z  «      | _        t!        ||z  «      |z  | _        || _        | j                  | j                  z  }t        ||fi |
¤Ž| _        t        ||fi |
¤Ž| _        t        || j$                  fi |
¤Ž| _        t        | j$                  | j$                  fd| j$                  d	œ|
¤Ž| _        t        j0                  | j                  | j                  fd
di|
¤Ž| _        t        j0                  | j                  | j                  fd
di|
¤Ž| _         |«       | _        t        | j$                  |dfi |
¤Ž| _        t:        j                  j=                  t;        j>                  || j                  fi |
¤Ž«      | _         | jC                  dt;        j>                  | j                  | j                  f|t:        jD                  ¬«      d¬«       i | _#        | jI                  «        y c c}w )NrG   ç      à¿r6   ©r>   r?   rB   Úbilinear)Úscale_factorÚmoder   r   )r>   rB   r>   Úattention_bias_idxsF©Ú
persistent)%rJ   rK   rh   Úscalerg   r   ÚtupleÚmathÚceilr;   Ústride_convÚnnÚUpsampleÚupsamplerj   ÚNr]   ÚdÚdhri   ÚqÚkÚvÚv_localÚConv2dÚtalking_head1Útalking_head2ÚactÚprojÚtorchÚ	ParameterÚemptyÚattention_biasesÚregister_bufferÚlongrd   Úreset_parameters)rN   rf   rg   rh   ri   rj   rk   r?   rH   rI   rO   ÚrÚkhrP   s                €rQ   rK   zAttention2d.__init__n   sc  ø€ ð ¨Ñ/ˆÜ‰ÑÔØ"ˆŒØ ‘_ˆŒ
ØˆŒä˜zÓ*ˆ
ØÐÜ¸zÖJ¸!¤§	¡	¨!¨f©*Õ 5ÒJÓKˆJÜ'¨¨SÐa¸aÈÐWZÑaÐ^`ÑaˆDÔÜŸK™K°VÀ*ÔMˆD�Mà#ˆDÔØ ˆDŒMà$ˆŒØ—‘ Ñ# d§o¡o°aÑ&8Ñ8ˆŒÜ�Z 'Ñ)Ó*ˆŒÜ�j 7Ñ*Ó+¨iÑ7ˆŒØ$ˆŒØ�\‰\˜DŸN™NÑ*ˆä˜#˜rÑ( RÑ(ˆŒÜ˜#˜rÑ( RÑ(ˆŒÜ˜#˜tŸw™wÑ-¨"Ñ-ˆŒÜ §¡¨¯©ÐV¸aÈÏÉÑVÐSUÑVˆŒÜŸY™Y t§~¡~°t·~±~Ñ[ÐSTÐ[ÐXZÑ[ˆÔÜŸY™Y t§~¡~°t·~±~Ñ[ÐSTÐ[ÐXZÑ[ˆÔá“;ˆŒÜ˜TŸW™W c¨1Ñ3°Ñ3ˆŒ	ä %§¡× 2Ñ 2´5·;±;¸yÈ$Ï&É&Ñ3WÐTVÑ3WÓ XˆÔØ×ÑØ!Ü�K‰K˜Ÿ™ §¡Ð(°¼u¿z¹zÔJØð 	ô 	
ð
 %'ˆÔ!ð 	×ÑÕùòC  Ks   ÁKc                 óR   •— t         ‰| �  |«       |r| j                  ri | _        y y y rT   ©rJ   Útrainrd   ©rN   rq   rP   s     €rQ   r”   zAttention2d.train¥   ó)   ø€ ä‰‰�dÔÙ�D×-Ò-Ø(*ˆDÕ%ð .ˆ4rR   Úreturnc                 óv   — t         j                  j                  | j                  «       | j	                  «        y©z"Initialize parameters and buffers.N©rz   ÚinitÚzeros_rŒ   Ú_init_buffers©rN   s    rQ   r�   zAttention2d.reset_parameters«   ó$   € ä
�‰�‰�t×,Ñ,Ô-Ø×ÑÕrR   c           
      ó¢  — t        j                  t        t        j                  | j                  d   |t         j
                  ¬«      t        j                  | j                  d   |t         j
                  ¬«      «      «      j                  d«      }|ddd…df   |dddd…f   z
  j                  «       }|d   | j                  d   z  |d   z   }|S )ú5Compute relative position indices for attention bias.r   rG   r   .N©r‰   Ústackr   Úarangerj   rŽ   ÚflattenÚabs)rN   rH   ÚposÚrel_poss       rQ   Ú_compute_attention_bias_idxsz(Attention2d._compute_attention_bias_idxs°   s¯   € ä�k‰kœ&Ü�L‰L˜Ÿ™¨Ñ+°FÄ%Ç*Á*ÔMÜ�L‰L˜Ÿ™¨Ñ+°FÄ%Ç*Á*ÔMó
ó ÷ ‰7�1‹:ð 	ð �sšA˜t�|Ñ$ s¨3°²a¨<Ñ'8Ñ8×=Ñ=Ó?ˆØ˜1‘: §¡°Ñ 2Ñ2°g¸a±jÑ@ˆØˆrR   c                 ó�   — | j                   j                  | j                  | j                   j                  ¬«      «       i | _        y©z.Compute and fill non-persistent buffer values.)rH   N©rr   Úcopy_r©   rH   rd   rž   s    rQ   r�   zAttention2d._init_buffersº   ó=   € à× Ñ ×&Ñ&Ø×-Ñ-°T×5MÑ5M×5TÑ5TÐ-ÓUô	
ð %'ˆÕ!rR   c                 ó$   — | j                  «        y©z"Initialize non-persistent buffers.N©r�   rž   s    rQ   Úinit_non_persistent_buffersz'Attention2d.init_non_persistent_buffersÁ   ó   € à×ÑÕrR   rH   c                 ó4  — t         j                  j                  «       s| j                  r| j                  d d …| j
                  f   S t        |«      }|| j                  vr*| j                  d d …| j
                  f   | j                  |<   | j                  |   S rT   ©r‰   ÚjitÚ
is_tracingÚtrainingrŒ   rr   r^   rd   ©rN   rH   Ú
device_keys      rQ   Úget_attention_biasesz Attention2d.get_attention_biasesÅ   óƒ   € Ü�9‰9×ÑÔ! T§]¢]Ø×(Ñ(ª¨D×,DÑ,DÐ)DÑEÐEä˜V›ˆJØ ×!:Ñ!:Ñ:Ø8<×8MÑ8MÊaÐQU×QiÑQiÐNiÑ8j�×)Ñ)¨*Ñ5Ø×,Ñ,¨ZÑ8Ð8rR   c                 ó$  — |j                   \  }}}}| j                  �| j                  |«      }| j                  |«      j                  || j                  d| j
                  «      j                  dddd«      }| j                  |«      j                  || j                  d| j
                  «      j                  dddd«      }| j                  |«      }| j                  |«      }	|j                  || j                  d| j
                  «      j                  dddd«      }||z  | j                  z  }
|
| j                  |j                  «      z   }
| j                  |
«      }
|
j                  d¬«      }
| j                  |
«      }
|
|z  j!                  dd«      }|j                  || j"                  | j$                  d   | j$                  d   «      |	z   }| j&                  �| j'                  |«      }| j)                  |«      }| j+                  |«      }|S ©Néÿÿÿÿr   r   r6   r9   ©rf   )Úshapery   r€   Úreshaperh   r}   Úpermuter�   r‚   rƒ   ru   r»   rH   r…   Úsoftmaxr†   Ú	transposer   rj   r|   r‡   rˆ   ©rN   rV   ÚBÚCÚHÚWr€   r�   r‚   rƒ   Úattns              rQ   rW   zAttention2d.forwardÎ   s»  € Ø—W‘W‰
ˆˆ1ˆa�Ø×ÑÐ'Ø× Ñ  Ó#ˆAà�F‰F�1‹I×Ñ˜a §¡°°T·V±VÓ<×DÑDÀQÈÈ1ÈaÓPˆØ�F‰F�1‹I×Ñ˜a §¡°°T·V±VÓ<×DÑDÀQÈÈ1ÈaÓPˆØ�F‰F�1‹IˆØ—,‘,˜q“/ˆØ�I‰I�a˜Ÿ™¨¨T¯V©VÓ4×<Ñ<¸QÀÀ1ÀaÓHˆà�A‘˜Ÿ™Ñ#ˆØ�d×/Ñ/°·±Ó9Ñ9ˆØ×!Ñ! $Ó'ˆØ�|‰| ˆ|Ó#ˆØ×!Ñ! $Ó'ˆà�A‰X× Ñ   AÓ&ˆØ�I‰I�a˜Ÿ™ $§/¡/°!Ñ"4°d·o±oÀaÑ6HÓIÈGÑSˆØ�=‰=Ð$Ø—‘˜aÓ ˆAà�H‰H�Q‹KˆØ�I‰I�a‹LˆØˆrR   ©T©r—   NrT   )rZ   r[   r\   r   r^   r‰   ÚTensorÚ__annotations__rz   ÚGELUr]   r	   r   r   ÚModuler   rK   Úno_gradr”   r�   r©   r�   r²   rH   r»   rW   r`   ra   s   @rQ   rc   rc   k   sð   ø… Ø˜s E§L¡LÐ0Ñ1Ó1ð ØØØØ67Ø)+¯©Ø$(ØØñ5 àð5 ð ð5 ð ð	5 ð
 ð5 ð ˜c 5¨¨c¨¡?Ð2Ñ3ð5 ð ˜BŸI™I‘ð5 ð ˜S‘Mõ5 ðn €U‡]�]ƒ_ô+ó ð+ó
ó
ó'óð9¨5¯<©<ð 9¸E¿L¹Ló 9örR   rc   c                   ó4   ‡ — e Zd Z	 	 ddedefˆ fd„Zd„ Zˆ xZS )ÚLocalGlobalQueryÚin_dimÚout_dimc                 óÌ   •— ||dœ}t         ‰| �  «        t        j                  ddd«      | _        t        j
                  ||fddd|dœ|¤Ž| _        t        ||dfi |¤Ž| _        y )NrG   r   r9   r   r6   )r>   r?   r@   rB   )	rJ   rK   rz   Ú	AvgPool2dÚpoolr„   Úlocalr;   rˆ   )rN   rÕ   rÖ   rH   rI   rO   rP   s         €rQ   rK   zLocalGlobalQuery.__init__ê   sh   ø€ ð ¨Ñ/ˆÜ‰ÑÔÜ—L‘L  A qÓ)ˆŒ	Ü—Y‘Y˜v vÐg¸1ÀQÐPQÐZ`ÑgÐdfÑgˆŒ
Ü˜V W¨aÑ6°2Ñ6ˆ�	rR   c                 óv   — | j                  |«      }| j                  |«      }||z   }| j                  |«      }|S rT   )rÚ   rÙ   rˆ   )rN   rV   Úlocal_qÚpool_qr€   s        rQ   rW   zLocalGlobalQuery.forward÷   s8   € Ø—*‘*˜Q“-ˆØ—‘˜1“ˆØ�fÑˆØ�I‰I�a‹LˆØˆrR   )NN)rZ   r[   r\   r]   rK   rW   r`   ra   s   @rQ   rÔ   rÔ   é   s'   ø„ ð
 Øñ7àð7ð õ7örR   rÔ   c                   óT  ‡ — e Zd ZU eeej                  f   ed<   ddddddej                  ddf	de
d	e
d
e
de
dee
ee
e
f   f   dee
   deej                     fˆ fd„Z ej"                  «       dˆ fd„	«       Zdd„Zdd„Zdd„Zdd„Zdej.                  dej                  fd„Zd„ Zˆ xZS )ÚAttention2dDownsamplerd   r#   é   r5   r3   re   Nrf   rg   rh   ri   rj   rÖ   rk   c
           	      óÈ  •— ||	dœ}
t         ‰| �  «        || _        |dz  | _        || _        t        |«      | _        t        | j                  D �cg c]  }t        j                  |dz  «      ‘Œ c}«      | _
        | j                  d   | j                  d   z  | _        | j                  d   | j                  d   z  | _        t        ||z  «      | _        t        ||z  «      |z  | _        || _        |xs || _        | j                  | j                  z  }t%        ||fi |
¤Ž| _        t)        ||dfi |
¤Ž| _        t)        || j                  dfi |
¤Ž| _        t)        | j                  | j                  fdd| j                  dœ|
¤Ž| _         |«       | _        t)        | j                  | j"                  dfi |
¤Ž| _        t5        j6                  t9        j:                  || j                  fi |
¤Ž«      | _        | j?                  dt9        j:                  | j                  | j                  f|t8        j@                  ¬«      d	¬
«       i | _!        | jE                  «        y c c}w )NrG   rm   r9   r   r   r6   rn   rr   Frs   )#rJ   rK   rh   ru   rg   r   rj   rv   rw   rx   Úresolution2r}   ÚN2r]   r~   r   ri   rÖ   rÔ   r€   r;   r�   r‚   rƒ   r‡   rˆ   rz   rŠ   r‰   r‹   rŒ   r�   rŽ   rd   r�   )rN   rf   rg   rh   ri   rj   rÖ   rk   rH   rI   rO   r�   r‘   rP   s                €rQ   rK   zAttention2dDownsample.__init__  s  ø€ ð ¨Ñ/ˆÜ‰ÑÔà"ˆŒØ ‘_ˆŒ
ØˆŒÜ# JÓ/ˆŒÜ ¸D¿O¹OÖ!L°q¤$§)¡)¨A°©EÕ"2Ò!LÓMˆÔØ—‘ Ñ# d§o¡o°aÑ&8Ñ8ˆŒØ×"Ñ" 1Ñ%¨×(8Ñ(8¸Ñ(;Ñ;ˆŒä�Z 'Ñ)Ó*ˆŒÜ�j 7Ñ*Ó+¨iÑ7ˆŒØ$ˆŒØ’~ #ˆŒØ�\‰\˜DŸN™NÑ*ˆä! # rÑ0¨RÑ0ˆŒÜ˜#˜r 1Ñ+¨Ñ+ˆŒÜ˜#˜tŸw™w¨Ñ0¨RÑ0ˆŒÜ §¡¨¯©Ð`¸aÈÐRV×RYÑRYÑ`Ð]_Ñ`ˆŒá“;ˆŒÜ˜TŸW™W d§l¡l°AÑ<¸Ñ<ˆŒ	ä "§¡¬U¯[©[¸ÀDÇFÁFÑ-QÈbÑ-QÓ RˆÔØ×ÑØ!Ü�K‰K˜Ÿ™ $§&¡&Ð)°&ÄÇ
Á
ÔKØð 	ô 	
ð
 %'ˆÔ!ð 	×ÑÕùò7 "Ms   ÁIc                 óR   •— t         ‰| �  |«       |r| j                  ri | _        y y y rT   r“   r•   s     €rQ   r”   zAttention2dDownsample.train2  r–   rR   r—   c                 óv   — t         j                  j                  | j                  «       | j	                  «        yr™   rš   rž   s    rQ   r�   z&Attention2dDownsample.reset_parameters8  rŸ   rR   c                 óÊ  — t        j                  t        t        j                  | j                  d   |t         j
                  ¬«      t        j                  | j                  d   |t         j
                  ¬«      «      «      j                  d«      }t        j                  t        t        j                  d| j                  d   d|t         j
                  ¬«      t        j                  d| j                  d   d|t         j
                  ¬«      «      «      j                  d«      }|ddd…df   |dddd…f   z
  j                  «       }|d   | j                  d   z  |d   z   }|S )r¡   r   rG   r   r9   )ÚsteprH   rI   .Nr¢   )rN   rH   Úk_posÚq_posr¨   s        rQ   r©   z2Attention2dDownsample._compute_attention_bias_idxs=  s  € ä—‘œFÜ�L‰L˜Ÿ™¨Ñ+°FÄ%Ç*Á*ÔMÜ�L‰L˜Ÿ™¨Ñ+°FÄ%Ç*Á*ÔMó
ó ÷ ‰7�1‹:ð 	ô —‘œFÜ�L‰L˜˜DŸO™O¨AÑ.°Q¸vÌUÏZÉZÔXÜ�L‰L˜˜DŸO™O¨AÑ.°Q¸vÌUÏZÉZÔXó
ó ÷ ‰7�1‹:ð 	ð ˜ša ˜Ñ&¨¨s°Dº!¨|Ñ)<Ñ<×AÑAÓCˆØ˜1‘: §¡°Ñ 2Ñ2°g¸a±jÑ@ˆØˆrR   c                 ó�   — | j                   j                  | j                  | j                   j                  ¬«      «       i | _        yr«   r¬   rž   s    rQ   r�   z#Attention2dDownsample._init_buffersK  r®   rR   c                 ó$   — | j                  «        yr°   r±   rž   s    rQ   r²   z1Attention2dDownsample.init_non_persistent_buffersR  r³   rR   rH   c                 ó4  — t         j                  j                  «       s| j                  r| j                  d d …| j
                  f   S t        |«      }|| j                  vr*| j                  d d …| j
                  f   | j                  |<   | j                  |   S rT   rµ   r¹   s      rQ   r»   z*Attention2dDownsample.get_attention_biasesV  r¼   rR   c                 ól  — |j                   \  }}}}| j                  |«      j                  || j                  d| j                  «      j                  dddd«      }| j                  |«      j                  || j                  d| j                  «      j                  dddd«      }| j                  |«      }| j                  |«      }	|j                  || j                  d| j                  «      j                  dddd«      }||z  | j                  z  }
|
| j                  |j                  «      z   }
|
j                  d¬«      }
|
|z  j                  dd«      }|j                  || j                  | j                   d   | j                   d   «      |	z   }| j#                  |«      }| j%                  |«      }|S r¾   )rÁ   r€   rÂ   rh   rã   rÃ   r�   r}   r‚   rƒ   ru   r»   rH   rÄ   rÅ   r   râ   r‡   rˆ   rÆ   s              rQ   rW   zAttention2dDownsample.forward_  sk  € Ø—W‘W‰
ˆˆ1ˆa�à�F‰F�1‹I×Ñ˜a §¡°°T·W±WÓ=×EÑEÀaÈÈAÈqÓQˆØ�F‰F�1‹I×Ñ˜a §¡°°T·V±VÓ<×DÑDÀQÈÈ1ÈaÓPˆØ�F‰F�1‹IˆØ—,‘,˜q“/ˆØ�I‰I�a˜Ÿ™¨¨T¯V©VÓ4×<Ñ<¸QÀÀ1ÀaÓHˆà�A‘˜Ÿ™Ñ#ˆØ�d×/Ñ/°·±Ó9Ñ9ˆØ�|‰| ˆ|Ó#ˆà�A‰X× Ñ   AÓ&ˆØ�I‰I�a˜Ÿ™ $×"2Ñ"2°1Ñ"5°t×7GÑ7GÈÑ7JÓKÈgÑUˆØ�H‰H�Q‹KˆØ�I‰I�a‹LˆØˆrR   rÌ   rÍ   rT   )rZ   r[   r\   r   r^   r‰   rÎ   rÏ   rz   rÐ   r]   r	   r   r   r   rÑ   rK   rÒ   r”   r�   r©   r�   r²   rH   r»   rW   r`   ra   s   @rQ   rß   rß   ÿ   sð   ø… Ø˜s E§L¡LÐ0Ñ1Ó1ð ØØØØ67Ø%)Ø)+¯©ØØñ. àð. ð ð. ð ð	. ð
 ð. ð ˜c 5¨¨c¨¡?Ð2Ñ3ð. ð ˜c‘]ð. ð ˜BŸI™I‘õ. ð` €U‡]�]ƒ_ô+ó ð+ó
ó
ó'óð9¨5¯<©<ð 9¸E¿L¹Ló 9örR   rß   c                   ó  ‡ — e Zd Zdddddej                  ej
                  ddf	deded	eeeeef   f   d
eeeeef   f   deeeeef   f   deeeeef   f   de	de
ej                     dee
ej                        fˆ fd„Zd„ Zˆ xZS )Ú
Downsampler6   r9   r   re   FNÚin_chsÚout_chsr>   r?   r@   rj   Úuse_attnrk   rD   c                 ó
  •— |
|dœ}t         ‰| �  «        t        |«      }t        |«      }t        |«      }|	xs t        j                  «       }	t        ||f||||	dœ|¤Ž| _        |rt        d||||dœ|¤Ž| _        y d | _        y )NrG   )r>   r?   r@   rD   )rf   rÖ   rj   rk   © )	rJ   rK   r   rz   ÚIdentityr;   rL   rß   rË   )rN   rð   rñ   r>   r?   r@   rj   rò   rk   rD   rH   rI   rO   rP   s                €rQ   rK   zDownsample.__init__t  s¬   ø€ ð ¨Ñ/ˆÜ‰ÑÔä Ó,ˆÜ˜6Ó"ˆÜ˜GÓ$ˆØÒ0¤2§;¡;£=ˆ
ÜØØð
ð $ØØØ!ñ
ð ñ
ˆŒ	ñ Ü-ð ØØØ%Ø#ñ	ð
 ñˆD�Ið ˆD�IrR   c                 óh   — | j                  |«      }| j                  �| j                  |«      |z   S |S rT   )rL   rË   )rN   rV   Úouts      rQ   rW   zDownsample.forwardž  s1   € Ø�i‰i˜‹lˆØ�9‰9Ð Ø—9‘9˜Q“< #Ñ%Ð%Øˆ
rR   )rZ   r[   r\   rz   rÐ   ÚBatchNorm2dr]   r	   r   r_   r   rÑ   r   rK   rW   r`   ra   s   @rQ   rï   rï   s  sè   ø„ ð
 89Ø23Ø34Ø67Ø"Ø)+¯©Ø46·N±NØØñ(àð(ð ð(ð ˜s E¨#¨s¨(¡OÐ3Ñ4ð	(ð
 ˜#˜u S¨# X™Ð.Ñ/ð(ð ˜3  c¨3 h¡Ð/Ñ0ð(ð ˜c 5¨¨c¨¡?Ð2Ñ3ð(ð ð(ð ˜BŸI™I‘ð(ð !  b§i¡i¡Ñ1õ(öTrR   rï   c                   óÀ   ‡ — e Zd ZdZddej
                  ej                  ddddfdedee   dee   de	ej                     d	e	ej                     d
edefˆ fd„Zd„ Zˆ xZS )ÚConvMlpWithNormz`
    Implementation of MLP with 1*1 convolutions.
    Input: tensor with shape [B, C, H, W]
    Nç        FÚin_featuresÚhidden_featuresÚout_featuresrk   rD   ÚdropÚmid_convc
                 ól  •— ||	dœ}
t         ‰| �  «        |xs |}|xs |}t        ||dfd||dœ|
¤Ž| _        |rt        ||df|d||dœ|
¤Ž| _        nt        j                  «       | _        t        j                  |«      | _        t        ||dfd|i|
¤Ž| _
        t        j                  |«      | _        y )NrG   r   T)rC   rD   rk   r6   )rB   rC   rD   rk   rD   )rJ   rK   r   Úfc1Úmidrz   rõ   ÚDropoutÚdrop1r;   Úfc2Údrop2)rN   rü   rý   rþ   rk   rD   rÿ   r   rH   rI   rO   rP   s              €rQ   rK   zConvMlpWithNorm.__init__«  sÞ   ø€ ð ¨Ñ/ˆÜ‰ÑÔØ#Ò2 {ˆØ)Ò8¨[ˆÜØØØð
ð Ø!Øñ
ð ñ
ˆŒñ Ü"ØØØð	ð 'ØØ%Ø#ñ	ð ñ	ˆD�Hô —{‘{“}ˆDŒHÜ—Z‘Z Ó%ˆŒ
Ü˜O¨\¸1ÑZÈÐZÐWYÑZˆŒÜ—Z‘Z Ó%ˆ�
rR   c                 ó°   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j	                  |«      }|S rT   )r  r  r  r  r  rU   s     rQ   rW   zConvMlpWithNorm.forwardÕ  sH   € Ø�H‰H�Q‹KˆØ�H‰H�Q‹KˆØ�J‰J�q‹MˆØ�H‰H�Q‹KˆØ�J‰J�q‹MˆØˆrR   )rZ   r[   r\   Ú__doc__rz   rÐ   rø   r]   r   r   rÑ   Úfloatr_   rK   rW   r`   ra   s   @rQ   rú   rú   ¥  s‘   ø„ ñð .2Ø*.Ø)+¯©Ø*,¯.©.ØØ"ØØñ(&àð(&ð & c™]ð(&ð # 3™-ð	(&ð
 ˜BŸI™I‘ð(&ð ˜RŸY™Y™ð(&ð ð(&ð õ(&öTrR   rú   c                   óâ   ‡ — e Zd Zdej                  ej
                  ddddddddfdeded	eej                     d
eej                     dedede
e   deeeeef   f   de
e   defˆ fd„Zd„ Zˆ xZS )ÚEfficientFormerV2Blockç      @rû   çñhãˆµøä>re   NTrf   Ú	mlp_ratiork   rD   Ú	proj_dropÚ	drop_pathÚlayer_scale_init_valuerj   r?   rò   c           
      ó  •— ||dœ}t         ‰| �  «        |
rgt        |f|||	dœ|¤Ž| _        |�t	        ||fi |¤Žnt        j                  «       | _        |dkD  rt        |«      nt        j                  «       | _	        nd | _        d | _        d | _	        t        d|t        ||z  «      |||ddœ|¤Ž| _        |�t	        ||fi |¤Žnt        j                  «       | _        |dkD  rt        |«      | _        y t        j                  «       | _        y )NrG   )rj   rk   r?   rû   T)rü   rý   rk   rD   rÿ   r   rô   )rJ   rK   rc   Útoken_mixerr   rz   rõ   Úls1r   Ú
drop_path1rú   r]   ÚmlpÚls2Ú
drop_path2)rN   rf   r  rk   rD   r  r  r  rj   r?   rò   rH   rI   rO   rP   s                 €rQ   rK   zEfficientFormerV2Block.__init__ß  s1  ø€ ð ¨Ñ/ˆÜ‰ÑÔáÜ*Øð à%Ø#Øñ	 ð
 ñ ˆDÔð 7MÐ6Xô $ØÐ+ñ3Ø/1ò3Ü^`×^iÑ^iÓ^kð ŒHà5>À²^œh yÔ1ÌÏÉËˆD�Oà#ˆDÔØˆDŒHØ"ˆDŒOä"ð 
ØÜ  i¡Ó0ØØ!ØØñ
ð ñ
ˆŒð 3IÐ2Tô  ØÐ'ñ/Ø+-ò/ÜZ\×ZeÑZeÓZgð 	Œà1:¸R²œ( 9Ó-ˆ�ÄRÇ[Á[Ã]ˆ�rR   c                 óæ   — | j                   �2|| j                  | j                  | j                  |«      «      «      z   }|| j                  | j	                  | j                  |«      «      «      z   }|S rT   )r  r  r  r  r  r  rU   s     rQ   rW   zEfficientFormerV2Block.forward  s^   € Ø×ÑÐ'Ø�D—O‘O D§H¡H¨T×-=Ñ-=¸aÓ-@Ó$AÓBÑBˆAØ�—‘ §¡¨¯©°!«Ó 5Ó6Ñ6ˆØˆrR   )rZ   r[   r\   rz   rÐ   rø   r]   r
  r   rÑ   r   r	   r   r_   rK   rW   r`   ra   s   @rQ   r  r  Þ  sÐ   ø„ ð  "Ø)+¯©Ø*,¯.©.Ø!Ø!Ø6:Ø67Ø$(Ø!ØØñ-Sàð-Sð ð-Sð ˜BŸI™I‘ð	-Sð
 ˜RŸY™Y™ð-Sð ð-Sð ð-Sð %-¨U¡Oð-Sð ˜c 5¨¨c¨¡?Ð2Ñ3ð-Sð ˜S‘Mð-Sð õ-Sö^rR   r  c            
       ó–   ‡ — e Zd Zej                  ej
                  ddfdededeej                     deej                     fˆ fd„Z	ˆ xZ
S )ÚStem4Nrð   rñ   rk   rD   c           
      óª   •— ||dœ}t         ‰| �  «        d| _        t        ||dz  fdddd||dœ|¤Ž| _        t        |dz  |fdddd||dœ|¤Ž| _        y )NrG   r3   r9   r6   r   T)r>   r?   r@   rC   rD   rk   )rJ   rK   r?   r   Úconv1Úconv2)	rN   rð   rñ   rk   rD   rH   rI   rO   rP   s	           €rQ   rK   zStem4.__init__  s”   ø€ ð ¨Ñ/ˆÜ‰ÑÔØˆŒÜ ØØ�q‰Lð	
ð Ø˜aØØ!Øñ	
ð ñ	
ˆŒ
ô !Ø�q‰LØð

ð ØØØØ!Øñ

ð ñ

ˆ�
rR   )rZ   r[   r\   rz   rÐ   rø   r]   r   rÑ   rK   r`   ra   s   @rQ   r  r    sY   ø„ ð
 *,¯©Ø*,¯.©.ØØñ 
àð 
ð ð 
ð ˜BŸI™I‘ð	 
ð
 ˜RŸY™Y™÷ 
ñ  
rR   r  c                    ó   ‡ — e Zd Zddddddddddej                  ej
                  ddfd	ed
ededeeeeef   f   de	de
e   de	de	dedeeeedf   f   dedeeee   f   de
e   deej                     deej                     fˆ fd„Zd„ Zˆ xZS )ÚEfficientFormerV2Stagere   TNFr   r  rû   r  rf   Údim_outÚdepthrj   Ú
downsampleÚblock_strideÚdownsample_use_attnÚblock_use_attnÚnum_vitr  .r  r  r  rk   rD   c                 ó  •— ||dœ}t         ‰| �  «        d| _         t        |«      |
«      }
t	        |«      }|rIt        ||f||||dœ|¤Ž| _        |}t        |D �cg c]  }t        j                  |dz  «      ‘Œ c}«      }n ||k(  sJ ‚t        j                  «       | _        g }t        |«      D ]3  }||	z
  dz
  }t        |f|||
|   |xr ||kD  |||   |||dœ	|¤Ž}||gz  }Œ5 t        j                  |Ž | _        y c c}w )NrG   F)rò   rj   rD   rk   r9   r   )	rj   r?   r  rò   r  r  r  rk   rD   )rJ   rK   Úgrad_checkpointingr   r   rï   r$  rv   rw   rx   rz   rõ   Úranger  Ú
SequentialÚblocks)rN   rf   r"  r#  rj   r$  r%  r&  r'  r(  r  r  r  r  rk   rD   rH   rI   rO   r�   r-  Ú	block_idxÚ
remain_idxÚbrP   s                           €rQ   rK   zEfficientFormerV2Stage.__init__;  sG  ø€ ð( ¨Ñ/ˆÜ‰ÑÔØ"'ˆÔØ$”I˜eÓ$ YÓ/ˆ	Ü˜zÓ*ˆ
áÜ(ØØðð -Ø%Ø%Ø#ñð ñˆDŒOð ˆCÜ¸*ÖE°Q¤§	¡	¨!¨a©%Õ 0ÒEÓF‰Jà˜'’>Ð!�>Ü Ÿk™k›mˆDŒOàˆÜ˜u›ò 	ˆIØ ™¨1Ñ,ˆJÜ&Øðà%Ø#Ø# IÑ.Ø'ÒB¨I¸
Ñ,BØ#Ø# IÑ.Ø'=Ø#Ø%ñð ñˆAð �q�c‰M‰Fð	ô  —m‘m VÐ,ˆ�ùò-  Fs   ÁC>c                 óÎ   — | j                  |«      }| j                  r6t        j                  j	                  «       st        | j                  |«      }|S | j                  |«      }|S rT   )r$  r*  r‰   r¶   Úis_scriptingr   r-  rU   s     rQ   rW   zEfficientFormerV2Stage.forwardx  sS   € Ø�O‰O˜AÓˆØ×"Ò"¬5¯9©9×+AÑ+AÔ+CÜ˜tŸ{™{¨AÓ.ˆAð ˆð —‘˜A“ˆAØˆrR   )rZ   r[   r\   rz   rÐ   rø   r]   r	   r   r_   r   r
  r   r   rÑ   rK   rW   r`   ra   s   @rQ   r!  r!  9  s!  ø„ ð 78Ø#Ø*.Ø(-Ø#(ØØ9;Ø!Ø35Ø6:Ø)+¯©Ø*,¯.©.ØØñ%;-àð;-ð ð;-ð ð	;-ð
 ˜c 5¨¨c¨¡?Ð2Ñ3ð;-ð ð;-ð # 3™-ð;-ð "&ð;-ð !ð;-ð ð;-ð ˜U E¨%°¨*Ñ$5Ð5Ñ6ð;-ð ð;-ð ˜U D¨¡KÐ/Ñ0ð;-ð %-¨U¡Oð;-ð ˜BŸI™I‘ð;-ð  ˜RŸY™Y™õ!;-özrR   r!  c            #       ó`  ‡ — e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d,deedf   dedeeeeef   f   dedeeedf      deeedf      dee	ee	df   eee	df   df   f   d	ed
e	dedede	de	de	dee	   dedef"ˆ fd„Z
d-defd„Zd-defd„Zej                  j                  d„ «       Zej                  j                  d.d„«       Zej                  j                  d-d„«       Zej                  j                  dej(                  fd„«       Zd/dedee   fd„Zej                  j                  d-d„«       Z	 	 	 	 	 d0dej0                  deeeee   f      d ed!ed"ed#edeeej0                     eej0                  eej0                     f   f   fd$„Z	 	 	 d1deeee   f   d%ed&efd'„Zd(„ Zd.d)efd*„Zd+„ Zˆ xZS )2r   Údepths.Úin_chansÚimg_sizeÚglobal_poolÚ
embed_dimsÚdownsamplesÚ
mlp_ratiosrD   Únorm_epsrk   Únum_classesÚ	drop_rateÚproj_drop_rateÚdrop_path_rater  r(  Údistillationc                 óü  •— t         ‰| �  «        ||dœ}|dv sJ ‚|| _        || _        || _        g | _        t        |«      }t        t        |«      |	¬«      }t        |
«      }
t        ||d   f|
|dœ|¤Ž| _        |d   }d}t        |«      }t        ||d¬«      }|xs d	d
t        |«      dz
  z  z   } t        |«      |«      }g }t        |«      D ]­  }t!        |D �cg c]  }t#        j$                  ||z  «      ‘Œ c}«      }t'        |||   f||   |||   |dk(  rdnd |dk\  |dk\  |||   |||   ||
|dœ|¤Ž}||   r|dz  }||   }| xj
                  t)        ||d|› �¬«      gz  c_        |j+                  |«       Œ¯ t-        j.                  |Ž | _        |d   x| _        | _         ||d   fi |¤Ž| _        t-        j8                  |«      | _        |dkD  rt-        j<                  |d   |fi |¤Žnt-        j>                  «       | _         || _!        | jB                  r9|dkD  rt-        j<                  |d   |fi |¤Žnt-        j>                  «       | _"        nd | _"        | jG                  d¬«       d| _$        y c c}w )NrG   )ÚavgrX   )Úepsr   )rk   rD   r3   T)Ú	stagewise©FrÌ   r   r9   r6   )r#  rj   r$  r%  r&  r'  r(  r  r  r  r  rk   rD   zstages.)Únum_chsÚ	reductionÚmoduler¿   F©Úneeds_reset)%rJ   rK   r<  r5  r7  Úfeature_infor   r   r   r   r  ÚstemÚlenr   r   r+  rv   rw   rx   r!  ÚdictÚappendrz   r,  ÚstagesÚnum_featuresÚhead_hidden_sizeÚnormr  Ú	head_dropÚLinearrõ   ÚheadÚdistÚ	head_distÚinit_weightsÚdistilled_training)rN   r4  r5  r6  r7  r8  r9  r:  rD   r;  rk   r<  r=  r>  r?  r  r(  r@  rH   rI   rO   Úprev_dimr?   Ú
num_stagesÚdprrP  ÚiÚsÚcurr_resolutionÚstagerP   s                                 €rQ   rK   zEfficientFormerV2.__init__‚  s³  ø€ ô, 	‰ÑÔØ¨Ñ/ˆØ˜kÑ)Ð)Ð)Ø&ˆÔØ ˆŒØ&ˆÔØˆÔÜ˜XÓ&ˆÜœ^¨JÓ7¸XÔFˆ
Ü! )Ó,ˆ	ä˜( J¨q¡MÐd¸YÐS]ÑdÐacÑdˆŒ	Ø˜a‘=ˆØˆä˜“[ˆ
Ü'¨¸È$ÔOˆØ!ÒK X°¼3¸v»;È¹?Ñ0KÑ%KˆØ*”Y˜zÓ*¨:Ó6ˆ
ØˆÜ�zÓ"ò 	!ˆAÜ#ÀHÖ$M¸q¤T§Y¡Y¨q°6©zÕ%:Ò$MÓNˆOÜ*ØØ˜1‘ðð ˜Q‘iØ*Ø& q™>Ø"# q¢&™Q¨dØ$%¨¡FØ  A™vØØ$ Q™-Ø(Ø˜a™&Ø'=Ø#Ø%ñð  ñ!ˆEð$ ˜1Š~Ø˜!‘�Ø! !‘}ˆHØ×Ò¤$¨xÀ6ÐT[Ð\]Ð[^ÐR_Ô"`Ð!aÑaÕØ�M‰M˜%Õ ð1	!ô2 —m‘m VÐ,ˆŒð 5?¸r±NÐBˆÔ˜DÔ1Ù˜z¨"™~Ñ4°Ñ4ˆŒ	ÜŸ™ IÓ.ˆŒØDOÐRSÂO”B—I‘I˜j¨™n¨kÑ@¸RÒ@ÔY[×YdÑYdÓYfˆŒ	Ø ˆŒ	Ø�9Š9ØMXÐ[\Ê_œRŸY™Y z°"¡~°{ÑIÀbÒIÔbd×bmÑbmÓboˆD�Nà!ˆDŒNð 	×Ñ eÐÔ,à"'ˆÕùòQ %Ns   ÃI9
rJ  c                 ó  — t        |t        j                  «      rOt        |j                  d¬«       |j
                  �+t        j                  j                  |j
                  d«       y y |rt        |d«      r|j                  «        y y y )Nç{®Gáz”?)Ústdr   r�   )
Ú
isinstancerz   rU  r   ÚweightrC   r›   Ú	constant_Úhasattrr�   )rN   ÚmrJ  s      rQ   Ú_init_weightszEfficientFormerV2._init_weights×  se   € Ü�aœŸ™Ô#Ü˜!Ÿ(™(¨Õ,Ø�v‰vÐ!Ü—‘×!Ñ! !§&¡&¨!Õ,ð "áœW QÐ(:Ô;Ø×ÑÕ ð <ˆ[rR   c                 óP   — | j                  t        | j                  |¬«      «       y )NrI  )Úapplyr   rj  )rN   rJ  s     rQ   rY  zEfficientFormerV2.init_weightsß  s   € Ø�
‰
”7˜4×-Ñ-¸;ÔGÕHrR   c                 ó^   — | j                  «       D ��ch c]  \  }}d|v sŒ|’Œ c}}S c c}}w )NrŒ   )Únamed_parameters)rN   r�   Ú_s      rQ   Úno_weight_decayz!EfficientFormerV2.no_weight_decayâ  s+   € à"×3Ñ3Ó5×Q‘d�a˜Ð9KÈqÒ9P’ÓQÐQùÓQs   ”)¡)c                 ó$   — t        dddg¬«      }|S )Nz^stem)z^stages\.(\d+)N)z^norm)iŸ† )rL  r-  )rN  )rN   ÚcoarseÚmatchers      rQ   Úgroup_matcherzEfficientFormerV2.group_matcheræ  s   € äØØ-Ð/CÐDô
ˆð ˆrR   c                 ó4   — | j                   D ]	  }||_        Œ y rT   )rP  r*  )rN   Úenabler_  s      rQ   Úset_grad_checkpointingz(EfficientFormerV2.set_grad_checkpointingî  s   € à—‘ò 	*ˆAØ#)ˆAÕ ñ	*rR   r—   c                 ó2   — | j                   | j                  fS rT   ©rV  rX  rž   s    rQ   Úget_classifierz EfficientFormerV2.get_classifieró  s   € à�y‰y˜$Ÿ.™.Ð(Ð(rR   c                 ó(  — || _         |�|| _        |dkD  r t        j                  | j                  |«      nt        j
                  «       | _        |dkD  r&t        j                  | j                  |«      | _        y t        j
                  «       | _        y )Nr   )r<  r7  rz   rU  rQ  rõ   rV  rX  )rN   r<  r7  s      rQ   Úreset_classifierz"EfficientFormerV2.reset_classifier÷  sq   € Ø&ˆÔØÐ"Ø*ˆDÔØALÈqÂ”B—I‘I˜d×/Ñ/°Ô=ÔVX×VaÑVaÓVcˆŒ	ØFQÐTUÂoœŸ™ 4×#4Ñ#4°kÓBˆ�Ô[]×[fÑ[fÓ[hˆ�rR   c                 ó   — || _         y rT   )rZ  )rN   rv  s     rQ   Úset_distilled_trainingz(EfficientFormerV2.set_distilled_trainingþ  s
   € à"(ˆÕrR   rV   ÚindicesrS  Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 ó&  — |dv sJ d«       ‚g }t        t        | j                  «      |«      \  }}	| j                  |«      }t        | j                  «      dz
  }
t        j
                  j                  «       s|s| j                  }n| j                  d|	dz    }t        |«      D ]O  \  }} ||«      }||v sŒ||
k(  r'|r| j                  |«      n|}|j                  |«       Œ?|j                  |«       ŒQ |r|S |
k(  r| j                  |«      }||fS )aï   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )ÚNCHWzOutput shape must be NCHW.r   N)
r   rM  rP  rL  r‰   r¶   r2  Ú	enumeraterS  rO  )rN   rV   r  rS  r€  r�  r‚  ÚintermediatesÚtake_indicesÚ	max_indexÚlast_idxrP  Úfeat_idxra  Úx_inters                  rQ   Úforward_intermediatesz'EfficientFormerV2.forward_intermediates  s  € ð* ˜YÑ&ÐDÐ(DÓDÐ&ØˆÜ"6´s¸4¿;¹;Ó7GÈÓ"QÑˆ�ið �I‰I�a‹Lˆä�t—{‘{Ó# aÑ'ˆÜ�9‰9×!Ñ!Ô#©:Ø—[‘[‰Fà—[‘[  )¨a¡-Ð0ˆFä(¨Ó0ò 	,‰OˆH�eÙ�a“ˆAØ˜<Ò'Ø˜xÒ'Ù.2˜dŸi™i¨œl¸�GØ!×(Ñ(¨Õ1à!×(Ñ(¨Õ+ð	,ñ Ø Ð à�xÒØ—	‘	˜!“ˆAà�-ÐÐrR   Ú
prune_normÚ
prune_headc                 óÖ   — t        t        | j                  «      |«      \  }}| j                  d|dz    | _        |rt        j                  «       | _        |r| j                  dd«       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   rX   )r   rM  rP  rz   rõ   rS  r|  )rN   r  r�  rŽ  r‡  rˆ  s         rQ   Úprune_intermediate_layersz+EfficientFormerV2.prune_intermediate_layers5  s]   € ô #7´s¸4¿;¹;Ó7GÈÓ"QÑˆ�iØ—k‘k . 9¨q¡=Ð1ˆŒÙÜŸ™›ˆDŒIÙØ×!Ñ! ! RÔ(ØÐrR   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rT   )rL  rP  rS  rU   s     rQ   Úforward_featuresz"EfficientFormerV2.forward_featuresE  s.   € Ø�I‰I�a‹LˆØ�K‰K˜‹NˆØ�I‰I�a‹LˆØˆrR   Ú
pre_logitsc                 ó6  — | j                   dk(  r|j                  d¬«      }| j                  |«      }|r|S | j                  |«      | j	                  |«      }}| j
                  r.| j                  r"t        j                  j                  «       s||fS ||z   dz  S )NrB  )r9   r6   rÀ   r9   )
r7  ÚmeanrT  rV  rX  rZ  r¸   r‰   r¶   r2  )rN   rV   r“  Úx_dists       rQ   Úforward_headzEfficientFormerV2.forward_headK  s‰   € Ø×Ñ˜uÒ$Ø—‘˜6�Ó"ˆAØ�N‰N˜1ÓˆÙØˆHØ—I‘I˜a“L $§.¡.°Ó"3ˆ6ˆØ×"Ò" t§}¢}¼U¿Y¹Y×=SÑ=SÔ=Uà�f�9Ðð ˜‘J !Ñ#Ð#rR   c                 óJ   — | j                  |«      }| j                  |«      }|S rT   )r’  r—  rU   s     rQ   rW   zEfficientFormerV2.forwardY  s'   € Ø×!Ñ! !Ó$ˆØ×Ñ˜aÓ ˆØˆrR   )r6   r)   rB  NNr3   rY   r  Úgeluéè  rû   rû   rû   r  r   TNNrÌ   rE  rT   )NFFr„  F)r   FT) rZ   r[   r\   r   r]   r	   r^   r   r_   r
  rK   rj  rY  r‰   r¶   Úignorerp  rt  rw  rz   rÑ   rz  r|  r~  rÎ   r   rŒ  r�  r’  r—  rW   r`   ra   s   @rQ   r   r   �  s  ø„ ð Ø47Ø$Ø48Ø6:ØYZØ+Ø"Ø#Ø#Ø!Ø$&Ø$&Ø6:ØØ!%ØØñ)S(à˜#˜s˜(‘OðS(ð ðS(ð ˜C  s¨C x¡Ð0Ñ1ð	S(ð
 ðS(ð !  s¨C x¡Ñ1ðS(ð " %¨¨c¨	Ñ"2Ñ3ðS(ð ˜e U¨5°#¨:Ñ%6¸¸eÀEÈ3ÀJÑ>OÐQTÐ>TÑ8UÐUÑVðS(ð ðS(ð ðS(ð ðS(ð ðS(ð ðS(ð "ðS(ð "ðS(ð  %-¨U¡Oð!S(ð" ð#S(ð$ õ%S(ñj!¨Dó !ñI¨ó Ið ‡Y�Y×ÑñRó ðRð ‡Y�Y×Ñòó ðð ‡Y�Y×Ñò*ó ð*ð ‡Y�Y×Ñð) §	¡	ò )ó ð)ñi¨Cð i¸hÀs¹mó ið ‡Y�Y×Ñò)ó ð)ð 8<ØØ$Ø$Ø',ñ1 à�|‰|ð1 ð ˜e C¨¨c© NÑ3Ñ4ð1 ð ð	1 ð
 ð1 ð ð1 ð !%ð1 ð 
ˆt�E—L‘LÑ! 5¨¯©°t¸E¿L¹LÑ7IÐ)IÑ#JÐJÑ	Kó1 ðj ./Ø$Ø#ñ	à˜3  S¡	˜>Ñ*ðð ðð ó	ò ñ$¨$ó $örR   c                 ó4   — | ddd dddt         t        dddd	œ|¥S )
Nrš  )r6   r)   r)   Tgffffffî?Úbicubicry  zstem.conv1.convz
apache-2.0)Úurlr<  Ú
input_sizeÚ	pool_sizeÚfixed_input_sizeÚcrop_pctÚinterpolationr•  rd  Ú
classifierÚ
first_convÚlicenser
   )rž  Úkwargss     rQ   Ú_cfgr¨  _  s7   € àØ¨=ÀtÐaeØ¨)Ü%Ô.BØ+Ð;LØñð ðð rR   ztimm/)Ú	hf_hub_id)z#efficientformerv2_s0.snap_dist_in1kz#efficientformerv2_s1.snap_dist_in1kz#efficientformerv2_s2.snap_dist_in1kz"efficientformerv2_l.snap_dist_in1kc                 óh   — |j                  dd«      }t        t        | |fdt        d|¬«      i|¤Ž}|S )NÚout_indices)r   r   r9   r6   Úfeature_cfgT)Úflatten_sequentialr«  )Úpopr   r   rN  )ÚvariantÚ
pretrainedr§  r«  Úmodels        rQ   Ú_create_efficientformerv2r²  {  sC   € Ø—*‘*˜]¨LÓ9€KÜ Ü˜7 Jñä¨D¸kÔJðð ñ€Eð €LrR   r—   c           	      óz   — t        t        d   t        d   ddt        d   ¬«      }t	        dd| it        |fi |¤Ž¤ŽS )Nr/   r9   rû   ©r4  r8  r(  r?  r:  r°  )Úefficientformerv2_s0©rN  ÚEfficientFormer_depthÚEfficientFormer_widthÚ EfficientFormer_expansion_ratiosr²  ©r°  r§  Ú
model_argss      rQ   rµ  rµ  „  óL   € äÜ$ TÑ*Ü(¨Ñ.ØØÜ3°DÑ9ô€Jô %ÑqÈ
ÐqÔVZÐ[eÑVpÐioÑVpÑqÐqrR   c           	      óz   — t        t        d   t        d   ddt        d   ¬«      }t	        dd| it        |fi |¤Ž¤ŽS )Nr.   r9   rû   r´  r°  )Úefficientformerv2_s1r¶  rº  s      rQ   r¾  r¾  �  r¼  rR   c           	      óz   — t        t        d   t        d   ddt        d   ¬«      }t	        dd| it        |fi |¤Ž¤ŽS )Nr-   r3   rc  r´  r°  )Úefficientformerv2_s2r¶  rº  s      rQ   rÀ  rÀ  œ  sL   € äÜ$ TÑ*Ü(¨Ñ.ØØÜ3°DÑ9ô€Jô %ÑqÈ
ÐqÔVZÐ[eÑVpÐioÑVpÑqÐqrR   c           	      óz   — t        t        d   t        d   ddt        d   ¬«      }t	        dd| it        |fi |¤Ž¤ŽS )Nr,   r8   gš™™™™™¹?r´  r°  )Úefficientformerv2_lr¶  rº  s      rQ   rÂ  rÂ  ¨  sL   € äÜ$ SÑ)Ü(¨Ñ-ØØÜ3°CÑ8ô€Jô %ÑpÀzÐpÔUYÐZdÑUoÐhnÑUoÑpÐprR   )rX   rE  )>r	  rw   Ú	functoolsr   Útypingr   r   r   r   r   r	   r‰   Útorch.nnrz   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_manipulater   Ú	_registryr   r   Ú__all__r¸  r·  r¹  rÑ   r;   rc   rÔ   rß   rï   rú   r  r,  r  r!  r   r¨  Údefault_cfgsr²  rµ  r¾  rÀ  rÂ  rô   rR   rQ   ú<module>rÎ     s  ðñó  Ý ß ;× ;ã Ý ç A÷÷ ÷ ó õ +Ý +Ý 'ß <ð Ð
€ð 
Ø
Ø
Ø
ñ	Ð ð 
Ø
Ø
Ø
ñ	Ð ð 
_Ø
PØ
AØ
2ñ	$Ð  ô#ˆr�y‰yô #ôL{�%—(‘(—/‘/ô {ô|�u—x‘x—‘ô ô,q˜EŸH™HŸO™Oô qôh/�—‘ô /ôd6�b—i‘iô 6ôr4˜RŸY™Yô 4ôn!
ˆB�M‰Mô !
ôHE˜RŸY™Yô EôP[˜Ÿ	™	ô [ó|	ñ %Ù+/Øô,ñ ,0Øô,ñ ,0Øô,ñ +/Øô+ñ&ó €ó ð ñrÐ8Iò ró ðrð ñrÐ8Iò ró ðrð ñrÐ8Iò ró ðrð ñqÐ7Hò qó ñqrR   