Ë
    éÓ^j7\  ã                   ó  — d Z ddlmZmZmZmZmZmZ ddlZddl	m
Z
 ddlmZmZ ddlmZmZmZmZmZmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZ dgZ ddddœZ!ddddœZ" G d„ dej                  jF                  «      Z$ G d„ de
jJ                  «      Z& G d„ de
jF                  «      Z' G d„ de
jF                  «      Z( G d„ de
jF                  «      Z) G d„ de
jF                  «      Z* G d„ d e
jF                  «      Z+ G d!„ d"e
jF                  «      Z, G d#„ d$e
jF                  «      Z- G d%„ de
jF                  «      Z.d&„ Z/d0d'„Z0 e e0d(¬)«       e0d(¬)«       e0d(¬)«      d*œ«      Z1d1d+„Z2ed1d,e.fd-„«       Z3ed1d,e.fd.„«       Z4ed1d,e.fd/„«       Z5y)2aû   EfficientFormer

@article{li2022efficientformer,
  title={EfficientFormer: Vision Transformers at MobileNet Speed},
  author={Li, Yanyu and Yuan, Geng and Wen, Yang and Hu, Eric and Evangelidis, Georgios and Tulyakov,
   Sergey and Wang, Yanzhi and Ren, Jian},
  journal={arXiv preprint arXiv:2206.01191},
  year={2022}
}

Based on Apache 2.0 licensed code at https://github.com/snap-research/EfficientFormer, Copyright (c) 2022 Snap Inc.

Modifications and timm support by / Copyright 2022, Ross Wightman
é    )ÚDictÚListÚOptionalÚTupleÚTypeÚUnionN©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚDropPathÚ
LayerScaleÚLayerScale2dÚMlpÚcalculate_drop_path_ratesÚtrunc_normal_Ú	to_2tupleÚndgridé   )Úbuild_model_with_cfg)Úfeature_take_indices)Úcheckpoint_seq)Úgenerate_default_cfgsÚregister_modelÚEfficientFormer©é0   é`   éà   iÀ  )é@   é€   i@  i   )r   éÀ   é€  i   )Úl1Úl3Úl7©é   é   é   é   )r*   r*   é   r)   )r)   r)   é   é   c                   óà   ‡ — e Zd ZU eeej                  f   ed<   	 	 	 	 	 	 	 ddededede	def
ˆ fd„Z
 ej                  «       dˆ fd„	«       Zd	ej                  d
ej                  fd„Zd„ Zˆ xZS )Ú	AttentionÚattention_bias_cacheÚdimÚkey_dimÚ	num_headsÚ
attn_ratioÚ
resolutionc           
      ó°  •— ||dœ}t         ‰| �  «        || _        |dz  | _        || _        ||z  | _        t        ||z  «      | _        | j                  |z  | _        || _	        t        j                  || j
                  dz  | j                  z   fi |¤Ž| _        t        j                  | j                  |fi |¤Ž| _        t        |«      }t        j                   t#        t        j$                  |d   |t        j&                  ¬«      t        j$                  |d   |t        j&                  ¬«      «      «      j)                  d«      }	|	dd d …d f   |	dd d d …f   z
  j+                  «       }
|
d   |d   z  |
d   z   }
t        j                  j-                  t        j.                  ||d   |d   z  fi |¤Ž«      | _        | j3                  d|
«       i | _        y )N©ÚdeviceÚdtypeg      à¿r(   r   r   .Úattention_bias_idxs)ÚsuperÚ__init__r3   Úscaler2   Úkey_attn_dimÚintÚval_dimÚval_attn_dimr4   ÚnnÚLinearÚqkvÚprojr   ÚtorchÚstackr   ÚarangeÚlongÚflattenÚabsÚ	ParameterÚzerosÚattention_biasesÚregister_bufferr0   )Úselfr1   r2   r3   r4   r5   r8   r9   ÚddÚposÚrel_posÚ	__class__s              €úf/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/efficientformer.pyr<   zAttention.__init__7   s¨  ø€ ð ¨Ñ/ˆÜ‰ÑÔØ"ˆŒØ ‘_ˆŒ
ØˆŒØ# iÑ/ˆÔÜ˜:¨Ñ/Ó0ˆŒØ ŸL™L¨9Ñ4ˆÔØ$ˆŒä—9‘9˜S $×"3Ñ"3°aÑ"7¸$×:KÑ:KÑ"KÑRÈrÑRˆŒÜ—I‘I˜d×/Ñ/°Ñ;¸Ñ;ˆŒ	ä˜zÓ*ˆ
Ü�k‰kœ&Ü�L‰L˜ A™¨v¼U¿Z¹ZÔHÜ�L‰L˜ A™¨v¼U¿Z¹ZÔHó
ó ÷ ‰7�1‹:ð 	ð �sšA˜t�|Ñ$ s¨3°²a¨<Ñ'8Ñ8×=Ñ=Ó?ˆØ˜1‘: 
¨1¡Ñ-°¸±Ñ;ˆÜ %§¡× 2Ñ 2´5·;±;¸yÈ*ÐUVÉ-ÐZdÐefÑZgÑJgÑ3nÐkmÑ3nÓ oˆÔØ×ÑÐ2°GÔ<Ø$&ˆÕ!ó    c                 óR   •— t         ‰| �  |«       |r| j                  ri | _        y y y ©N)r;   Útrainr0   )rP   ÚmoderT   s     €rU   rY   zAttention.trainY   s)   ø€ ä‰‰�dÔÙ�D×-Ò-Ø(*ˆDÕ%ð .ˆ4rV   r8   Úreturnc                 ó4  — t         j                  j                  «       s| j                  r| j                  d d …| j
                  f   S t        |«      }|| j                  vr*| j                  d d …| j
                  f   | j                  |<   | j                  |   S rX   )rF   ÚjitÚ
is_tracingÚtrainingrN   r:   Ústrr0   )rP   r8   Ú
device_keys      rU   Úget_attention_biaseszAttention.get_attention_biases_   sƒ   € Ü�9‰9×ÑÔ! T§]¢]Ø×(Ñ(ª¨D×,DÑ,DÐ)DÑEÐEä˜V›ˆJØ ×!:Ñ!:Ñ:Ø8<×8MÑ8MÊaÐQU×QiÑQiÐNiÑ8j�×)Ñ)¨*Ñ5Ø×,Ñ,¨ZÑ8Ð8rV   c                 ó>  — |j                   \  }}}| j                  |«      }|j                  ||| j                  d«      j	                  dddd«      }|j                  | j                  | j                  | j                  gd¬«      \  }}}||j                  dd«      z  | j                  z  }	|	| j                  |j                  «      z   }	|	j                  d¬«      }	|	|z  j                  dd«      j                  ||| j                  «      }| j                  |«      }|S )Néÿÿÿÿr   r(   r   r'   ©r1   éþÿÿÿ)ÚshaperD   Úreshaper3   ÚpermuteÚsplitr2   r@   Ú	transposer=   rb   r8   ÚsoftmaxrA   rE   )
rP   ÚxÚBÚNÚCrD   ÚqÚkÚvÚattns
             rU   ÚforwardzAttention.forwardh   sö   € Ø—'‘'‰ˆˆ1ˆaØ�h‰h�q‹kˆØ�k‰k˜!˜Q §¡°Ó3×;Ñ;¸A¸qÀ!ÀQÓGˆØ—)‘)˜TŸ\™\¨4¯<©<¸¿¹ÐFÈA�)ÓN‰ˆˆ1ˆaà�A—K‘K  BÓ'Ñ'¨4¯:©:Ñ5ˆØ�d×/Ñ/°·±Ó9Ñ9ˆà�|‰| ˆ|Ó#ˆØ�A‰X× Ñ   AÓ&×.Ñ.¨q°!°T×5FÑ5FÓGˆØ�I‰I�a‹LˆØˆrV   )r"   é    r-   r*   é   NN©T)Ú__name__Ú
__module__Ú__qualname__r   r`   rF   ÚTensorÚ__annotations__r?   Úfloatr<   Úno_gradrY   r8   rb   ru   Ú__classcell__©rT   s   @rU   r/   r/   4   s    ø… Ø˜s E§L¡LÐ0Ñ1Ó1ð ØØØ !ØØØñ 'àð 'ð ð 'ð ð	 'ð
 ð 'ð õ 'ðD €U‡]�]ƒ_ô+ó ð+ð
9¨5¯<©<ð 9¸E¿L¹Ló 9örV   r/   c            
       ó–   ‡ — e Zd Zej                  ej
                  ddfdededeej                     deej                     fˆ fd„Z	ˆ xZ
S )ÚStem4NÚin_chsÚout_chsÚ	act_layerÚ
norm_layerc           
      ó¸  •— ||dœ}t         ‰| �  «        d| _        | j                  dt	        j
                  ||dz  fddddœ|¤Ž«       | j                  d ||dz  fi |¤Ž«       | j                  d	 |«       «       | j                  d
t	        j
                  |dz  |fddddœ|¤Ž«       | j                  d ||fi |¤Ž«       | j                  d |«       «       y )Nr7   r*   Úconv1r(   r'   r   ©Úkernel_sizeÚstrideÚpaddingÚnorm1Úact1Úconv2Únorm2Úact2)r;   r<   rŒ   Ú
add_modulerB   ÚConv2d)	rP   r„   r…   r†   r‡   r8   r9   rQ   rT   s	           €rU   r<   zStem4.__init__x   sÑ   ø€ ð ¨Ñ/ˆÜ‰ÑÔØˆŒà�‰˜¤§¡¨6°7¸a±<Ð!jÈQÐWXÐbcÑ!jÐgiÑ!jÔkØ�‰˜¡¨G°q©LÑ!?¸BÑ!?Ô@Ø�‰˜¡	£Ô,Ø�‰˜¤§¡¨7°a©<¸Ð!kÈaÐXYÐcdÑ!kÐhjÑ!kÔlØ�‰˜¡¨GÑ!:°rÑ!:Ô;Ø�‰˜¡	£Õ,rV   )ry   rz   r{   rB   ÚReLUÚBatchNorm2dr?   r   ÚModuler<   r€   r�   s   @rU   rƒ   rƒ   w   sY   ø„ ð
 *,¯©Ø*,¯.©.ØØñ-àð-ð ð-ð ˜BŸI™I‘ð	-ð
 ˜RŸY™Y™÷-ñ -rV   rƒ   c                   ó„   ‡ — e Zd ZdZdddej
                  ddfdedededed	ee   d
eej                     fˆ fd„Z
d„ Zˆ xZS )Ú
DownsamplezŽ
    Downsampling via strided conv w/ norm
    Input: tensor in shape [B, C, H, W]
    Output: tensor in shape [B, C, H/stride, W/stride]
    r'   r(   Nr„   r…   r‹   rŒ   r�   r‡   c	                 ó–   •— ||dœ}	t         ‰
| �  «        |€|dz  }t        j                  ||f|||dœ|	¤Ž| _         ||fi |	¤Ž| _        y )Nr7   r(   rŠ   )r;   r<   rB   r”   ÚconvÚnorm)rP   r„   r…   r‹   rŒ   r�   r‡   r8   r9   rQ   rT   s             €rU   r<   zDownsample.__init__”   s\   ø€ ð ¨Ñ/ˆÜ‰ÑÔØˆ?Ø! QÑ&ˆGÜ—I‘I˜f gÐm¸;ÈvÐ_fÑmÐjlÑmˆŒ	Ù˜wÑ-¨"Ñ-ˆ�	rV   c                 óJ   — | j                  |«      }| j                  |«      }|S rX   )r›   rœ   ©rP   rm   s     rU   ru   zDownsample.forward¦   s!   € Ø�I‰I�a‹LˆØ�I‰I�a‹LˆØˆrV   )ry   rz   r{   Ú__doc__rB   r–   r?   r   r   r—   r<   ru   r€   r�   s   @rU   r™   r™   �   sp   ø„ ñð  !ØØ%)Ø*,¯.©.ØØñ.àð.ð ð.ð ð	.ð
 ð.ð ˜c‘]ð.ð ˜RŸY™Y™õ.ö$rV   r™   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚFlatc                 ó"   •— t         ‰| �  «        y rX   )r;   r<   )rP   rT   s    €rU   r<   zFlat.__init__®   s   ø€ Ü‰ÑÕrV   c                 óH   — |j                  d«      j                  dd«      }|S )Nr(   r   )rJ   rk   rž   s     rU   ru   zFlat.forward±   s!   € Ø�I‰I�a‹L×"Ñ" 1 aÓ(ˆØˆrV   )ry   rz   r{   r<   ru   r€   r�   s   @rU   r¡   r¡   ¬   s   ø„ ôörV   r¡   c                   ó0   ‡ — e Zd ZdZddefˆ fd„Zd„ Zˆ xZS )ÚPoolingzP
    Implementation of pooling for PoolFormer
    --pool_size: pooling size
    Ú	pool_sizec                 ód   •— t         ‰| �  «        t        j                  |d|dz  d¬«      | _        y )Nr   r(   F)rŒ   r�   Úcount_include_pad)r;   r<   rB   Ú	AvgPool2dÚpool)rP   r¦   rT   s     €rU   r<   zPooling.__init__¼   s)   ø€ Ü‰ÑÔÜ—L‘L °1¸iÈ1¹nÐ`eÔfˆ�	rV   c                 ó*   — | j                  |«      |z
  S rX   )rª   rž   s     rU   ru   zPooling.forwardÀ   s   € Ø�y‰y˜‹|˜aÑÐrV   )r'   )ry   rz   r{   rŸ   r?   r<   ru   r€   r�   s   @rU   r¥   r¥   ¶   s   ø„ ññ
g #õ gö rV   r¥   c                   óº   ‡ — e Zd ZdZddej
                  ej                  dddfdedee   dee   de	ej                     de	ej                     d	efˆ fd
„Zd„ Zˆ xZS )ÚConvMlpWithNormz`
    Implementation of MLP with 1*1 convolutions.
    Input: tensor with shape [B, C, H, W]
    Nç        Úin_featuresÚhidden_featuresÚout_featuresr†   r‡   Údropc	                 ó”  •— ||dœ}	t         ‰
| �  «        |xs |}|xs |}t        j                  ||dfi |	¤Ž| _        |�	 ||fi |	¤Žnt        j
                  «       | _         |«       | _        t        j                  ||dfi |	¤Ž| _        |�	 ||fi |	¤Žnt        j
                  «       | _	        t        j                  |«      | _        y )Nr7   r   )r;   r<   rB   r”   Úfc1ÚIdentityrŽ   ÚactÚfc2r‘   ÚDropoutr²   )rP   r¯   r°   r±   r†   r‡   r²   r8   r9   rQ   rT   s             €rU   r<   zConvMlpWithNorm.__init__Ê   s½   ø€ ð ¨Ñ/ˆÜ‰ÑÔØ#Ò2 {ˆØ)Ò8¨[ˆÜ—9‘9˜[¨/¸1ÑCÀÑCˆŒØ:DÐ:P‘Z Ñ6°2Ò6ÔVX×VaÑVaÓVcˆŒ
Ù“;ˆŒÜ—9‘9˜_¨l¸AÑDÀÑDˆŒØ7AÐ7M‘Z Ñ3°Ò3ÔSU×S^ÑS^ÓS`ˆŒ
Ü—J‘J˜tÓ$ˆ�	rV   c                 óô   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j	                  |«      }| j                  |«      }| j                  |«      }|S rX   )r´   rŽ   r¶   r²   r·   r‘   rž   s     rU   ru   zConvMlpWithNorm.forwardà   sb   € Ø�H‰H�Q‹KˆØ�J‰J�q‹MˆØ�H‰H�Q‹KˆØ�I‰I�a‹LˆØ�H‰H�Q‹KˆØ�J‰J�q‹MˆØ�I‰I�a‹LˆØˆrV   )ry   rz   r{   rŸ   rB   ÚGELUr–   r?   r   r   r—   r~   r<   ru   r€   r�   s   @rU   r­   r­   Ä   sƒ   ø„ ñð .2Ø*.Ø)+¯©Ø*,¯.©.ØØØñ%àð%ð & c™]ð%ð # 3™-ð	%ð
 ˜BŸI™I‘ð%ð ˜RŸY™Y™ð%ð õ%ö,rV   r­   c                   ó°   ‡ — e Zd Zdej                  ej
                  dddddfdededeej                     deej                     d	ed
edefˆ fd„Z
d„ Zˆ xZS )ÚMetaBlock1dç      @r®   çñhãˆµøä>Nr1   Ú	mlp_ratior†   r‡   Ú	proj_dropÚ	drop_pathÚlayer_scale_init_valuec
                 óÀ  •— ||	dœ}
t         ‰| �  «         ||fi |
¤Ž| _        t        |fi |
¤Ž| _        t        ||fi |
¤Ž| _        |dkD  rt        |«      nt        j                  «       | _
         ||fi |
¤Ž| _        t        d|t        ||z  «      ||dœ|
¤Ž| _        t        ||fi |
¤Ž| _        |dkD  rt        |«      | _        y t        j                  «       | _        y )Nr7   r®   )r¯   r°   r†   r²   © )r;   r<   rŽ   r/   Útoken_mixerr   Úls1r   rB   rµ   Ú
drop_path1r‘   r   r?   ÚmlpÚls2Ú
drop_path2)rP   r1   r¿   r†   r‡   rÀ   rÁ   rÂ   r8   r9   rQ   rT   s              €rU   r<   zMetaBlock1d.__init__í   sà   ø€ ð ¨Ñ/ˆÜ‰ÑÔÙ Ñ* rÑ*ˆŒ
Ü$ SÑ/¨BÑ/ˆÔÜ˜cÐ#9Ñ@¸RÑ@ˆŒØ1:¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒá Ñ* rÑ*ˆŒ
Üð 
ØÜ  i¡Ó0ØØñ	
ð
 ñ
ˆŒô ˜cÐ#9Ñ@¸RÑ@ˆŒØ1:¸R²œ( 9Ó-ˆ�ÄRÇ[Á[Ã]ˆ�rV   c           
      ó
  — || j                  | j                  | j                  | j                  |«      «      «      «      z   }|| j	                  | j                  | j                  | j                  |«      «      «      «      z   }|S rX   )rÇ   rÆ   rÅ   rŽ   rÊ   rÉ   rÈ   r‘   rž   s     rU   ru   zMetaBlock1d.forward  sc   € Ø�—‘ §¡¨×)9Ñ)9¸$¿*¹*ÀQ»-Ó)HÓ IÓJÑJˆØ�—‘ §¡¨¯©°$·*±*¸Q³-Ó)@Ó AÓBÑBˆØˆrV   )ry   rz   r{   rB   rº   Ú	LayerNormr?   r~   r   r—   r<   ru   r€   r�   s   @rU   r¼   r¼   ë   s‹   ø„ ð
  "Ø)+¯©Ø*,¯,©,Ø!Ø!Ø,0ØØñSàðSð ðSð ˜BŸI™I‘ð	Sð
 ˜RŸY™Y™ðSð ðSð ðSð %*õSö<rV   r¼   c                   ó¶   ‡ — e Zd Zddej                  ej
                  dddddf	dededed	eej                     d
eej                     dededefˆ fd„Z
d„ Zˆ xZS )ÚMetaBlock2dr'   r½   r®   r¾   Nr1   r¦   r¿   r†   r‡   rÀ   rÁ   rÂ   c                 óŠ  •— |	|
dœ}t         ‰| �  «        t        |¬«      | _        t	        ||fi |¤Ž| _        |dkD  rt        |«      nt        j                  «       | _	        t        |ft        ||z  «      |||dœ|¤Ž| _        t	        ||fi |¤Ž| _        |dkD  rt        |«      | _        y t        j                  «       | _        y )Nr7   )r¦   r®   )r°   r†   r‡   r²   )r;   r<   r¥   rÅ   r   rÆ   r   rB   rµ   rÇ   r­   r?   rÈ   rÉ   rÊ   )rP   r1   r¦   r¿   r†   r‡   rÀ   rÁ   rÂ   r8   r9   rQ   rT   s               €rU   r<   zMetaBlock2d.__init__  s¼   ø€ ð ¨Ñ/ˆÜ‰ÑÔÜ"¨YÔ7ˆÔÜ Ð%;ÑB¸rÑBˆŒØ1:¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒä"Øð
ä  i¡Ó0ØØ!Øñ
ð ñ
ˆŒô   Ð%;ÑB¸rÑBˆŒØ1:¸R²œ( 9Ó-ˆ�ÄRÇ[Á[Ã]ˆ�rV   c                 óÎ   — || j                  | j                  | j                  |«      «      «      z   }|| j                  | j	                  | j                  |«      «      «      z   }|S rX   )rÇ   rÆ   rÅ   rÊ   rÉ   rÈ   rž   s     rU   ru   zMetaBlock2d.forward1  sR   € Ø�—‘ §¡¨×)9Ñ)9¸!Ó)<Ó =Ó>Ñ>ˆØ�—‘ §¡¨¯©°!«Ó 5Ó6Ñ6ˆØˆrV   )ry   rz   r{   rB   rº   r–   r?   r~   r   r—   r<   ru   r€   r�   s   @rU   rÎ   rÎ     s™   ø„ ð
 Ø!Ø)+¯©Ø*,¯.©.Ø!Ø!Ø,0ØØñSàðSð ðSð ð	Sð
 ˜BŸI™I‘ðSð ˜RŸY™Y™ðSð ðSð ðSð %*õSö<rV   rÎ   c                   óþ   ‡ — e Zd Zddddej                  ej
                  ej                  dddddfded	ed
edededede	de
ej                     de
ej                     de
ej                     de	de	de	fˆ fd„Zd„ Zˆ xZS )ÚEfficientFormerStageTr   r'   r½   r®   r¾   Nr1   Údim_outÚdepthÚ
downsampleÚnum_vitr¦   r¿   r†   r‡   Únorm_layer_clrÀ   rÁ   rÂ   c                 ó8  •— ||dœ}t         ‰| �  «        d| _        |rt        d|||	dœ|¤Ž| _        |}n ||k(  sJ ‚t        j                  «       | _        g }|r||k\  r|j                  t        «       «       t        |«      D ]  }||z
  dz
  }|r+||kD  r&|j                  t        |f|||
|||   |dœ|¤Ž«       Œ8|j                  t        |f||||	|||   |dœ|¤Ž«       |sŒa||k(  sŒg|j                  t        «       «       Œ� t        j                  |Ž | _        y )Nr7   F)r„   r…   r‡   r   )r¿   r†   r‡   rÀ   rÁ   rÂ   )r¦   r¿   r†   r‡   rÀ   rÁ   rÂ   rÄ   )r;   r<   Úgrad_checkpointingr™   rÕ   rB   rµ   Úappendr¡   Úranger¼   rÎ   Ú
SequentialÚblocks)rP   r1   rÓ   rÔ   rÕ   rÖ   r¦   r¿   r†   r‡   r×   rÀ   rÁ   rÂ   r8   r9   rQ   rÝ   Ú	block_idxÚ
remain_idxrT   s                       €rU   r<   zEfficientFormerStage.__init__9  sQ  ø€ ð$ ¨Ñ/ˆÜ‰ÑÔØ"'ˆÔáÜ(Ðb°¸WÐQ[ÑbÐ_aÑbˆDŒOØ‰Cà˜'’>Ð!�>Ü Ÿk™k›mˆDŒOàˆÙ�w %Ò'Ø�M‰Mœ$›&Ô!ä˜u›ò 	*ˆIØ Ñ*¨QÑ.ˆJÙ˜7 ZÒ/Ø—‘ÜØð	à"+Ø"+Ø#0Ø"+Ø"+¨IÑ"6Ø/Eñ	ð ñ	õ
ð —‘ÜØð
à"+Ø"+Ø"+Ø#-Ø"+Ø"+¨IÑ"6Ø/Eñ
ð ñ
ôò ˜w¨*Ó4Ø—M‘M¤$£&Õ)ð9	*ô< —m‘m VÐ,ˆ�rV   c                 óÎ   — | j                  |«      }| j                  r6t        j                  j	                  «       st        | j                  |«      }|S | j                  |«      }|S rX   )rÕ   rÙ   rF   r]   Úis_scriptingr   rÝ   rž   s     rU   ru   zEfficientFormerStage.forwardz  sS   € Ø�O‰O˜AÓˆØ×"Ò"¬5¯9©9×+AÑ+AÔ+CÜ˜tŸ{™{¨AÓ.ˆAð ˆð —‘˜A“ˆAØˆrV   )ry   rz   r{   rB   rº   r–   rÌ   r?   Úboolr~   r   r—   r<   ru   r€   r�   s   @rU   rÒ   rÒ   7  sØ   ø„ ð  $ØØØ!Ø)+¯©Ø*,¯.©.Ø-/¯\©\Ø!Ø!Ø,0ØØñ!?-àð?-ð ð?-ð ð	?-ð
 ð?-ð ð?-ð ð?-ð ð?-ð ˜BŸI™I‘ð?-ð ˜RŸY™Y™ð?-ð   §	¡	™?ð?-ð ð?-ð ð?-ð %*õ?-öBrV   rÒ   c            !       ó†  ‡ — e Zd Zdddddddddd	ej                  ej
                  ej                  d
d
d
ddfdeedf   deedf   dedede	de
eedf      dededededeej                     deej                     deej                     dededef ˆ fd„Zd„ Zej$                  j&                  d„ «       Zej$                  j&                  d3d„«       Zej$                  j&                  d4d „«       Zej$                  j&                  d!ej                  fd"„«       Zd5dede
e	   fd#„Zej$                  j&                  d4d$„«       Z	 	 	 	 	 d6d%ej4                  d&e
eeee   f      d'ed(ed)e	d*ed!eeej4                     eej4                  eej4                     f   f   fd+„Z	 	 	 d7d&eeee   f   d,ed-efd.„Zd/„ Zd3d0efd1„Z d2„ Z!ˆ xZ"S )8r   r&   r   r'   éè  ÚavgNr   r*   r¾   r®   Údepths.Ú
embed_dimsÚin_chansÚnum_classesÚglobal_poolÚdownsamplesrÖ   Ú
mlp_ratiosr¦   rÂ   r†   r‡   r×   Ú	drop_rateÚproj_drop_rateÚdrop_path_ratec                 ó&  •— t         ‰| �  «        ||dœ}|| _        || _        || _        t        ||d   fd|i|¤Ž| _        |d   }t        |«      | _        | j                  dz
  }t        ||d¬«      }|xs dd| j                  dz
  z  z   }g }g | _
        t        | j                  «      D ]r  }t        |||   ||   f||   ||k(  r|nd|	|||||||   |
d	œ
|¤Ž}||   }|j                  |«       | xj                  t        ||   d
|d
z   z  d|› �¬«      gz  c_
        Œt t        j                   |Ž | _        |d   x| _        | _         || j$                  fi |¤Ž| _        t        j*                  |«      | _        |dkD  r!t        j.                  | j$                  |fi |¤Žnt        j0                  «       | _        |dkD  rt        j.                  |d   |fi |¤Žnt        j0                  «       | _        d| _        | j9                  | j:                  «       y )Nr7   r   r‡   r   T)Ú	stagewise©Frx   )
rÕ   rÖ   r¦   r¿   r†   r×   r‡   rÀ   rÁ   rÂ   r(   ústages.)Únum_chsÚ	reductionÚmodulerd   F)r;   r<   ré   rè   rê   rƒ   ÚstemÚlenÚ
num_stagesr   Úfeature_inforÛ   rÒ   rÚ   ÚdictrB   rÜ   ÚstagesÚnum_featuresÚhead_hidden_sizerœ   r¸   Ú	head_droprC   rµ   ÚheadÚ	head_distÚdistilled_trainingÚapplyÚ_init_weights)rP   ræ   rç   rè   ré   rê   rë   rÖ   rì   r¦   rÂ   r†   r‡   r×   rí   rî   rï   r8   r9   ÚkwargsrQ   Úprev_dimÚ
last_stageÚdprrü   ÚiÚstagerT   s                              €rU   r<   zEfficientFormer.__init__…  s.  ø€ ô, 	‰ÑÔØ¨Ñ/ˆØ&ˆÔØ ˆŒØ&ˆÔä˜( J¨q¡MÑO¸jÐOÈBÑOˆŒ	Ø˜a‘=ˆô ˜f›+ˆŒØ—_‘_ qÑ(ˆ
Ü'¨¸È$ÔOˆØ!ÒO X°¸4¿?¹?ÈQÑ;NÑ0OÑ%OˆØˆØˆÔÜ�t—‘Ó'ò 	iˆAÜ(ØØ˜1‘Ø�q‘	ðð ' q™>Ø#$¨
¢?™¸Ø#Ø$Ø#Ø+Ø%Ø(Ø˜a™&Ø'=ñð ñˆEð  " !‘}ˆHØ�M‰M˜%Ô Ø×Ò¤$¨z¸!©}ÈÈAÈaÉCÉÐ[bÐcdÐbeÐYfÔ"gÐ!hÑhÖð'	iô( —m‘m VÐ,ˆŒð 5?¸r±NÐBˆÔ˜DÔ1Ù! $×"3Ñ"3Ñ:°rÑ:ˆŒ	ÜŸ™ IÓ.ˆŒØGRÐUVÂ”B—I‘I˜d×/Ñ/°ÑCÀÒCÔ\^×\gÑ\gÓ\iˆŒ	àITÐWXÊœŸ™ :¨b¡>°;ÑEÀ"ÒEÔ^`×^iÑ^iÓ^kˆŒØ"'ˆÔà�
‰
�4×%Ñ%Õ&rV   c                 ó  — t        |t        j                  «      rjt        |j                  d¬«       t        |t        j                  «      r8|j
                  �+t        j                  j                  |j
                  d«       y y y y )Ng{®Gáz”?)Ústdr   )Ú
isinstancerB   rC   r   ÚweightÚbiasÚinitÚ	constant_)rP   Úms     rU   r  zEfficientFormer._init_weightsÍ  sZ   € Ü�aœŸ™Ô#Ü˜!Ÿ(™(¨Õ,Ü˜!œRŸY™YÔ'¨A¯F©FÐ,>Ü—‘×!Ñ! !§&¡&¨!Õ,ð -?Ð'ð $rV   c                 ó^   — | j                  «       D ��ch c]  \  }}d|v sŒ|’Œ c}}S c c}}w )NrN   )Únamed_parameters)rP   rr   Ú_s      rU   Úno_weight_decayzEfficientFormer.no_weight_decayÓ  s+   € à"×3Ñ3Ó5×Q‘d�a˜Ð9KÈqÒ9P’ÓQÐQùÓQs   ”)¡)c                 ó$   — t        dddg¬«      }|S )Nz^stem)z^stages\.(\d+)N)z^norm)iŸ† )r÷   rÝ   )rû   )rP   ÚcoarseÚmatchers      rU   Úgroup_matcherzEfficientFormer.group_matcher×  s   € äØØ-Ð/CÐDô
ˆð ˆrV   c                 ó4   — | j                   D ]	  }||_        Œ y rX   )rü   rÙ   )rP   ÚenableÚss      rU   Úset_grad_checkpointingz&EfficientFormer.set_grad_checkpointingß  s   € à—‘ò 	*ˆAØ#)ˆAÕ ñ	*rV   r[   c                 ó2   — | j                   | j                  fS rX   ©r   r  )rP   s    rU   Úget_classifierzEfficientFormer.get_classifierä  s   € à�y‰y˜$Ÿ.™.Ð(Ð(rV   c                 ó(  — || _         |�|| _        |dkD  r t        j                  | j                  |«      nt        j
                  «       | _        |dkD  r&t        j                  | j                  |«      | _        y t        j
                  «       | _        y )Nr   )ré   rê   rB   rC   rý   rµ   r   r  )rP   ré   rê   s      rU   Úreset_classifierz EfficientFormer.reset_classifierè  sq   € Ø&ˆÔØÐ"Ø*ˆDÔØALÈqÂ”B—I‘I˜d×/Ñ/°Ô=ÔVX×VaÑVaÓVcˆŒ	ØFQÐTUÂoœŸ™ 4×#4Ñ#4°kÓBˆ�Ô[]×[fÑ[fÓ[hˆ�rV   c                 ó   — || _         y rX   )r  )rP   r  s     rU   Úset_distilled_trainingz&EfficientFormer.set_distilled_trainingï  s
   € à"(ˆÕrV   rm   Úindicesrœ   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc           	      óº  — |dv sJ d«       ‚g }t        t        | j                  «      |«      \  }}	| j                  |«      }|j                  \  }
}}}| j
                  dz
  }t        j                  j                  «       s|s| j                  }n| j                  d|	dz    }d}t        |«      D ]�  \  }} ||«      }||k  r|j                  \  }
}}}||v sŒ)||k(  rQ|r| j                  |«      n|}|j                  |j                  |
|dz  |dz  d«      j                  dddd«      «       Œ|j                  |«       Œ‘ |r|S ||k(  r| j                  |«      }||fS )	aï   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )ÚNCHWzOutput shape must be NCHW.r   Nr   r(   rd   r'   )r   rø   rü   r÷   rg   rù   rF   r]   rá   Ú	enumeraterœ   rÚ   rh   ri   )rP   rm   r&  rœ   r'  r(  r)  ÚintermediatesÚtake_indicesÚ	max_indexrn   rp   ÚHÚWÚlast_idxrü   Úfeat_idxr
  Úx_inters                      rU   Úforward_intermediatesz%EfficientFormer.forward_intermediatesó  sg  € ð* ˜YÑ&ÐDÐ(DÓDÐ&ØˆÜ"6´s¸4¿;¹;Ó7GÈÓ"QÑˆ�ið �I‰I�a‹LˆØ—W‘W‰
ˆˆ1ˆa�à—?‘? QÑ&ˆÜ�9‰9×!Ñ!Ô#©:Ø—[‘[‰Fà—[‘[  )¨a¡-Ð0ˆFØˆÜ(¨Ó0ò 		,‰OˆH�eÙ�a“ˆAØ˜(Ò"ØŸW™W‘
��1�a˜Ø˜<Ò'Ø˜xÒ'Ù.2˜dŸi™i¨œl¸�GØ!×(Ñ(¨¯©¸¸AÀ¹FÀAÈÁFÈBÓ)O×)WÑ)WÐXYÐ[\Ð^_ÐabÓ)cÕdà!×(Ñ(¨Õ+ð		,ñ Ø Ð à�xÒØ—	‘	˜!“ˆAà�-ÐÐrV   Ú
prune_normÚ
prune_headc                 óÖ   — t        t        | j                  «      |«      \  }}| j                  d|dz    | _        |rt        j                  «       | _        |r| j                  dd«       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   Ú )r   rø   rü   rB   rµ   rœ   r#  )rP   r&  r6  r7  r.  r/  s         rU   Úprune_intermediate_layersz)EfficientFormer.prune_intermediate_layers)  s]   € ô #7´s¸4¿;¹;Ó7GÈÓ"QÑˆ�iØ—k‘k . 9¨q¡=Ð1ˆŒÙÜŸ™›ˆDŒIÙØ×!Ñ! ! RÔ(ØÐrV   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rX   )r÷   rü   rœ   rž   s     rU   Úforward_featuresz EfficientFormer.forward_features9  s.   € Ø�I‰I�a‹LˆØ�K‰K˜‹NˆØ�I‰I�a‹LˆØˆrV   Ú
pre_logitsc                 ó6  — | j                   dk(  r|j                  d¬«      }| j                  |«      }|r|S | j                  |«      | j	                  |«      }}| j
                  r.| j                  r"t        j                  j                  «       s||fS ||z   dz  S )Nrå   r   re   r(   )
rê   Úmeanrÿ   r   r  r  r_   rF   r]   rá   )rP   rm   r=  Úx_dists       rU   Úforward_headzEfficientFormer.forward_head?  sˆ   € Ø×Ñ˜uÒ$Ø—‘˜1�“ˆAØ�N‰N˜1ÓˆÙØˆHØ—I‘I˜a“L $§.¡.°Ó"3ˆ6ˆØ×"Ò" t§}¢}¼U¿Y¹Y×=SÑ=SÔ=Uà�f�9Ðð ˜‘J !Ñ#Ð#rV   c                 óJ   — | j                  |«      }| j                  |«      }|S rX   )r<  rA  rž   s     rU   ru   zEfficientFormer.forwardM  s'   € Ø×!Ñ! !Ó$ˆØ×Ñ˜aÓ ˆØˆrV   rò   rx   rX   )NFFr+  F)r   FT)#ry   rz   r{   rB   rº   r–   rÌ   r   r?   r`   r   râ   r~   r   r—   r<   r  rF   r]   Úignorer  r  r  r!  r#  r%  r|   r   r   r5  r:  r<  rA  ru   r€   r�   s   @rU   r   r   ƒ  sÞ  ø„ ð '3Ø*<ØØ#Ø$Ø6:ØØ !ØØ,0Ø)+¯©Ø*,¯.©.Ø-/¯\©\Ø!Ø$&Ø$&ØØñ'E'à˜#˜s˜(‘OðE'ð ˜c 3˜h™ðE'ð ð	E'ð
 ðE'ð ðE'ð " %¨¨c¨	Ñ"2Ñ3ðE'ð ðE'ð ðE'ð ðE'ð %*ðE'ð ˜BŸI™I‘ðE'ð ˜RŸY™Y™ðE'ð   §	¡	™?ðE'ð ðE'ð  "ð!E'ð" "õ#E'òP-ð ‡Y�Y×ÑñRó ðRð ‡Y�Y×Ñòó ðð ‡Y�Y×Ñò*ó ð*ð ‡Y�Y×Ñð) §	¡	ò )ó ð)ñi¨Cð i¸hÀs¹mó ið ‡Y�Y×Ñò)ó ð)ð 8<ØØ$Ø$Ø',ñ4 à�|‰|ð4 ð ˜e C¨¨c© NÑ3Ñ4ð4 ð ð	4 ð
 ð4 ð ð4 ð !%ð4 ð 
ˆt�E—L‘LÑ! 5¨¯©°t¸E¿L¹LÑ7IÐ)IÑ#JÐJÑ	Kó4 ðp ./Ø$Ø#ñ	à˜3  S¡	˜>Ñ*ðð ðð ó	ò ñ$¨$ó $örV   c                 ó  — d| v r| S i }ddl }d}| j                  «       D ]ä  \  }}|j                  d«      rH|j                  dd«      }|j                  dd«      }|j                  d	d
«      }|j                  dd«      }|j	                  d|«      r|dz  }|j                  dd|› d�|«      }|j                  dd|› d�|«      }|j                  dd|› d�|«      }|j                  dd|«      }|j                  dd«      }|||<   Œæ |S )z$ Remap original checkpoints -> timm zstem.0.weightr   NÚpatch_embedzpatch_embed.0ú
stem.conv1zpatch_embed.1z
stem.norm1zpatch_embed.3z
stem.conv2zpatch_embed.4z
stem.norm2znetwork\.(\d+)\.proj\.weightr   znetwork.(\d+).(\d+)ró   z
.blocks.\2znetwork.(\d+).projz.downsample.convznetwork.(\d+).normz.downsample.normzlayer_scale_([0-9])z
ls\1.gammaÚ	dist_headr  )ÚreÚitemsÚ
startswithÚreplaceÚmatchÚsub)Ú
state_dictÚmodelÚout_dictrH  Ú	stage_idxrr   rs   s          rU   Úcheckpoint_filter_fnrR  S  s(  € à˜*Ñ$ØÐà€HÛØ€IØ× Ñ Ó"ò ‰ˆˆ1Ø�<‰<˜Ô&Ø—	‘	˜/¨<Ó8ˆAØ—	‘	˜/¨<Ó8ˆAØ—	‘	˜/¨<Ó8ˆAØ—	‘	˜/¨<Ó8ˆAà�8‰8Ð3°QÔ7Ø˜‰NˆIØ�F‰FÐ)¨W°Y°K¸{Ð+KÈQÓOˆØ�F‰FÐ(¨G°I°;Ð>NÐ*OÐQRÓSˆØ�F‰FÐ(¨G°I°;Ð>NÐ*OÐQRÓSˆà�F‰FÐ)¨=¸!Ó<ˆØ�I‰I�k ;Ó/ˆØˆ�Šðð  €OrV   c                 ó4   — | ddd dddt         t        dddd	œ|¥S )
Nrä   )r'   r   r   Tgffffffî?ÚbicubicrF  r   z
apache-2.0)Úurlré   Ú
input_sizer¦   Úfixed_input_sizeÚcrop_pctÚinterpolationr?  r  Ú
first_convÚ
classifierÚlicenser	   )rU  r  s     rU   Ú_cfgr]  n  s7   € àØ¨=ÀtÐaeØ¨)Ü%Ô.BØ"Ð2GØñð ðð rV   ztimm/)Ú	hf_hub_id)z!efficientformer_l1.snap_dist_in1kz!efficientformer_l3.snap_dist_in1kz!efficientformer_l7.snap_dist_in1kc                 ór   — |j                  dd«      }t        t        | |ft        t	        |d¬«      dœ|¤Ž}|S )NÚout_indicesr*   Úgetter)r`  Úfeature_cls)Úpretrained_filter_fnÚfeature_cfg)Úpopr   r   rR  rû   )ÚvariantÚ
pretrainedr  r`  rO  s        rU   Ú_create_efficientformerrh  ‡  sF   € Ø—*‘*˜]¨AÓ.€KÜ Ü˜ *ðä1Ü [¸hÔGñð ñ	€Eð €LrV   r[   c           	      óh   — t        t        d   t        d   d¬«      }t        dd| it        |fi |¤Ž¤ŽS )Nr#   r   ©ræ   rç   rÖ   rg  )Úefficientformer_l1©rû   ÚEfficientFormer_depthÚEfficientFormer_widthrh  ©rg  r  Ú
model_argss      rU   rk  rk  ’  óA   € äÜ$ TÑ*Ü(¨Ñ.Øô€Jô
 #ÑmÀJÐmÔRVÐWaÑRlÐekÑRlÑmÐmrV   c           	      óh   — t        t        d   t        d   d¬«      }t        dd| it        |fi |¤Ž¤ŽS )Nr$   r*   rj  rg  )Úefficientformer_l3rl  ro  s      rU   rs  rs  œ  rq  rV   c           	      óh   — t        t        d   t        d   d¬«      }t        dd| it        |fi |¤Ž¤ŽS )Nr%   r-   rj  rg  )Úefficientformer_l7rl  ro  s      rU   ru  ru  ¦  rq  rV   )r9  rò   )6rŸ   Útypingr   r   r   r   r   r   rF   Útorch.nnrB   Ú	timm.datar
   r   Útimm.layersr   r   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_manipulater   Ú	_registryr   r   Ú__all__rn  rm  r—   r/   rÜ   rƒ   r™   r¡   r¥   r­   r¼   rÎ   rÒ   r   rR  r]  Údefault_cfgsrh  rk  rs  ru  rÄ   rV   rU   ú<module>r€     sµ  ðñ÷ <× ;ã Ý ç A÷	÷ 	ó 	õ +Ý +Ý 'ß <àÐ
€ð Ø
Ø
ñÐ ð Ø
Ø
ñÐ ô@�—‘—‘ô @ôF-ˆB�M‰Mô -ô,�—‘ô ô>ˆ2�9‰9ô ô ˆb�i‰iô  ô$�b—i‘iô $ôN#�"—)‘)ô #ôL#�"—)‘)ô #ôLI˜2Ÿ9™9ô IôXM�b—i‘iô Mò`ó6	ñ %Ù)-Øô*ñ *.Øô*ñ *.Øô*ñ
&ó 
€óð ñn°oò nó ðnð ñn°oò nó ðnð ñn°oò nó ñnrV   