Ë
    éÓ^juž  ã                   ó²	  — d Z ddlmZ ddlmZ ddlmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmc mZ ddlmZ ddlmZ ddlmZmZ dd	lmZmZmZmZmZmZmZmZm Z  d
dl!m"Z" d
dl#m$Z$ d
dl%m&Z&m'Z' d
dl(m)Z)m*Z* dgZ+ G d„ dejX                  «      Z- G d„ dejX                  «      Z. G d„ dejX                  «      Z/ G d„ dejX                  «      Z0 G d„ dejX                  «      Z1 G d„ dejX                  «      Z2 G d„ de«      Z3 G d„ de«      Z4 G d „ d!ej:                  «      Z5 G d"„ d#ejX                  «      Z6 G d$„ d%ejX                  «      Z7 G d&„ d'ejX                  «      Z8 G d(„ d)ejX                  «      Z9 G d*„ d+ejX                  «      Z: G d,„ dejX                  «      Z;d-„ Z<d�d.„Z=d‚d/„Z> e)i d0 e>d1d2¬3«      “d4 e>d1d2¬3«      “d5 e>d1d2¬3«      “d6 e>d1d7¬3«      “d8 e>d1d7¬3«      “d9 e>d1¬:«      “d; e>d1¬:«      “d< e>d1¬:«      “d= e>d1¬:«      “d> e>d1¬:«      “d? e>d1d@¬A«      “dB e>d1d@dCdD¬E«      “dF e>d1d@¬A«      “dG e>d1d@dCdD¬E«      “dH e>d1d@dI¬J«      “dK e>d1d@¬A«      “dL e>d1d@dCdD¬E«      “i dM e>d1d@¬A«      “dN e>d1d@dCdD¬E«      “dO e>d1d@dI¬J«      “dP e>d1d@¬A«      “dQ e>d1d@dCdD¬E«      “dR e>d1d@¬A«      “dS e>d1d@dCdD¬E«      “dT e>d1d@dI¬J«      “dU e>d1d@¬A«      “dV e>d1d@dCdD¬E«      “dW e>d1d@¬A«      “dX e>d1d@dCdD¬E«      “dY e>d1d@dI¬J«      “dZ e>d1d@¬A«      “d[ e>d1d@dCdD¬E«      “d\ e>d1d@¬A«      “d] e>d1d@dCdD¬E«      “¥i d^ e>d1d@dI¬J«      “d_ e>d1d@¬A«      “d` e>d1d@dCdD¬E«      “da e>d1d@¬A«      “db e>d1d@dCdD¬E«      “dc e>d1d@dI¬J«      “dd e>d1d@¬A«      “de e>d1d@dCdD¬E«      “df e>d1d@¬A«      “dg e>d1d@dCdD¬E«      “dh e>d1d@dI¬J«      “di e>d1d@¬A«      “dj e>d1d@dCdD¬E«      “dk e>d1d@¬A«      “dl e>d1d@dCdD¬E«      “dm e>d1d@dI¬J«      “¥«      Z?e*d�dne;fdo„«       Z@e*d�dne;fdp„«       ZAe*d�dne;fdq„«       ZBe*d�dne;fdr„«       ZCe*d�dne;fds„«       ZDe*d�dne;fdt„«       ZEe*d�dne;fdu„«       ZFe*d�dne;fdv„«       ZGe*d�dne;fdw„«       ZHe*d�dne;fdx„«       ZIe*d�dne;fdy„«       ZJe*d�dne;fdz„«       ZKe*d�dne;fd{„«       ZLe*d�dne;fd|„«       ZMe*d�dne;fd}„«       ZNe*d�dne;fd~„«       ZOe*d�dne;fd„«       ZPe*d�dne;fd€„«       ZQy)ƒaì  
Poolformer from MetaFormer is Actually What You Need for Vision https://arxiv.org/abs/2111.11418

IdentityFormer, RandFormer, PoolFormerV2, ConvFormer, and CAFormer
from MetaFormer Baselines for Vision https://arxiv.org/abs/2210.13452

All implemented models support feature extraction and variable input resolution.

Original implementation by Weihao Yu et al.,
adapted for timm by Fredo Guan and Ross Wightman.

Adapted from https://github.com/sail-sg/metaformer, original copyright below
é    )ÚOrderedDict)Úpartial)ÚListÚOptionalÚTupleÚUnionÚTypeN)ÚTensor)ÚFinal©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)	Útrunc_normal_ÚDropPathÚcalculate_drop_path_ratesÚSelectAdaptivePool2dÚ
GroupNorm1Ú	LayerNormÚLayerNorm2dÚMlpÚuse_fused_attné   )Úbuild_model_with_cfg)Úfeature_take_indices)Ú
checkpointÚcheckpoint_seq)Úgenerate_default_cfgsÚregister_modelÚ
MetaFormerc            	       ó^   ‡ — e Zd ZdZ	 	 	 ddededeeej                        fˆ fd„Z	d„ Z
ˆ xZS )ÚStemzc
    Stem implemented by a layer of convolution.
    Conv2d params constant across all models.
    Úin_channelsÚout_channelsÚ
norm_layerc                 óÀ   •— ||dœ}t         ‰| �  «        t        j                  ||fddddœ|¤Ž| _        |r ||fi |¤Ž| _        y t        j
                  «       | _        y )N©ÚdeviceÚdtypeé   é   é   ©Úkernel_sizeÚstrideÚpadding)ÚsuperÚ__init__ÚnnÚConv2dÚconvÚIdentityÚnorm)Úselfr"   r#   r$   r'   r(   ÚddÚ	__class__s          €úa/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/metaformer.pyr1   zStem.__init__A   sl   ø€ ð ¨Ñ/ˆÜ‰ÑÔÜ—I‘IØØð
ð ØØñ
ð ñ
ˆŒ	ñ 7A‘J˜|Ñ2¨rÑ2ˆ�	ÄbÇkÁkÃmˆ�	ó    c                 óJ   — | j                  |«      }| j                  |«      }|S ©N)r4   r6   ©r7   Úxs     r:   ÚforwardzStem.forwardU   ó!   € Ø�I‰I�a‹LˆØ�I‰I�a‹LˆØˆr;   ©NNN©Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úintr   r	   r2   ÚModuler1   r@   Ú__classcell__©r9   s   @r:   r!   r!   ;   sJ   ø„ ñð 59ØØñTàðTð ðTð !  b§i¡i¡Ñ1õ	Tö(r;   r!   c                   ón   ‡ — e Zd ZdZ	 	 	 	 	 d
dedededededeeej                        fˆ fd„Z	d	„ Z
ˆ xZS )ÚDownsamplingz=
    Downsampling implemented by a layer of convolution.
    r"   r#   r-   r.   r/   r$   c	                 ó´   •— ||dœ}	t         ‰
| �  «        |r	 ||fi |	¤Žnt        j                  «       | _        t        j
                  ||f|||dœ|	¤Ž| _        y )Nr&   r,   )r0   r1   r2   r5   r6   r3   r4   )r7   r"   r#   r-   r.   r/   r$   r'   r(   r8   r9   s             €r:   r1   zDownsampling.__init__`   se   ø€ ð ¨Ñ/ˆÜ‰ÑÔÙ5?‘J˜{Ñ1¨bÒ1ÄRÇ[Á[Ã]ˆŒ	Ü—I‘IØØð
ð $ØØñ
ð ñ
ˆ�	r;   c                 óJ   — | j                  |«      }| j                  |«      }|S r=   )r6   r4   r>   s     r:   r@   zDownsampling.forwardw   rA   r;   )r   r   NNNrC   rK   s   @r:   rM   rM   [   sj   ø„ ñð ØØ48ØØñ
àð
ð ð
ð ð	
ð
 ð
ð ð
ð !  b§i¡i¡Ñ1õ
ö.r;   rM   c            	       óF   ‡ — e Zd ZdZ	 	 	 	 	 ddedededefˆ fd„Zd„ Zˆ xZ	S )	ÚScalez2
    Scale vector by element multiplications.
    ÚdimÚ
init_valueÚ	trainableÚuse_nchwc                 ó°   •— ||dœ}t         ‰| �  «        |r|ddfn|f| _        t        j                  |t        j                  |fi |¤Žz  |¬«      | _        y )Nr&   r   ©Úrequires_grad)r0   r1   Úshaper2   Ú	ParameterÚtorchÚonesÚscale)	r7   rR   rS   rT   rU   r'   r(   r8   r9   s	           €r:   r1   zScale.__init__‚   sS   ø€ ð ¨Ñ/ˆÜ‰ÑÔÙ$,�c˜1˜a‘[°3°&ˆŒ
Ü—\‘\ *¬u¯z©z¸#Ñ/DÀÑ/DÑ"DÐT]Ô^ˆ�
r;   c                 óR   — || j                   j                  | j                  «      z  S r=   )r]   ÚviewrY   r>   s     r:   r@   zScale.forward�   s   € Ø�4—:‘:—?‘? 4§:¡:Ó.Ñ.Ð.r;   )ç      ð?TTNN)
rD   rE   rF   rG   rH   ÚfloatÚboolr1   r@   rJ   rK   s   @r:   rQ   rQ   }   sN   ø„ ñð !$Ø"Ø!ØØñ_àð_ð ð_ð ð	_ð
 õ_ö/r;   rQ   c                   ó0   ‡ — e Zd ZdZddefˆ fd„Zd„ Zˆ xZS )ÚSquaredReLUz<
        Squared ReLU: https://arxiv.org/abs/2109.08668
    Úinplacec                 óX   •— t         ‰| �  «        t        j                  |¬«      | _        y )N©re   )r0   r1   r2   ÚReLUÚrelu)r7   re   r9   s     €r:   r1   zSquaredReLU.__init__™   s   ø€ Ü‰ÑÔÜ—G‘G GÔ,ˆ�	r;   c                 óJ   — t        j                  | j                  |«      «      S r=   )r[   Úsquareri   r>   s     r:   r@   zSquaredReLU.forward�   s   € Ü�|‰|˜DŸI™I a›LÓ)Ð)r;   ©F)rD   rE   rF   rG   rb   r1   r@   rJ   rK   s   @r:   rd   rd   ”   s   ø„ ññ- õ -ö*r;   rd   c                   óZ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 d
dededededee   defˆ fd„Zd	„ Z	ˆ xZ
S )ÚStarReLUz(
    StarReLU: s * relu(x) ** 2 + b
    Úscale_valueÚ
bias_valueÚscale_learnableÚbias_learnableÚmodere   c	                 ó8  •— ||dœ}	t         ‰
| �  «        || _        t        j                  |¬«      | _        t        j                  |t        j                  di |	¤Žz  |¬«      | _	        t        j                  |t        j                  di |	¤Žz  |¬«      | _
        y )Nr&   rg   rW   )r   )r0   r1   re   r2   rh   ri   rZ   r[   r\   r]   Úbias)r7   ro   rp   rq   rr   rs   re   r'   r(   r8   r9   s             €r:   r1   zStarReLU.__init__¦   sy   ø€ ð ¨Ñ/ˆÜ‰ÑÔØˆŒÜ—G‘G GÔ,ˆŒ	Ü—\‘\ +´·
±
Ñ0CÀÑ0CÑ"CÐSbÔcˆŒ
Ü—L‘L ¬e¯j©jÑ.A¸bÑ.AÑ!AÐQ_Ô`ˆ�	r;   c                 ó^   — | j                   | j                  |«      dz  z  | j                  z   S )Nr+   )r]   ri   ru   r>   s     r:   r@   zStarReLU.forward¸   s'   € Ø�z‰z˜DŸI™I a›L¨AÑ-Ñ-°·	±	Ñ9Ð9r;   )r`   ç        TTNFNN)rD   rE   rF   rG   ra   rb   r   Ústrr1   r@   rJ   rK   s   @r:   rn   rn   ¡   sq   ø„ ñð "%Ø #Ø$(Ø#'Ø"&Ø!ØØñaàðað ðað "ð	að
 !ðað ˜3‘-ðað õaö$:r;   rn   c                   óp   ‡ — e Zd ZU dZee   ed<   	 	 	 	 	 	 	 	 ddededee   dede	de	d	efˆ fd
„Z
d„ Zˆ xZS )Ú	Attentionzl
    Vanilla self-attention from Transformer: https://arxiv.org/abs/1706.03762.
    Modified from timm.
    Ú
fused_attnrR   Úhead_dimÚ	num_headsÚqkv_biasÚ	attn_dropÚ	proj_dropÚ	proj_biasc
                 óþ  •— ||	dœ}t         ‰| �  «        || _        |dz  | _        t	        «       | _        |r|n||z  | _        | j                  dk(  rd| _        | j                  | j                  z  | _        t        j                  || j                  dz  fd|i|¤Ž| _
        t        j                  |«      | _        t        j                  | j                  |fd|i|¤Ž| _        t        j                  |«      | _        y )Nr&   g      à¿r   r   é   ru   )r0   r1   r|   r]   r   r{   r}   Úattention_dimr2   ÚLinearÚqkvÚDropoutr   Úprojr€   )r7   rR   r|   r}   r~   r   r€   r�   r'   r(   Úkwargsr8   r9   s               €r:   r1   zAttention.__init__Ã   s×   ø€ ð ¨Ñ/ˆÜ‰ÑÔà ˆŒØ Ñ%ˆŒ
Ü(Ó*ˆŒá&/™°S¸H±_ˆŒØ�>‰>˜QÒØˆDŒNà!Ÿ^™^¨d¯m©mÑ;ˆÔä—9‘9˜S $×"4Ñ"4°qÑ"8ÑN¸xÐNÈ2ÑNˆŒÜŸ™ IÓ.ˆŒÜ—I‘I˜d×0Ñ0°#ÑL¸IÐLÈÑLˆŒ	ÜŸ™ IÓ.ˆ�r;   c                 ó”  — |j                   \  }}}| j                  |«      j                  ||d| j                  | j                  «      j                  ddddd«      }|j                  d«      \  }}}| j                  r<t        j                  |||| j                  r| j                  j                  nd¬«      }nJ||j                  dd	«      z  | j                  z  }	|	j                  d	¬
«      }	| j                  |	«      }	|	|z  }|j                  dd«      j                  |||«      }| j!                  |«      }| j#                  |«      }|S )Nrƒ   r+   r   r   r*   rw   )Ú	dropout_péþÿÿÿéÿÿÿÿ)rR   )rY   r†   Úreshaper}   r|   ÚpermuteÚunbindr{   ÚFÚscaled_dot_product_attentionÚtrainingr   ÚpÚ	transposer]   Úsoftmaxrˆ   r€   )
r7   r?   ÚBÚNÚCr†   ÚqÚkÚvÚattns
             r:   r@   zAttention.forwardâ   s  € Ø—'‘'‰ˆˆ1ˆaØ�h‰h�q‹k×!Ñ! ! Q¨¨4¯>©>¸4¿=¹=ÓI×QÑQÐRSÐUVÐXYÐ[\Ð^_Ó`ˆØ—*‘*˜Q“-‰ˆˆ1ˆaà�?Š?Ü×.Ñ.Ø�1�aØ.2¯mªm˜$Ÿ.™.×*Ò*Àô‰Að
 ˜Ÿ™ B¨Ó+Ñ+¨t¯z©zÑ9ˆDØ—<‘< B�<Ó'ˆDØ—>‘> $Ó'ˆDØ�q‘ˆAà�K‰K˜˜1Ó×%Ñ% a¨¨AÓ.ˆØ�I‰I�a‹LˆØ�N‰N˜1ÓˆØˆr;   )é    NFrw   rw   FNN)rD   rE   rF   rG   r   rb   Ú__annotations__rH   r   ra   r1   r@   rJ   rK   s   @r:   rz   rz   ¼   s€   ø… ñð �d‘Óð
 Ø'+Ø"Ø!Ø!Ø#ØØñ/àð/ð ð/ð   ‘}ð	/ð
 ð/ð ð/ð ð/ð õ/ö>r;   rz   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚGroupNorm1NoBiasÚnum_channelsc                 ób   •— t        ‰| �  |fi |¤Ž |j                  dd«      | _        d | _        y ©NÚepsç�íµ ÷Æ°>©r0   r1   Úgetr¥   ru   ©r7   r¢   r‰   r9   s      €r:   r1   zGroupNorm1NoBias.__init__ü   ó.   ø€ Ü‰Ñ˜Ñ0¨Ò0Ø—:‘:˜e TÓ*ˆŒØˆ�	r;   ©rD   rE   rF   rH   r1   rJ   rK   s   @r:   r¡   r¡   û   ó   ø„ ð S÷ ñ r;   r¡   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚLayerNorm2dNoBiasr¢   c                 ób   •— t        ‰| �  |fi |¤Ž |j                  dd«      | _        d | _        y r¤   r§   r©   s      €r:   r1   zLayerNorm2dNoBias.__init__  rª   r;   r«   rK   s   @r:   r®   r®     r¬   r;   r®   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚLayerNormNoBiasr¢   c                 ób   •— t        ‰| �  |fi |¤Ž |j                  dd«      | _        d | _        y r¤   r§   r©   s      €r:   r1   zLayerNormNoBias.__init__
  rª   r;   r«   rK   s   @r:   r±   r±   	  r¬   r;   r±   c                   ó    ‡ — e Zd ZdZdeej                  dddddfdeded	e	ej                     d
e	ej                     dededefˆ fd„Zd„ Zˆ xZS )ÚSepConvz\
    Inverted separable convolution from MobileNetV2: https://arxiv.org/abs/1801.04381.
    r+   Fr)   rƒ   NrR   Úexpansion_ratioÚ
act1_layerÚ
act2_layerru   r-   r/   c
                 ó–  •— ||	dœ}t         ‰| �  «        t        ||z  «      }t        j                  ||fd|dœ|¤Ž| _        t        |t        «      r |di |¤Žn |«       | _        t        j                  ||f||||dœ|¤Ž| _	        t        |t        «      r |di |¤Žn |«       | _
        t        j                  ||fd|dœ|¤Ž| _        y )Nr&   r   )r-   ru   )r-   r/   Úgroupsru   © )r0   r1   rH   r2   r3   Úpwconv1Ú
issubclassrn   Úact1ÚdwconvÚact2Úpwconv2)r7   rR   rµ   r¶   r·   ru   r-   r/   r'   r(   r‰   r8   Úmid_channelsr9   s                €r:   r1   zSepConv.__init__  sÔ   ø€ ð ¨Ñ/ˆÜ‰ÑÔÜ˜?¨SÑ0Ó1ˆÜ—y‘y  lÐSÀÈÑSÐPRÑSˆŒÜ(2°:¼xÔ(H‘JÑ$ Ò$ÉjËlˆŒ	Ü—i‘iØØð
ð $ØØØñ
ð ñ
ˆŒô )3°:¼xÔ(H‘JÑ$ Ò$ÉjËlˆŒ	Ü—y‘y ¨sÐSÀÈÑSÐPRÑSˆ�r;   c                 ó°   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j	                  |«      }|S r=   )r»   r½   r¾   r¿   rÀ   r>   s     r:   r@   zSepConv.forward3  sH   € Ø�L‰L˜‹OˆØ�I‰I�a‹LˆØ�K‰K˜‹NˆØ�I‰I�a‹LˆØ�L‰L˜‹OˆØˆr;   )rD   rE   rF   rG   rn   r2   r5   rH   ra   r	   rI   rb   r1   r@   rJ   rK   s   @r:   r´   r´     sŒ   ø„ ñð &'Ø*2Ø*,¯+©+ØØ ØØØñTàðTð #ðTð ˜RŸY™Y™ð	Tð
 ˜RŸY™Y™ðTð ðTð ðTð õTö<r;   r´   c                   ó0   ‡ — e Zd ZdZddefˆ fd„Zd„ Zˆ xZS )ÚPoolingzT
    Implementation of pooling for PoolFormer: https://arxiv.org/abs/2111.11418
    Ú	pool_sizec                 ód   •— t         ‰| �  «        t        j                  |d|dz  d¬«      | _        y )Nr   r+   F)r.   r/   Úcount_include_pad)r0   r1   r2   Ú	AvgPool2dÚpool)r7   rÅ   r‰   r9   s      €r:   r1   zPooling.__init__A  s)   ø€ Ü‰ÑÔÜ—L‘L °1¸iÈ1¹nÐ`eÔfˆ�	r;   c                 ó.   — | j                  |«      }||z
  S r=   )rÉ   )r7   r?   Úys      r:   r@   zPooling.forwardE  s   € Ø�I‰I�a‹LˆØ�1‰uˆr;   )rƒ   )rD   rE   rF   rG   rH   r1   r@   rJ   rK   s   @r:   rÄ   rÄ   <  s   ø„ ññg #õ gör;   rÄ   c                   óŒ   ‡ — e Zd ZdZddeeddddfdeded	ed
ee	j                     dee	j                     dedefˆ fd„Zd„ Zˆ xZS )ÚMlpHeadz MLP classification head
    éè  r*   rw   TNrR   Únum_classesÚ	mlp_ratioÚ	act_layerr$   Ú	drop_rateru   c
                 ó(  •— ||	dœ}
t         ‰| �  «        t        ||z  «      }t        j                  ||fd|i|
¤Ž| _         |«       | _         ||fi |
¤Ž| _        t        j                  ||fd|i|
¤Ž| _        t        j                  |«      | _
        y )Nr&   ru   )r0   r1   rH   r2   r…   Úfc1Úactr6   Úfc2r‡   Ú	head_drop)r7   rR   rÏ   rÐ   rÑ   r$   rÒ   ru   r'   r(   r8   Úhidden_featuresr9   s               €r:   r1   zMlpHead.__init__N  sŠ   ø€ ð ¨Ñ/ˆÜ‰ÑÔÜ˜i¨#™oÓ.ˆÜ—9‘9˜S /ÑC¸ÐCÀÑCˆŒÙ“;ˆŒÙ˜Ñ5°"Ñ5ˆŒ	Ü—9‘9˜_¨kÑKÀÐKÈÑKˆŒÜŸ™ IÓ.ˆ�r;   c                 ó°   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j	                  |«      }|S r=   )rÔ   rÕ   r6   r×   rÖ   r>   s     r:   r@   zMlpHead.forwardc  sI   € Ø�H‰H�Q‹KˆØ�H‰H�Q‹KˆØ�I‰I�a‹LˆØ�N‰N˜1ÓˆØ�H‰H�Q‹KˆØˆr;   )rD   rE   rF   rG   rd   r   rH   ra   r	   r2   rI   rb   r1   r@   rJ   rK   s   @r:   rÍ   rÍ   J  s€   ø„ ñð  $Ø Ø)4Ø*3Ø!ØØØñ/àð/ð ð/ð ð	/ð
 ˜BŸI™I‘ð/ð ˜RŸY™Y™ð/ð ð/ð õ/ö*r;   rÍ   c                   óÄ   ‡ — e Zd ZdZeededddddddfdedee	j                     dee	j                     d	ed
ee	j                     dedededee   dee   fˆ fd„Zd„ Zˆ xZS )ÚMetaFormerBlockz1
    Implementation of one MetaFormer block.
    Frw   TNrR   Útoken_mixerÚmlp_actÚmlp_biasr$   r€   Ú	drop_pathrU   Úlayer_scale_init_valueÚres_scale_init_valuec                 óÞ  •— ||dœ}t         ‰| �  «        t        t        f||	|dœ|¤Ž}t        t        f||
|dœ|¤Ž} ||fi |¤Ž| _         |d||dœ|¤|¤Ž| _        |dkD  rt        |«      nt        j                  «       | _	        |	� |«       nt        j                  «       | _
        |
� |«       nt        j                  «       | _         ||fi |¤Ž| _        t        |t        d|z  «      f||||dœ|¤Ž| _        |dkD  rt        |«      nt        j                  «       | _        |	� |«       nt        j                  «       | _        |
� |«       | _        y t        j                  «       | _        y )Nr&   )rR   rS   rU   )rR   r€   rw   r*   )rÑ   ru   ÚdropÚuse_convrº   )r0   r1   r   rQ   Únorm1rÜ   r   r2   r5   Ú
drop_path1Úlayer_scale1Ú
res_scale1Únorm2r   rH   ÚmlpÚ
drop_path2Úlayer_scale2Ú
res_scale2)r7   rR   rÜ   rÝ   rÞ   r$   r€   rß   rU   rà   rá   r'   r(   r‰   r8   Úls_layerÚrs_layerr9   s                    €r:   r1   zMetaFormerBlock.__init__q  sa  ø€ ð  ¨Ñ/ˆÜ‰ÑÔÜœ5Ðf cÐ6LÐW_ÑfÐceÑfˆÜœ5Ðd cÐ6JÐU]ÑdÐacÑdˆá Ñ* rÑ*ˆŒ
Ù&ÐT¨3¸)ÑTÀrÐTÈVÑTˆÔØ1:¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒØ*@Ð*L™HœJÔRT×R]ÑR]ÓR_ˆÔØ(<Ð(H™(œ*ÌbÏkÉkËmˆŒá Ñ* rÑ*ˆŒ
ÜØÜ��C‘‹Lð
ð ØØØñ
ð ñ
ˆŒð 2;¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒØ*@Ð*L™HœJÔRT×R]ÑR]ÓR_ˆÔØ(<Ð(H™(›*ˆ�ÌbÏkÉkËmˆ�r;   c           
      óF  — | j                  |«      | j                  | j                  | j                  | j	                  |«      «      «      «      z   }| j                  |«      | j                  | j                  | j                  | j                  |«      «      «      «      z   }|S r=   )
rè   rç   ræ   rÜ   rå   rí   rì   rë   rê   ré   r>   s     r:   r@   zMetaFormerBlock.forwardš  s�   € Ø�O‰O˜AÓØ×ÑØ—‘Ø×$Ñ$ T§Z¡Z°£]Ó3óóñˆð �O‰O˜AÓØ×ÑØ—‘Ø—H‘H˜TŸZ™Z¨›]Ó+óóñˆð ˆr;   )rD   rE   rF   rG   rÄ   rn   r   rH   r	   r2   rI   rb   ra   r   r1   r@   rJ   rK   s   @r:   rÛ   rÛ   l  sÃ   ø„ ñð ,3Ø'/Ø"Ø*5Ø!Ø!Ø!Ø6:Ø48ØØñ'\àð'\ð ˜bŸi™i™ð'\ð ˜"Ÿ)™)‘_ð	'\ð
 ð'\ð ˜RŸY™Y™ð'\ð ð'\ð ð'\ð ð'\ð %-¨U¡Oð'\ð #+¨5¡/õ'\öRr;   rÛ   c                   óL  ‡ — e Zd Zdej                  edeeddgdz  ddddfdedededeej                     d	eej                     d
e
deeej                        deej                     dedee   dee   dee   fˆ fd„Zej                   j"                  dd„«       Zdefd„Zˆ xZS )ÚMetaFormerStager+   Frw   NÚin_chsÚout_chsÚdepthrÜ   rÝ   rÞ   Údownsample_normr$   r€   Údp_ratesrà   rá   c                 ór  •— ||dœ}t         ‰| �  «        d| _        t        |t        «       | _        ||k(  rt        j                  «       nt        ||fddd|dœ|¤Ž| _	        t        j                  t        |«      D �cg c]'  }t        d||||||	|
|   ||| j
                  dœ
|¤|¤Ž‘Œ) c}Ž | _        y c c}w )	Nr&   Frƒ   r+   r   )r-   r.   r/   r$   )
rR   rÜ   rÝ   rÞ   r$   r€   rß   rà   rá   rU   rº   )r0   r1   Úgrad_checkpointingr¼   rz   rU   r2   r5   rM   Ú
downsampleÚ
SequentialÚrangerÛ   Úblocks)r7   ró   rô   rõ   rÜ   rÝ   rÞ   rö   r$   r€   r÷   rà   rá   r'   r(   r‰   r8   Úir9   s                     €r:   r1   zMetaFormerStage.__init__¬  sï   ø€ ð$ ¨Ñ/ˆÜ‰ÑÔØ"'ˆÔÜ& {´IÓ>Ð>ˆŒð ,2°WÒ+<œ"Ÿ+™+œ-Ä,ØØðC
ð ØØØ&ñC
ð ñC
ˆŒô —m‘mô ˜“<ö&!ð ô '6ð '
ØØ#ØØØ!ØØ˜q‘kØ#9Ø!5Ø—]‘]ñ'
ð ð'
ð ó'
ò &!ð "ˆ�ùò &!s   Á>,B4c                 ó   — || _         y r=   )rù   )r7   Úenables     r:   Úset_grad_checkpointingz&MetaFormerStage.set_grad_checkpointingÝ  s
   € à"(ˆÕr;   r?   c                 ó¬  — | j                  |«      }|j                  \  }}}}| j                  s#|j                  ||d«      j	                  dd«      }| j
                  r5t        j                  j                  «       st        | j                  |«      }n| j                  |«      }| j                  s$|j	                  dd«      j                  ||||«      }|S )Nr�   r   r+   )rú   rY   rU   rŽ   r•   rù   r[   ÚjitÚis_scriptingr   rý   )r7   r?   r—   r™   ÚHÚWs         r:   r@   zMetaFormerStage.forwardá  s©   € Ø�O‰O˜AÓˆØ—W‘W‰
ˆˆ1ˆa�à�}Š}Ø—	‘	˜!˜Q Ó#×-Ñ-¨a°Ó3ˆAà×"Ò"¬5¯9©9×+AÑ+AÔ+CÜ˜tŸ{™{¨AÓ.‰Aà—‘˜A“ˆAà�}Š}Ø—‘˜A˜qÓ!×)Ñ)¨!¨Q°°1Ó5ˆAàˆr;   ©T)rD   rE   rF   r2   r5   rn   r   rH   r	   rI   rb   r   ra   r   r1   r[   r  Úignorer  r
   r@   rJ   rK   s   @r:   rò   rò   ª  s
  ø„ ð Ø+-¯;©;Ø'/Ø"Ø9DØ*5Ø!Ø%' D¨1¡HØ6:Ø48ØØñ/"àð/"ð ð/"ð ð	/"ð
 ˜bŸi™i™ð/"ð ˜"Ÿ)™)‘_ð/"ð ð/"ð & d¨2¯9©9¡oÑ6ð/"ð ˜RŸY™Y™ð/"ð ð/"ð ˜5‘kð/"ð %-¨U¡Oð/"ð #+¨5¡/õ/"ðb ‡Y�Y×Ñò)ó ð)ð˜÷ r;   rò   c            #       ór  ‡ — e Zd ZdZdddddeeddddd	d
eeedd	d	fdedede	de
edf   de
edf   deeej                     eeej                        f   deej                     dededededeeeee   f      dee
ee   df   eee      f   deeej                        deeej                     eeej                        f   deej                     def"ˆ fd„Zd„ Zej,                  j.                  d2d „«       Zej,                  j.                  d!ej                  fd"„«       Zd3dedee	   fd#„Z	 	 	 	 	 d4d$ej6                  d%eeeee   f      d&ed'ed(e	d)ed!eeej6                     e
ej6                  eej6                     f   f   fd*„Z	 	 	 d5d%eeee   f   d+ed,efd-„Zd6d$ed.efd/„Zd$efd0„Zd$efd1„Z ˆ xZ!S )7r   aM   MetaFormer
        A PyTorch impl of : `MetaFormer Baselines for Vision`  -
          https://arxiv.org/abs/2210.13452

    Args:
        in_chans (int): Number of input image channels.
        num_classes (int): Number of classes for classification head.
        global_pool: Pooling for classifier head.
        depths (list or tuple): Number of blocks at each stage.
        dims (list or tuple): Feature dimension at each stage.
        token_mixers (list, tuple or token_fcn): Token mixer for each stage.
        mlp_act: Activation layer for MLP.
        mlp_bias (boolean): Enable or disable mlp bias term.
        drop_path_rate (float): Stochastic depth rate.
        drop_rate (float): Dropout rate.
        layer_scale_init_values (list, tuple, float or None): Init value for Layer Scale.
            None means not use the layer scale. Form: https://arxiv.org/abs/2103.17239.
        res_scale_init_values (list, tuple, float or None): Init value for res Scale on residual connections.
            None means not use the res scale. From: https://arxiv.org/abs/2110.09456.
        downsample_norm (nn.Module): Norm layer used in stem and downsampling layers.
        norm_layers (list, tuple or norm_fcn): Norm layers for each stage.
        output_norm: Norm layer before classifier head.
        use_mlp_head: Use MLP classification head.
    rƒ   rÎ   Úavg©r+   r+   é   r+   ©é@   é€   i@  é   Frw   N)NNr`   r`   TÚin_chansrÏ   Úglobal_poolÚdepths.ÚdimsÚtoken_mixersrÝ   rÞ   Údrop_path_rateÚproj_drop_raterÒ   Úlayer_scale_init_valuesÚres_scale_init_valuesrö   Únorm_layersÚoutput_normÚuse_mlp_headc                 óš  •— t         ‰| �  «        ||dœ}|t        fv rt        |fi |¤Ž}|| _        || _        |d   | _        || _        || _        t        |«      | _
        t        |t        t        f«      s|g}t        |t        t        f«      s|g}t        |t        t        f«      s|g| j                  z  }t        |t        t        f«      s|g| j                  z  }t        |t        t        f«      s|g| j                  z  }t        |t        t        f«      s|g| j                  z  }d| _        g | _        t!        ||d   fd|i|¤Ž| _        g }|d   }t%        |	|d¬«      }t'        | j                  «      D ]h  }|t)        |||   f||   ||   |||
||   ||   ||   |||   dœ
|¤|¤Žgz  }||   }| xj                  t+        ||   d	|d	z   z  d
|› �¬«      gz  c_        Œj t-        j.                  |Ž | _        |dkD  rt| j                  r5t3        | j                  |fd| j                  i|¤Ž}| j                  | _        nGt-        j6                  | j                  |fi |¤Ž}| j                  | _        nt-        j8                  «       }t-        j.                  t;        dt=        |¬«      fd || j                  fi |¤Žfd|rt-        j>                  d«      nt-        j8                  «       fd| j                  rt-        j@                  |«      nt-        j8                  «       fd|fg«      «      | _!        | jE                  | jF                  «       y )Nr&   r�   Fr   r$   T)Ú	stagewise)
rõ   rÜ   rÝ   rÞ   r€   r÷   rà   rá   rö   r$   r+   zstages.)Únum_chsÚ	reductionÚmodulerÒ   r  ©Ú	pool_typer6   Úflattenr   rã   Úfc)$r0   r1   rn   r   rÏ   r  Únum_featuresrÒ   r  ÚlenÚ
num_stagesÚ
isinstanceÚlistÚtuplerù   Úfeature_infor!   Ústemr   rü   rò   Údictr2   rû   ÚstagesrÍ   Úhead_hidden_sizer…   r5   r   r   ÚFlattenr‡   ÚheadÚapplyÚ_init_weights)r7   r  rÏ   r  r  r  r  rÝ   rÞ   r  r  rÒ   r  r  rö   r  r  r  r'   r(   r‰   r8   r/  Úprev_dimr÷   rþ   Úfinalr9   s                              €r:   r1   zMetaFormer.__init__  sT  ø€ ô. 	‰ÑÔØ¨Ñ/ˆà”x�kÑ!Ü˜gÑ,¨Ñ,ˆGà&ˆÔØ ˆŒØ  ™HˆÔØ"ˆŒØ(ˆÔÜ˜f›+ˆŒô ˜&¤4¬ -Ô0Ø�XˆFÜ˜$¤¤u Ô.Ø�6ˆDÜ˜,¬¬u¨Ô6Ø(˜>¨D¯O©OÑ;ˆLÜ˜+¬¬e }Ô5Ø&˜-¨$¯/©/Ñ9ˆKÜÐ1´D¼%°=ÔAØ'>Ð&?À$Ç/Á/Ñ&QÐ#ÜÐ/´$¼°Ô?Ø%:Ð$;¸d¿o¹oÑ$MÐ!à"'ˆÔØˆÔäØØ�‰Gñ
ð 'ð
ð ñ	
ˆŒ	ð ˆØ˜‘7ˆÜ,¨^¸VÈtÔTˆÜ�t—‘Ó'ò 	cˆAØ”ØØ�Q‘ðð ˜Q‘iØ(¨™OØØ!Ø(Ø! !™Ø'>¸qÑ'AØ%:¸1Ñ%=Ø /Ø& q™>ñð ðð ñð ñ ˆFð  ˜A‘wˆHØ×Ò¤$¨t°A©wÀ!ÀaÈÁcÁ(ÐU\Ð]^Ð\_ÐS`Ô"aÐ!bÑbÖð%	cô( —m‘m VÐ,ˆŒð ˜Š?Ø× Ò ä × 1Ñ 1°;Ñ_È$Ï.É.Ð_Ð\^Ñ_�Ø(,×(9Ñ(9�Õ%äŸ	™	 $×"3Ñ"3°[ÑGÀBÑG�Ø(,×(9Ñ(9�Õ%ä—K‘K“MˆEä—M‘M¤+ØÔ0¸;ÔGÐHØ‘[ ×!2Ñ!2Ñ9°bÑ9Ð:Ø©œŸ
™
 1œ¼"¿+¹+»-ÐHØ¨d×.?Ò.?”R—Z‘Z 	Ô*ÄRÇ[Á[Ã]ÐSØ�5ˆMð/
ó #ó ˆŒ	ð 	�
‰
�4×%Ñ%Õ&r;   c                 óö   — t        |t        j                  t        j                  f«      rOt	        |j
                  d¬«       |j                  �+t        j                  j                  |j                  d«       y y y )Ng{®Gáz”?)Ústdr   )	r)  r2   r3   r…   r   Úweightru   ÚinitÚ	constant_)r7   Úms     r:   r4  zMetaFormer._init_weightsx  sS   € Ü�aœ"Ÿ)™)¤R§Y¡YÐ/Ô0Ü˜!Ÿ(™(¨Õ,Ø�v‰vÐ!Ü—‘×!Ñ! !§&¡&¨!Õ,ð "ð 1r;   c                 óX   — || _         | j                  D ]  }|j                  |¬«       Œ y )N)r   )rù   r/  r  )r7   r   Ústages      r:   r  z!MetaFormer.set_grad_checkpointing~  s.   € à"(ˆÔØ—[‘[ò 	8ˆEØ×(Ñ(°Ð(Õ7ñ	8r;   Úreturnc                 ó.   — | j                   j                  S r=   )r2  r%  )r7   s    r:   Úget_classifierzMetaFormer.get_classifier„  s   € à�y‰y�|‰|Ðr;   c                 óÂ  — ||dœ}|| _         |�Ut        |¬«      | j                  _        |rt	        j
                  d«      nt	        j                  «       | j                  _        |dkD  rR| j                  r$t        | j                  |fd| j                  i|¤Ž}n6t	        j                  | j                  |fi |¤Ž}nt	        j                  «       }|| j                  _        y )Nr&   r"  r   r   rÒ   )rÏ   r   r2  r  r2   r1  r5   r$  r  rÍ   r&  rÒ   r…   r%  )r7   rÏ   r  r'   r(   r8   r6  s          r:   Úreset_classifierzMetaFormer.reset_classifierˆ  s«   € Ø¨Ñ/ˆØ&ˆÔØÐ"Ü$8À;Ô$OˆD�I‰IÔ!Ù1<¤§
¡
¨1¤Ä"Ç+Á+Ã-ˆD�I‰IÔØ˜Š?Ø× Ò Ü × 1Ñ 1°;Ñ_È$Ï.É.Ð_Ð\^Ñ_‘äŸ	™	 $×"3Ñ"3°[ÑGÀBÑG‘ä—K‘K“MˆEØˆ�	‰	�r;   r?   Úindicesr6   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 óà  — |dv sJ d«       ‚g }t        t        | j                  «      |«      \  }}	| j                  |«      }t        j
                  j                  «       s|s| j                  }
n| j                  d|	dz    }
t        |
«      D ]Z  \  }}| j                  r+t        j
                  j                  «       st        ||«      }n ||«      }||v sŒJ|j                  |«       Œ\ |r|S ||fS )aï   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )ÚNCHWzOutput shape must be NCHW.Nr   )r   r'  r/  r-  r[   r  r  Ú	enumeraterù   r   Úappend)r7   r?   rD  r6   rE  rF  rG  ÚintermediatesÚtake_indicesÚ	max_indexr/  Úfeat_idxr>  s                r:   Úforward_intermediatesz MetaFormer.forward_intermediates—  sæ   € ð* ˜YÑ&ÐDÐ(DÓDÐ&ØˆÜ"6´s¸4¿;¹;Ó7GÈÓ"QÑˆ�ið �I‰I�a‹LˆÜ�9‰9×!Ñ!Ô#©:Ø—[‘[‰Fà—[‘[  )¨a¡-Ð0ˆFä(¨Ó0ò 	(‰OˆH�eØ×&Ò&¬u¯y©y×/EÑ/EÔ/GÜ˜u aÓ(‘á˜!“H�Ø˜<Ò'Ø×$Ñ$ QÕ'ð	(ñ Ø Ð à�-ÐÐr;   Ú
prune_normÚ
prune_headc                 ó    — t        t        | j                  «      |«      \  }}| j                  d|dz    | _        |r| j                  dd«       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   Ú )r   r'  r/  rC  )r7   rD  rQ  rR  rM  rN  s         r:   Úprune_intermediate_layersz$MetaFormer.prune_intermediate_layersÄ  sM   € ô #7´s¸4¿;¹;Ó7GÈÓ"QÑˆ�iØ—k‘k . 9¨q¡=Ð1ˆŒÙØ×!Ñ! ! RÔ(ØÐr;   Ú
pre_logitsc                 ó  — | j                   j                  |«      }| j                   j                  |«      }| j                   j                  |«      }| j                   j	                  |«      }|r|S | j                   j                  |«      S r=   )r2  r  r6   r$  rã   r%  )r7   r?   rV  s      r:   Úforward_headzMetaFormer.forward_headÒ  sg   € à�I‰I×!Ñ! !Ó$ˆØ�I‰I�N‰N˜1ÓˆØ�I‰I×Ñ˜aÓ ˆØ�I‰I�N‰N˜1ÓˆÙˆqÐ3 D§I¡I§L¡L°£OÐ3r;   c                 óÎ   — | j                  |«      }| j                  r6t        j                  j	                  «       st        | j                  |«      }|S | j                  |«      }|S r=   )r-  rù   r[   r  r  r   r/  r>   s     r:   Úforward_featureszMetaFormer.forward_featuresÚ  sR   € Ø�I‰I�a‹LˆØ×"Ò"¬5¯9©9×+AÑ+AÔ+CÜ˜tŸ{™{¨AÓ.ˆAð ˆð —‘˜A“ˆAØˆr;   c                 óJ   — | j                  |«      }| j                  |«      }|S r=   )rZ  rX  r>   s     r:   r@   zMetaFormer.forwardâ  s'   € Ø×!Ñ! !Ó$ˆØ×Ñ˜aÓ ˆØˆr;   r  rB   )NFFrI  F)r   FTrl   )"rD   rE   rF   rG   rÄ   rn   r®   r   rH   rx   r   r   r	   r2   rI   r   rb   ra   r   r1   r4  r[   r  r  r  rA  rC  r
   rP  rU  rX  rZ  r@   rJ   rK   s   @r:   r   r   ó  s  ø„ ñð6 Ø#Ø$Ø&2Ø$7ØJQØ'/Ø"Ø$&Ø$&Ø"ØKOØ_uØ9JØIZØ+6Ø!%ØØñ)i'àði'ð ði'ð ð	i'ð
 ˜#˜s˜(‘Oði'ð ˜˜S˜‘/ði'ð    R§Y¡Y¡°°d¸2¿9¹9±oÑ1FÐ FÑGði'ð ˜"Ÿ)™)‘_ði'ð ði'ð "ði'ð "ði'ð ði'ð &.¨e°E¸4À¹;Ð4FÑ.GÑ%Hði'ð $)¨¨x¸©ÀÐ/CÑ)DÀdÈ8ÐTYÉ?ÑF[Ð)[Ñ#\ði'ð & d¨2¯9©9¡oÑ6ði'ð  ˜t B§I¡I™°°T¸"¿)¹)±_Ñ0EÐEÑFð!i'ð" ˜bŸi™i™ð#i'ð$ õ%i'òV-ð ‡Y�Y×Ñò8ó ð8ð
 ‡Y�Y×Ñð §	¡	ò ó ðñ¨Cð ¸hÀs¹mó ð$ 8<ØØ$Ø$Ø',ñ+ à�|‰|ð+ ð ˜e C¨¨c© NÑ3Ñ4ð+ ð ð	+ ð
 ð+ ð ð+ ð !%ð+ ð 
ˆt�E—L‘LÑ! 5¨¯©°t¸E¿L¹LÑ7IÐ)IÑ#JÐJÑ	Kó+ ð^ ./Ø$Ø#ñ	à˜3  S¡	˜>Ñ*ðð ðð ó	ñ4˜fð 4°$ó 4ð &ó ð˜÷ r;   c                 óÂ  — d| v r| S dd l }i }d| v }|j                  «       }| j                  «       D �]ª  \  }}|r‘|j                  dd|«      }|j	                  dd«      }|j	                  dd	«      }|j	                  d
d«      }|j	                  dd«      }|j	                  dd«      }|j	                  dd«      }|j	                  dd«      }|j                  dd|«      }|j	                  dd«      }|j	                  dd«      }|j                  dd|«      }|j	                  dd«      }|j	                  dd«      }|j	                  dd«      }|j	                  dd«      }|j                  dd |«      }|j                  d!d"|«      }|j
                  ||   k7  rB|j                  «       ||   j                  «       k(  r|j                  ||   j
                  «      }|||<   �Œ­ |S )#Nzstem.conv.weightr   znetwork.0.0.mlp.fc1.weightzlayer_scale_([0-9]+)zlayer_scale\1.scalez	network.1zdownsample_layers.1z	network.3zdownsample_layers.2z	network.5zdownsample_layers.3z	network.2z	network.4z	network.6Únetworkr/  zdownsample_layers.([0-9]+)zstages.\1.downsamplezdownsample.projzdownsample.convzpatch_embed.projzpatch_embed.convz([0-9]+).([0-9]+)z\1.blocks.\2zstages.0.downsampleÚpatch_embedr-  Ú	post_normr6   Úpre_normz^headúhead.fcz^normz	head.norm)ÚreÚ
state_dictÚitemsÚsubÚreplacerY   ÚnumelrŽ   )rc  Úmodelrb  Úout_dictÚis_poolformerv1Úmodel_state_dictr›   rœ   s           r:   Úcheckpoint_filter_fnrl  é  sß  € Ø˜ZÑ'ØÐãØ€HØ2°jÐ@€OØ×'Ñ'Ó)ÐØ× Ñ Ó"ó ‰ˆˆ1ÙØ—‘Ð.Ð0FÈÓJˆAØ—	‘	˜+Ð'<Ó=ˆAØ—	‘	˜+Ð'<Ó=ˆAØ—	‘	˜+Ð'<Ó=ˆAØ—	‘	˜+ {Ó3ˆAØ—	‘	˜+ {Ó3ˆAØ—	‘	˜+ {Ó3ˆAØ—	‘	˜) XÓ.ˆAà�F‰FÐ0Ð2IÈ1ÓMˆØ�I‰IÐ'Ð):Ó;ˆØ�I‰IÐ(Ð*<Ó=ˆØ�F‰FÐ'¨¸!Ó<ˆØ�I‰IÐ+¨]Ó;ˆØ�I‰I�m VÓ,ˆØ�I‰I�k 6Ó*ˆØ�I‰I�j &Ó)ˆØ�F‰F�8˜Y¨Ó*ˆØ�F‰F�8˜[¨!Ó,ˆà�7‰7Ð& qÑ)Ò)¨a¯g©g«iÐ;KÈAÑ;N×;TÑ;TÓ;VÒ.VØ—	‘	Ð*¨1Ñ-×3Ñ3Ó4ˆAàˆ�‹ð3ð4 €Or;   c           	      óÈ   — t        d„ t        |j                  dd«      «      D «       «      }|j                  d|«      }t	        t
        | |ft        t        d|¬«      dœ|¤Ž}|S )Nc              3   ó&   K  — | ]	  \  }}|–— Œ y ­wr=   rº   )Ú.0rþ   Ú_s      r:   ú	<genexpr>z%_create_metaformer.<locals>.<genexpr>  s   è ø€ Ò\¡d a¨¤Ñ\ùs   ‚r  r  Úout_indicesT)Úflatten_sequentialrr  )Úpretrained_filter_fnÚfeature_cfg)r+  rJ  r¨   Úpopr   r   rl  r.  )ÚvariantÚ
pretrainedr‰   Údefault_out_indicesrr  rh  s         r:   Ú_create_metaformerrz    sl   € ÜÑ\¬i¸¿
¹
À8È\Ó8ZÓ.[Ô\Ó\ÐØ—*‘*˜]Ð,?Ó@€Kä ÜØØðô 2Ü¨D¸kÔJñð ñ€Eð €Lr;   c                 ó2   — | dddddt         t        dddd	œ|¥S )
NrÎ   )rƒ   éà   r|  )r)   r)   r`   Úbicubicra  z	stem.convz
apache-2.0)ÚurlrÏ   Ú
input_sizerÅ   Úcrop_pctÚinterpolationÚmeanr8  Ú
classifierÚ
first_convÚlicenser   )r~  r‰   s     r:   Ú_cfgr†    s3   € àØ¨=ÀvØ¨)Ü%Ô.BØ¨{Øñð ðð r;   zpoolformer_s12.sail_in1kztimm/gÍÌÌÌÌÌì?)Ú	hf_hub_idr€  zpoolformer_s24.sail_in1kzpoolformer_s36.sail_in1kzpoolformer_m36.sail_in1kgffffffî?zpoolformer_m48.sail_in1kzpoolformerv2_s12.sail_in1k)r‡  zpoolformerv2_s24.sail_in1kzpoolformerv2_s36.sail_in1kzpoolformerv2_m36.sail_in1kzpoolformerv2_m48.sail_in1kzconvformer_s18.sail_in1kzhead.fc.fc2)r‡  rƒ  zconvformer_s18.sail_in1k_384)rƒ   é€  rˆ  )é   r‰  )r‡  rƒ  r  rÅ   z!convformer_s18.sail_in22k_ft_in1kz%convformer_s18.sail_in22k_ft_in1k_384zconvformer_s18.sail_in22kiQU  )r‡  rƒ  rÏ   zconvformer_s36.sail_in1kzconvformer_s36.sail_in1k_384z!convformer_s36.sail_in22k_ft_in1kz%convformer_s36.sail_in22k_ft_in1k_384zconvformer_s36.sail_in22kzconvformer_m36.sail_in1kzconvformer_m36.sail_in1k_384z!convformer_m36.sail_in22k_ft_in1kz%convformer_m36.sail_in22k_ft_in1k_384zconvformer_m36.sail_in22kzconvformer_b36.sail_in1kzconvformer_b36.sail_in1k_384z!convformer_b36.sail_in22k_ft_in1kz%convformer_b36.sail_in22k_ft_in1k_384zconvformer_b36.sail_in22kzcaformer_s18.sail_in1kzcaformer_s18.sail_in1k_384zcaformer_s18.sail_in22k_ft_in1kz#caformer_s18.sail_in22k_ft_in1k_384zcaformer_s18.sail_in22kzcaformer_s36.sail_in1kzcaformer_s36.sail_in1k_384zcaformer_s36.sail_in22k_ft_in1kz#caformer_s36.sail_in22k_ft_in1k_384zcaformer_s36.sail_in22kzcaformer_m36.sail_in1kzcaformer_m36.sail_in1k_384zcaformer_m36.sail_in22k_ft_in1kz#caformer_m36.sail_in22k_ft_in1k_384zcaformer_m36.sail_in22kzcaformer_b36.sail_in1kzcaformer_b36.sail_in1k_384zcaformer_b36.sail_in22k_ft_in1kz#caformer_b36.sail_in22k_ft_in1k_384zcaformer_b36.sail_in22kr?  c                 ór   — t        dg d¢g d¢d t        j                  dt        dd ddœ	|¤Ž}t	        d	d| i|¤ŽS )
Nr  r  Tçñhãˆµøä>F©	r  r  rö   rÝ   rÞ   r  r  r  r  rx  rº   )Úpoolformer_s12©r.  r2   ÚGELUr   rz  ©rx  r‰   Úmodel_kwargss      r:   r�  r�  Ã  sO   € äð 
ÚÚ ØÜ—‘ØÜØ $Ø"Øñ
ð ñ
€Lô ÑV¸:ÐVÈÑVÐVr;   c                 ór   — t        dg d¢g d¢d t        j                  dt        dd ddœ	|¤Ž}t	        d	d| i|¤ŽS )
N©r*   r*   r‰  r*   r  Tr‹  FrŒ  rx  rº   )Úpoolformer_s24rŽ  r�  s      r:   r”  r”  Ó  óO   € äð 
ÚÚ ØÜ—‘ØÜØ $Ø"Øñ
ð ñ
€Lô ÑV¸:ÐVÈÑVÐVr;   c                 ór   — t        dg d¢g d¢d t        j                  dt        dd ddœ	|¤Ž}t	        d	d| i|¤ŽS )
N©r  r  é   r  r  Tr¦   FrŒ  rx  rº   )Úpoolformer_s36rŽ  r�  s      r:   r™  r™  ã  r•  r;   c                 ór   — t        dg d¢g d¢d t        j                  dt        dd ddœ	|¤Ž}t	        d	d| i|¤ŽS )
Nr—  ©é`   éÀ   rˆ  é   Tr¦   FrŒ  rx  rº   )Úpoolformer_m36rŽ  r�  s      r:   rŸ  rŸ  ó  r•  r;   c                 ór   — t        dg d¢g d¢d t        j                  dt        dd ddœ	|¤Ž}t	        d	d| i|¤ŽS )
N©é   r¢  é   r¢  r›  Tr¦   FrŒ  rx  rº   )Úpoolformer_m48rŽ  r�  s      r:   r¤  r¤    r•  r;   c                 óL   — t        dg d¢g d¢t        ddœ|¤Ž}t        dd| i|¤ŽS )Nr  r  F©r  r  r  r  rx  rº   )Úpoolformerv2_s12©r.  r¡   rz  r�  s      r:   r§  r§    s<   € äð ÚÚ Ü$Øñ	ð
 ñ€Lô ÑX¸ZÐXÈ<ÑXÐXr;   c                 óL   — t        dg d¢g d¢t        ddœ|¤Ž}t        dd| i|¤ŽS )Nr“  r  Fr¦  rx  rº   )Úpoolformerv2_s24r¨  r�  s      r:   rª  rª    ó<   € äð ÚÚ Ü$Øñ	ð
 ñ€Lô ÑX¸ZÐXÈ<ÑXÐXr;   c                 óL   — t        dg d¢g d¢t        ddœ|¤Ž}t        dd| i|¤ŽS )Nr—  r  Fr¦  rx  rº   )Úpoolformerv2_s36r¨  r�  s      r:   r­  r­  )  r«  r;   c                 óL   — t        dg d¢g d¢t        ddœ|¤Ž}t        dd| i|¤ŽS )Nr—  r›  Fr¦  rx  rº   )Úpoolformerv2_m36r¨  r�  s      r:   r¯  r¯  4  r«  r;   c                 óL   — t        dg d¢g d¢t        ddœ|¤Ž}t        dd| i|¤ŽS )Nr¡  r›  Fr¦  rx  rº   )Úpoolformerv2_m48r¨  r�  s      r:   r±  r±  ?  r«  r;   c                 óT   — t        dg d¢g d¢t        t        dœ|¤Ž}t        dd| i|¤ŽS )N©rƒ   rƒ   é	   rƒ   r  ©r  r  r  r  rx  rº   )Úconvformer_s18©r.  r´   r®   rz  r�  s      r:   r¶  r¶  J  s<   € äð ÚÚ ÜÜ%ñ	ð
 ñ€Lô ÑV¸:ÐVÈÑVÐVr;   c                 óT   — t        dg d¢g d¢t        t        dœ|¤Ž}t        dd| i|¤ŽS )N©rƒ   r‰  r˜  rƒ   r  rµ  rx  rº   )Úconvformer_s36r·  r�  s      r:   rº  rº  U  ó<   € äð ÚÚ ÜÜ%ñ	ð
 ñ€Lô ÑV¸:ÐVÈÑVÐVr;   c                 óT   — t        dg d¢g d¢t        t        dœ|¤Ž}t        dd| i|¤ŽS )Nr¹  ©rœ  r�  rˆ  i@  rµ  rx  rº   )Úconvformer_m36r·  r�  s      r:   r¾  r¾  `  r»  r;   c                 óT   — t        dg d¢g d¢t        t        dœ|¤Ž}t        dd| i|¤ŽS )Nr¹  ©r  é   r  rž  rµ  rx  rº   )Úconvformer_b36r·  r�  s      r:   rÂ  rÂ  k  s<   € äð ÚÚ!ÜÜ%ñ	ð
 ñ€Lô ÑV¸:ÐVÈÑVÐVr;   c           	      ó’   — t        dg d¢g d¢t        t        t        t        gt        gdz  t        gdz  z   dœ|¤Ž}t        dd| i|¤ŽS )Nr³  r  r+   rµ  rx  rº   )Úcaformer_s18©r.  r´   rz   r®   r±   rz  r�  s      r:   rÄ  rÄ  v  sZ   € äð ÚÚ Üœw¬	´9Ð=Ü&Ð'¨!Ñ+¬Ð.?À!Ñ.CÑCñ	ð
 ñ€Lô ÑT¸ÐTÀ|ÑTÐTr;   c           	      ó’   — t        dg d¢g d¢t        t        t        t        gt        gdz  t        gdz  z   dœ|¤Ž}t        dd| i|¤ŽS )Nr¹  r  r+   rµ  rx  rº   )Úcaformer_s36rÅ  r�  s      r:   rÇ  rÇ  �  óZ   € äð ÚÚ Üœw¬	´9Ð=Ü&Ð'¨!Ñ+¬Ð.?À!Ñ.CÑCñ	ð
 ñ€Lô ÑT¸ÐTÀ|ÑTÐTr;   c           	      ó’   — t        dg d¢g d¢t        t        t        t        gt        gdz  t        gdz  z   dœ|¤Ž}t        dd| i|¤ŽS )Nr¹  r½  r+   rµ  rx  rº   )Úcaformer_m36rÅ  r�  s      r:   rÊ  rÊ  Œ  rÈ  r;   c           	      ó’   — t        dg d¢g d¢t        t        t        t        gt        gdz  t        gdz  z   dœ|¤Ž}t        dd| i|¤ŽS )Nr¹  rÀ  r+   rµ  rx  rº   )Úcaformer_b36rÅ  r�  s      r:   rÌ  rÌ  —  sZ   € äð ÚÚ!Üœw¬	´9Ð=Ü&Ð'¨!Ñ+¬Ð.?À!Ñ.CÑCñ	ð
 ñ€Lô ÑT¸ÐTÀ|ÑTÐTr;   rl   )rT  )RrG   Úcollectionsr   Ú	functoolsr   Útypingr   r   r   r   r	   r[   Útorch.nnr2   Útorch.nn.functionalÚ
functionalr‘   r
   Ú	torch.jitr   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_manipulater   r   Ú	_registryr   r   Ú__all__rI   r!   rM   rQ   rd   rn   rz   r¡   r®   r±   r´   rÄ   rÍ   rÛ   rò   r   rl  rz  r†  Údefault_cfgsr�  r”  r™  rŸ  r¤  r§  rª  r­  r¯  r±  r¶  rº  r¾  rÂ  rÄ  rÇ  rÊ  rÌ  rº   r;   r:   ú<module>rÜ     s¿  ðñõ8 $Ý ß 5Õ 5ã Ý ß Ð Ý Ý ç A÷
÷ 
õ 
õ +Ý +ß 3ß <àˆ.€ôˆ2�9‰9ô ô@�2—9‘9ô ôD/ˆB�I‰Iô /ô.
*�"—)‘)ô 
*ô:ˆr�y‰yô :ô69�—	‘	ô 9ô~�zô ô˜ô ô�b—l‘lô ô)ˆb�i‰iô )ôXˆb�i‰iô ôˆb�i‰iô ôD;�b—i‘iô ;ô|F�b—i‘iô FôRr�—‘ô ròl"óJó 	ñ %ð V&Ø¡ØØô!ðV&ð ¡ØØô!ð	V&ð ¡ØØô!ðV&ð ¡ØØô!ðV&ð ¡ØØô!ðV&ð" !¡$°Ô"9ð#V&ð$ !¡$°Ô"9ð%V&ð& !¡$°Ô"9ð'V&ð( !¡$°Ô"9ð)V&ð* !¡$°Ô"9ð+V&ð. ¡ØØ ô!"ð/V&ð4 #¡DØØ ¨]Àhô%Pð5V&ð: (©ØØ ô*"ð;V&ð@ ,©TØØ ¨]Àhô.PðAV&ðF  ¡ØØ ¨eô"5ðGV&ðN ¡ØØ ô!"ðOV&ðT #¡DØØ ¨]Àhô%PñUV&ðZ (©ØØ ô*"ð[V&ð` ,©TØØ ¨]Àhô.PðaV&ðf  ¡ØØ ¨eô"5ðgV&ðn ¡ØØ ô!"ðoV&ðt #¡DØØ ¨]Àhô%PðuV&ðz (©ØØ ô*"ð{V&ð@ ,©TØØ ¨]Àhô.PðAV&ðF  ¡ØØ ¨eô"5ðGV&ðN ¡ØØ ô!"ðOV&ðT #¡DØØ ¨]Àhô%PðUV&ðZ (©ØØ ô*"ð[V&ð` ,©TØØ ¨]Àhô.PðaV&ðf  ¡ØØ ¨eô"5ðgV&ðn ™dØØ ô"ðoV&ðt !¡$ØØ ¨]Àhô#PðuV&ðz &¡tØØ ô("ð{V&ð@ *©4ØØ ¨]Àhô,PòAV&ðF ™tØØ ¨eô 5ðGV&ðN ™dØØ ô"ðOV&ðT !¡$ØØ ¨]Àhô#PðUV&ðZ &¡tØØ ô("ð[V&ð` *©4ØØ ¨]Àhô,PðaV&ðf ™tØØ ¨eô 5ðgV&ðn ™dØØ ô"ðoV&ðt !¡$ØØ ¨]Àhô#PðuV&ðz &¡tØØ ô("ð{V&ð@ *©4ØØ ¨]Àhô,PðAV&ðF ™tØØ ¨eô 5ðGV&ðN ™dØØ ô"ðOV&ðT !¡$ØØ ¨]Àhô#PðUV&ðZ &¡tØØ ô("ð[V&ð` *©4ØØ ¨]Àhô,PðaV&ðf ™tØØ ¨eô 5ñgV&ó V€ðr ñW°*ò Wó ðWð ñW°*ò Wó ðWð ñW°*ò Wó ðWð ñW°*ò Wó ðWð ñW°*ò Wó ðWð ñY°Jò Yó ðYð ñY°Jò Yó ðYð ñY°Jò Yó ðYð ñY°Jò Yó ðYð ñY°Jò Yó ðYð ñW°*ò Wó ðWð ñW°*ò Wó ðWð ñW°*ò Wó ðWð ñW°*ò Wó ðWð ñU°
ò Uó ðUð ñU°
ò Uó ðUð ñU°
ò Uó ðUð ñU°
ò Uó ñUr;   