
    ^ju                     	   d Z ddlmZ ddlmZ ddlmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmc mZ ddlmZ ddlmZ ddlmZmZ dd	lmZmZmZmZmZmZmZmZm Z  d
dl!m"Z" d
dl#m$Z$ d
dl%m&Z&m'Z' d
dl(m)Z)m*Z* dgZ+ G d dejX                        Z- G d dejX                        Z. G d dejX                        Z/ G d dejX                        Z0 G d dejX                        Z1 G d dejX                        Z2 G d de      Z3 G d de      Z4 G d  d!ej:                        Z5 G d" d#ejX                        Z6 G d$ d%ejX                        Z7 G d& d'ejX                        Z8 G d( d)ejX                        Z9 G d* d+ejX                        Z: G d, dejX                        Z;d- Z<dd.Z=dd/Z> e)i d0 e>d1d23      d4 e>d1d23      d5 e>d1d23      d6 e>d1d73      d8 e>d1d73      d9 e>d1:      d; e>d1:      d< e>d1:      d= e>d1:      d> e>d1:      d? e>d1d@A      dB e>d1d@dCdDE      dF e>d1d@A      dG e>d1d@dCdDE      dH e>d1d@dIJ      dK e>d1d@A      dL e>d1d@dCdDE      i dM e>d1d@A      dN e>d1d@dCdDE      dO e>d1d@dIJ      dP e>d1d@A      dQ e>d1d@dCdDE      dR e>d1d@A      dS e>d1d@dCdDE      dT e>d1d@dIJ      dU e>d1d@A      dV e>d1d@dCdDE      dW e>d1d@A      dX e>d1d@dCdDE      dY e>d1d@dIJ      dZ e>d1d@A      d[ e>d1d@dCdDE      d\ e>d1d@A      d] e>d1d@dCdDE      i d^ e>d1d@dIJ      d_ e>d1d@A      d` e>d1d@dCdDE      da e>d1d@A      db e>d1d@dCdDE      dc e>d1d@dIJ      dd e>d1d@A      de e>d1d@dCdDE      df e>d1d@A      dg e>d1d@dCdDE      dh e>d1d@dIJ      di e>d1d@A      dj e>d1d@dCdDE      dk e>d1d@A      dl e>d1d@dCdDE      dm e>d1d@dIJ            Z?e*ddne;fdo       Z@e*ddne;fdp       ZAe*ddne;fdq       ZBe*ddne;fdr       ZCe*ddne;fds       ZDe*ddne;fdt       ZEe*ddne;fdu       ZFe*ddne;fdv       ZGe*ddne;fdw       ZHe*ddne;fdx       ZIe*ddne;fdy       ZJe*ddne;fdz       ZKe*ddne;fd{       ZLe*ddne;fd|       ZMe*ddne;fd}       ZNe*ddne;fd~       ZOe*ddne;fd       ZPe*ddne;fd       ZQy)a  
Poolformer from MetaFormer is Actually What You Need for Vision https://arxiv.org/abs/2111.11418

IdentityFormer, RandFormer, PoolFormerV2, ConvFormer, and CAFormer
from MetaFormer Baselines for Vision https://arxiv.org/abs/2210.13452

All implemented models support feature extraction and variable input resolution.

Original implementation by Weihao Yu et al.,
adapted for timm by Fredo Guan and Ross Wightman.

Adapted from https://github.com/sail-sg/metaformer, original copyright below
    )OrderedDict)partial)ListOptionalTupleUnionTypeN)Tensor)FinalIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)	trunc_normal_DropPathcalculate_drop_path_ratesSelectAdaptivePool2d
GroupNorm1	LayerNormLayerNorm2dMlpuse_fused_attn   )build_model_with_cfg)feature_take_indices)
checkpointcheckpoint_seq)generate_default_cfgsregister_model
MetaFormerc            	       ^     e Zd ZdZ	 	 	 ddededeeej                        f fdZ	d Z
 xZS )Stemzc
    Stem implemented by a layer of convolution.
    Conv2d params constant across all models.
    in_channelsout_channels
norm_layerc                     ||d}t         |           t        j                  ||fdddd|| _        |r ||fi || _        y t        j
                         | _        y )Ndevicedtype         kernel_sizestridepadding)super__init__nnConv2dconvIdentitynorm)selfr"   r#   r$   r'   r(   dd	__class__s          a/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/metaformer.pyr1   zStem.__init__A   sl     /II
 
 
	 7AJ|2r2	bkkm	    c                 J    | j                  |      }| j                  |      }|S N)r4   r6   r7   xs     r:   forwardzStem.forwardU   !    IIaLIIaLr;   NNN__name__
__module____qualname____doc__intr   r	   r2   Moduler1   r@   __classcell__r9   s   @r:   r!   r!   ;   sJ     59TT T !bii1	T(r;   r!   c                   n     e Zd ZdZ	 	 	 	 	 d
dedededededeeej                        f fdZ	d	 Z
 xZS )Downsamplingz=
    Downsampling implemented by a layer of convolution.
    r"   r#   r-   r.   r/   r$   c	                     ||d}	t         
|           |r	 ||fi |	nt        j                         | _        t        j
                  ||f|||d|	| _        y )Nr&   r,   )r0   r1   r2   r5   r6   r3   r4   )r7   r"   r#   r-   r.   r/   r$   r'   r(   r8   r9   s             r:   r1   zDownsampling.__init__`   se     /5?J{1b1R[[]	II
 $
 
	r;   c                 J    | j                  |      }| j                  |      }|S r=   )r6   r4   r>   s     r:   r@   zDownsampling.forwardw   rA   r;   )r   r   NNNrC   rK   s   @r:   rM   rM   [   sj     48

 
 	

 
 
 !bii1
.r;   rM   c            	       F     e Zd ZdZ	 	 	 	 	 ddedededef fdZd Z xZ	S )	Scalez2
    Scale vector by element multiplications.
    dim
init_value	trainableuse_nchwc                     ||d}t         |           |r|ddfn|f| _        t        j                  |t        j                  |fi |z  |      | _        y )Nr&   r   requires_grad)r0   r1   shaper2   	Parametertorchonesscale)	r7   rR   rS   rT   rU   r'   r(   r8   r9   s	           r:   r1   zScale.__init__   sS     /$,c1a[3&
\\*uzz#/D/D"DT]^
r;   c                 R    || j                   j                  | j                        z  S r=   )r]   viewrY   r>   s     r:   r@   zScale.forward   s    4::??4::...r;   )      ?TTNN)
rD   rE   rF   rG   rH   floatboolr1   r@   rJ   rK   s   @r:   rQ   rQ   }   sN     !$"!__ _ 	_
 _/r;   rQ   c                   0     e Zd ZdZddef fdZd Z xZS )SquaredReLUz<
        Squared ReLU: https://arxiv.org/abs/2109.08668
    inplacec                 X    t         |           t        j                  |      | _        y )Nre   )r0   r1   r2   ReLUrelu)r7   re   r9   s     r:   r1   zSquaredReLU.__init__   s    GGG,	r;   c                 J    t        j                  | j                  |            S r=   )r[   squareri   r>   s     r:   r@   zSquaredReLU.forward   s    ||DIIaL))r;   F)rD   rE   rF   rG   rb   r1   r@   rJ   rK   s   @r:   rd   rd      s    - -*r;   rd   c                   Z     e Zd ZdZ	 	 	 	 	 	 	 	 d
dededededee   def fdZd	 Z	 xZ
S )StarReLUz(
    StarReLU: s * relu(x) ** 2 + b
    scale_value
bias_valuescale_learnablebias_learnablemodere   c	                 8   ||d}	t         
|           || _        t        j                  |      | _        t        j                  |t        j                  di |	z  |      | _	        t        j                  |t        j                  di |	z  |      | _
        y )Nr&   rg   rW   )r   )r0   r1   re   r2   rh   ri   rZ   r[   r\   r]   bias)r7   ro   rp   rq   rr   rs   re   r'   r(   r8   r9   s             r:   r1   zStarReLU.__init__   sy     /GGG,	\\+

0C0C"CSbc
LLejj.Ab.A!AQ_`	r;   c                 ^    | j                   | j                  |      dz  z  | j                  z   S )Nr+   )r]   ri   ru   r>   s     r:   r@   zStarReLU.forward   s'    zzDIIaLA--		99r;   )r`           TTNFNN)rD   rE   rF   rG   ra   rb   r   strr1   r@   rJ   rK   s   @r:   rn   rn      sq     "% #$(#'"&!aa a "	a
 !a 3-a a$:r;   rn   c                   p     e Zd ZU dZee   ed<   	 	 	 	 	 	 	 	 ddededee   dede	de	d	ef fd
Z
d Z xZS )	Attentionzl
    Vanilla self-attention from Transformer: https://arxiv.org/abs/1706.03762.
    Modified from timm.
    
fused_attnrR   head_dim	num_headsqkv_bias	attn_drop	proj_drop	proj_biasc
                    ||	d}t         |           || _        |dz  | _        t	               | _        |r|n||z  | _        | j                  dk(  rd| _        | j                  | j                  z  | _        t        j                  || j                  dz  fd|i|| _
        t        j                  |      | _        t        j                  | j                  |fd|i|| _        t        j                  |      | _        y )Nr&   g      r   r      ru   )r0   r1   r|   r]   r   r{   r}   attention_dimr2   LinearqkvDropoutr   projr   )r7   rR   r|   r}   r~   r   r   r   r'   r(   kwargsr8   r9   s               r:   r1   zAttention.__init__   s     / %
(*&/SH_>>QDN!^^dmm;99S$"4"4q"8NxN2NI.IId00#LILL	I.r;   c                    |j                   \  }}}| j                  |      j                  ||d| j                  | j                        j                  ddddd      }|j                  d      \  }}}| j                  r<t        j                  |||| j                  r| j                  j                  nd      }nJ||j                  dd	      z  | j                  z  }	|	j                  d	
      }	| j                  |	      }	|	|z  }|j                  dd      j                  |||      }| j!                  |      }| j#                  |      }|S )Nr   r+   r   r   r*   rw   )	dropout_p)rR   )rY   r   reshaper}   r|   permuteunbindr{   Fscaled_dot_product_attentiontrainingr   p	transposer]   softmaxr   r   )
r7   r?   BNCr   qkvattns
             r:   r@   zAttention.forward   s   ''1ahhqk!!!Q4>>4==IQQRSUVXY[\^_`**Q-1a??..1a.2mm$..**A
 B++tzz9D<<B<'D>>$'DqAKK1%%aA.IIaLNN1r;   )    NFrw   rw   FNN)rD   rE   rF   rG   r   rb   __annotations__rH   r   ra   r1   r@   rJ   rK   s   @r:   rz   rz      s     d
 '+"!!#// /  }	/
 / / / />r;   rz   c                   $     e Zd Zdef fdZ xZS )GroupNorm1NoBiasnum_channelsc                 b    t        |   |fi | |j                  dd      | _        d | _        y Nepsư>r0   r1   getr   ru   r7   r   r   r9   s      r:   r1   zGroupNorm1NoBias.__init__   .    00::eT*	r;   rD   rE   rF   rH   r1   rJ   rK   s   @r:   r   r          S  r;   r   c                   $     e Zd Zdef fdZ xZS )LayerNorm2dNoBiasr   c                 b    t        |   |fi | |j                  dd      | _        d | _        y r   r   r   s      r:   r1   zLayerNorm2dNoBias.__init__  r   r;   r   rK   s   @r:   r   r     r   r;   r   c                   $     e Zd Zdef fdZ xZS )LayerNormNoBiasr   c                 b    t        |   |fi | |j                  dd      | _        d | _        y r   r   r   s      r:   r1   zLayerNormNoBias.__init__
  r   r;   r   rK   s   @r:   r   r   	  r   r;   r   c                        e Zd ZdZdeej                  dddddfdeded	e	ej                     d
e	ej                     dededef fdZd Z xZS )SepConvz\
    Inverted separable convolution from MobileNetV2: https://arxiv.org/abs/1801.04381.
    r+   Fr)   r   NrR   expansion_ratio
act1_layer
act2_layerru   r-   r/   c
                    ||	d}t         |           t        ||z        }t        j                  ||fd|d|| _        t        |t              r |di |n |       | _        t        j                  ||f||||d|| _	        t        |t              r |di |n |       | _
        t        j                  ||fd|d|| _        y )Nr&   r   )r-   ru   )r-   r/   groupsru    )r0   r1   rH   r2   r3   pwconv1
issubclassrn   act1dwconvact2pwconv2)r7   rR   r   r   r   ru   r-   r/   r'   r(   r   r8   mid_channelsr9   s                r:   r1   zSepConv.__init__  s     /?S01yylSSPRS(2:x(HJ$$jl	ii
 $
 
 )3:x(HJ$$jl	yysSSPRSr;   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }|S r=   )r   r   r   r   r   r>   s     r:   r@   zSepConv.forward3  sH    LLOIIaLKKNIIaLLLOr;   )rD   rE   rF   rG   rn   r2   r5   rH   ra   r	   rI   rb   r1   r@   rJ   rK   s   @r:   r   r     s     &'*2*,++ TT #T RYY	T
 RYYT T T T<r;   r   c                   0     e Zd ZdZddef fdZd Z xZS )PoolingzT
    Implementation of pooling for PoolFormer: https://arxiv.org/abs/2111.11418
    	pool_sizec                 d    t         |           t        j                  |d|dz  d      | _        y )Nr   r+   F)r.   r/   count_include_pad)r0   r1   r2   	AvgPool2dpool)r7   r   r   r9   s      r:   r1   zPooling.__init__A  s)    LL1i1n`ef	r;   c                 .    | j                  |      }||z
  S r=   )r   )r7   r?   ys      r:   r@   zPooling.forwardE  s    IIaL1ur;   )r   )rD   rE   rF   rG   rH   r1   r@   rJ   rK   s   @r:   r   r   <  s    g# gr;   r   c                        e Zd ZdZddeeddddfdeded	ed
ee	j                     dee	j                     dedef fdZd Z xZS )MlpHeadz MLP classification head
      r*   rw   TNrR   num_classes	mlp_ratio	act_layerr$   	drop_rateru   c
                 (   ||	d}
t         |           t        ||z        }t        j                  ||fd|i|
| _         |       | _         ||fi |
| _        t        j                  ||fd|i|
| _        t        j                  |      | _
        y )Nr&   ru   )r0   r1   rH   r2   r   fc1actr6   fc2r   	head_drop)r7   rR   r   r   r   r$   r   ru   r'   r(   r8   hidden_featuresr9   s               r:   r1   zMlpHead.__init__N  s     /i#o.99S/CCC;5"5	99_kKKKI.r;   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }|S r=   )r   r   r6   r   r   r>   s     r:   r@   zMlpHead.forwardc  sI    HHQKHHQKIIaLNN1HHQKr;   )rD   rE   rF   rG   rd   r   rH   ra   r	   r2   rI   rb   r1   r@   rJ   rK   s   @r:   r   r   J  s      $ )4*3!// / 	/
 BII/ RYY/ / /*r;   r   c                        e Zd ZdZeededddddddfdedee	j                     dee	j                     d	ed
ee	j                     dedededee   dee   f fdZd Z xZS )MetaFormerBlockz1
    Implementation of one MetaFormer block.
    Frw   TNrR   token_mixermlp_actmlp_biasr$   r   	drop_pathrU   layer_scale_init_valueres_scale_init_valuec                    ||d}t         |           t        t        f||	|d|}t        t        f||
|d|} ||fi || _         |d||d||| _        |dkD  rt        |      nt        j                         | _	        |	 |       nt        j                         | _
        |
 |       nt        j                         | _         ||fi || _        t        |t        d|z        f||||d|| _        |dkD  rt        |      nt        j                         | _        |	 |       nt        j                         | _        |
 |       | _        y t        j                         | _        y )Nr&   )rR   rS   rU   )rR   r   rw   r*   )r   ru   dropuse_convr   )r0   r1   r   rQ   norm1r   r   r2   r5   
drop_path1layer_scale1
res_scale1norm2r   rH   mlp
drop_path2layer_scale2
res_scale2)r7   rR   r   r   r   r$   r   r   rU   r   r   r'   r(   r   r8   ls_layerrs_layerr9   s                    r:   r1   zMetaFormerBlock.__init__q  sa     /5fc6LW_fcef5dc6JU]dacd*r*
&T3)TrTVT1:R(9-R[[]*@*LHJRTR]R]R_(<(H(*bkkm*r*
CL
 
 
 2;R(9-R[[]*@*LHJRTR]R]R_(<(H(*bkkmr;   c           
      F   | j                  |      | j                  | j                  | j                  | j	                  |                        z   }| j                  |      | j                  | j                  | j                  | j                  |                        z   }|S r=   )
r   r   r   r   r   r   r   r   r   r   r>   s     r:   r@   zMetaFormerBlock.forward  s    OOA$$TZZ]3 OOAHHTZZ]+ r;   )rD   rE   rF   rG   r   rn   r   rH   r	   r2   rI   rb   ra   r   r1   r@   rJ   rK   s   @r:   r   r   l  s     ,3'/"*5!!!6:48'\'\ bii'\ "))_	'\
 '\ RYY'\ '\ '\ '\ %-UO'\ #+5/'\Rr;   r   c                   L    e Zd Zdej                  edeeddgdz  ddddfdedededeej                     d	eej                     d
e
deeej                        deej                     dedee   dee   dee   f fdZej                   j"                  dd       ZdefdZ xZS )MetaFormerStager+   Frw   Nin_chsout_chsdepthr   r   r   downsample_normr$   r   dp_ratesr   r   c                 r   ||d}t         |           d| _        t        |t               | _        ||k(  rt        j                         nt        ||fddd|d|| _	        t        j                  t        |      D cg c]'  }t        d||||||	|
|   ||| j
                  d
||) c} | _        y c c}w )	Nr&   Fr   r+   r   )r-   r.   r/   r$   )
rR   r   r   r   r$   r   r   r   r   rU   r   )r0   r1   grad_checkpointingr   rz   rU   r2   r5   rM   
downsample
Sequentialranger   blocks)r7   r   r   r   r   r   r   r   r$   r   r   r   r   r'   r(   r   r8   ir9   s                     r:   r1   zMetaFormerStage.__init__  s    $ /"'&{I>> ,2W+<"++-,C
 &C
 C
 mm <&!  '6 '
#!qk#9!5]]'
 '
 '
 &! " &!s   >,B4c                     || _         y r=   )r   )r7   enables     r:   set_grad_checkpointingz&MetaFormerStage.set_grad_checkpointing  s
    "(r;   r?   c                    | j                  |      }|j                  \  }}}}| j                  s#|j                  ||d      j	                  dd      }| j
                  r5t        j                  j                         st        | j                  |      }n| j                  |      }| j                  s$|j	                  dd      j                  ||||      }|S )Nr   r   r+   )r   rY   rU   r   r   r   r[   jitis_scriptingr   r   )r7   r?   r   r   HWs         r:   r@   zMetaFormerStage.forward  s    OOAWW
1a}}		!Q#--a3A""599+A+A+Ct{{A.AAA}}Aq!))!Q15Ar;   T)rD   rE   rF   r2   r5   rn   r   rH   r	   rI   rb   r   ra   r   r1   r[   r  ignorer  r
   r@   rJ   rK   s   @r:   r   r     s
    +-;;'/"9D*5!%'D1H6:48/"/" /" 	/"
 bii/" "))_/" /" &d299o6/" RYY/" /" 5k/" %-UO/" #+5//"b YY) ) r;   r   c            #       r    e Zd ZdZdddddeeddddd	d
eeedd	d	fdedede	de
edf   de
edf   deeej                     eeej                        f   deej                     dededededeeeee   f      dee
ee   df   eee      f   deeej                        deeej                     eeej                        f   deej                     def" fdZd Zej,                  j.                  d2d        Zej,                  j.                  d!ej                  fd"       Zd3dedee	   fd#Z	 	 	 	 	 d4d$ej6                  d%eeeee   f      d&ed'ed(e	d)ed!eeej6                     e
ej6                  eej6                     f   f   fd*Z	 	 	 d5d%eeee   f   d+ed,efd-Zd6d$ed.efd/Zd$efd0Zd$efd1Z  xZ!S )7r   aM   MetaFormer
        A PyTorch impl of : `MetaFormer Baselines for Vision`  -
          https://arxiv.org/abs/2210.13452

    Args:
        in_chans (int): Number of input image channels.
        num_classes (int): Number of classes for classification head.
        global_pool: Pooling for classifier head.
        depths (list or tuple): Number of blocks at each stage.
        dims (list or tuple): Feature dimension at each stage.
        token_mixers (list, tuple or token_fcn): Token mixer for each stage.
        mlp_act: Activation layer for MLP.
        mlp_bias (boolean): Enable or disable mlp bias term.
        drop_path_rate (float): Stochastic depth rate.
        drop_rate (float): Dropout rate.
        layer_scale_init_values (list, tuple, float or None): Init value for Layer Scale.
            None means not use the layer scale. Form: https://arxiv.org/abs/2103.17239.
        res_scale_init_values (list, tuple, float or None): Init value for res Scale on residual connections.
            None means not use the res scale. From: https://arxiv.org/abs/2110.09456.
        downsample_norm (nn.Module): Norm layer used in stem and downsampling layers.
        norm_layers (list, tuple or norm_fcn): Norm layers for each stage.
        output_norm: Norm layer before classifier head.
        use_mlp_head: Use MLP classification head.
    r   r   avgr+   r+      r+   @      i@     Frw   N)NNr`   r`   Tin_chansr   global_pooldepths.dimstoken_mixersr   r   drop_path_rateproj_drop_rater   layer_scale_init_valuesres_scale_init_valuesr   norm_layersoutput_normuse_mlp_headc                    t         |           ||d}|t        fv rt        |fi |}|| _        || _        |d   | _        || _        || _        t        |      | _
        t        |t        t        f      s|g}t        |t        t        f      s|g}t        |t        t        f      s|g| j                  z  }t        |t        t        f      s|g| j                  z  }t        |t        t        f      s|g| j                  z  }t        |t        t        f      s|g| j                  z  }d| _        g | _        t!        ||d   fd|i|| _        g }|d   }t%        |	|d      }t'        | j                        D ]h  }|t)        |||   f||   ||   |||
||   ||   ||   |||   d
||gz  }||   }| xj                  t+        ||   d	|d	z   z  d
|       gz  c_        j t-        j.                  | | _        |dkD  rt| j                  r5t3        | j                  |fd| j                  i|}| j                  | _        nGt-        j6                  | j                  |fi |}| j                  | _        nt-        j8                         }t-        j.                  t;        dt=        |      fd || j                  fi |fd|rt-        j>                  d      nt-        j8                         fd| j                  rt-        j@                  |      nt-        j8                         fd|fg            | _!        | jE                  | jF                         y )Nr&   r   Fr   r$   T)	stagewise)
r   r   r   r   r   r   r   r   r   r$   r+   zstages.)num_chs	reductionmoduler   r  	pool_typer6   flattenr   r   fc)$r0   r1   rn   r   r   r  num_featuresr   r  len
num_stages
isinstancelisttupler   feature_infor!   stemr   r   r   dictr2   r   stagesr   head_hidden_sizer   r5   r   r   Flattenr   headapply_init_weights)r7   r  r   r  r  r  r  r   r   r  r  r   r  r  r   r  r  r  r'   r(   r   r8   r/  prev_dimr   r   finalr9   s                              r:   r1   zMetaFormer.__init__  sT   . 	/xk!g,,G&  H"(f+ &4-0XF$u.6D,u6(>DOO;L+e}5&-$//9K1D%=A'>&?$//&Q#/$?%:$;doo$M!"'G
 '
 	
	 7,^VtTt' 	cAQ Qi(O!(!!'>q'A%:1%= /&q>    F  AwH$tAw!ac(U\]^\_S`"a!bb%	c( mmV, ?   1 1;_$.._\^_(,(9(9%		$"3"3[GBG(,(9(9%KKMEMM+0;GH[!2!29b9:

1"++-Hd.?.?RZZ	*R[[]S5M/
 # 	 	

4%%&r;   c                     t        |t        j                  t        j                  f      rOt	        |j
                  d       |j                  +t        j                  j                  |j                  d       y y y )Ng{Gz?)stdr   )	r)  r2   r3   r   r   weightru   init	constant_)r7   ms     r:   r4  zMetaFormer._init_weightsx  sS    a"))RYY/0!((,vv!!!!&&!, " 1r;   c                 X    || _         | j                  D ]  }|j                  |        y )N)r   )r   r/  r  )r7   r   stages      r:   r  z!MetaFormer.set_grad_checkpointing~  s.    "([[ 	8E(((7	8r;   returnc                 .    | j                   j                  S r=   )r2  r%  )r7   s    r:   get_classifierzMetaFormer.get_classifier  s    yy||r;   c                    ||d}|| _         |Ut        |      | j                  _        |rt	        j
                  d      nt	        j                         | j                  _        |dkD  rR| j                  r$t        | j                  |fd| j                  i|}n6t	        j                  | j                  |fi |}nt	        j                         }|| j                  _        y )Nr&   r"  r   r   r   )r   r   r2  r  r2   r1  r5   r$  r  r   r&  r   r   r%  )r7   r   r  r'   r(   r8   r6  s          r:   reset_classifierzMetaFormer.reset_classifier  s    /&"$8;$ODII!1<

1"++-DII?   1 1;_$.._\^_		$"3"3[GBGKKME		r;   r?   indicesr6   
stop_early
output_fmtintermediates_onlyc                    |dv sJ d       g }t        t        | j                        |      \  }}	| j                  |      }t        j
                  j                         s|s| j                  }
n| j                  d|	dz    }
t        |
      D ]Z  \  }}| j                  r+t        j
                  j                         st        ||      }n ||      }||v sJ|j                  |       \ |r|S ||fS )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWzOutput shape must be NCHW.Nr   )r   r'  r/  r-  r[   r  r  	enumerater   r   append)r7   r?   rD  r6   rE  rF  rG  intermediatestake_indices	max_indexr/  feat_idxr>  s                r:   forward_intermediatesz MetaFormer.forward_intermediates  s    * Y&D(DD&"6s4;;7G"Qi IIaL99!!#:[[F[[)a-0F(0 	(OHe&&uyy/E/E/Gua(!H<'$$Q'	(   -r;   
prune_norm
prune_headc                     t        t        | j                        |      \  }}| j                  d|dz    | _        |r| j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r    )r   r'  r/  rC  )r7   rD  rQ  rR  rM  rN  s         r:   prune_intermediate_layersz$MetaFormer.prune_intermediate_layers  sM     #7s4;;7G"Qikk.9q=1!!!R(r;   
pre_logitsc                    | j                   j                  |      }| j                   j                  |      }| j                   j                  |      }| j                   j	                  |      }|r|S | j                   j                  |      S r=   )r2  r  r6   r$  r   r%  )r7   r?   rV  s      r:   forward_headzMetaFormer.forward_head  sg    II!!!$IINN1IIa IINN1q3DIILLO3r;   c                     | j                  |      }| j                  r6t        j                  j	                         st        | j                  |      }|S | j                  |      }|S r=   )r-  r   r[   r  r  r   r/  r>   s     r:   forward_featureszMetaFormer.forward_features  sR    IIaL""599+A+A+Ct{{A.A  AAr;   c                 J    | j                  |      }| j                  |      }|S r=   )rZ  rX  r>   s     r:   r@   zMetaFormer.forward  s'    !!!$a r;   r  rB   )NFFrI  F)r   FTrl   )"rD   rE   rF   rG   r   rn   r   r   rH   rx   r   r   r	   r2   rI   r   rb   ra   r   r1   r4  r[   r  r  r  rA  rC  r
   rP  rU  rX  rZ  r@   rJ   rK   s   @r:   r   r     s   6 #$&2$7JQ'/"$&$&"KO_u9JIZ+6!%)i'i' i' 	i'
 #s(Oi' S/i'  RYYd299o1F FGi' "))_i' i' "i' "i' i' &.eE4;4F.G%Hi' $)x/C)Dd8TY?F[)[#\i' &d299o6i'  tBIIT"))_0EEF!i'" bii#i'$ %i'V- YY8 8
 YY		  C hsm $ 8<$$',+ ||+  eCcN34+  	+ 
 +  +  !%+  
tELL!5tELL7I)I#JJ	K+ ^ ./$#	3S	>*  	4f 4$ 4&  r;   c                    d| v r| S dd l }i }d| v }|j                         }| j                         D ]  \  }}|r|j                  dd|      }|j	                  dd      }|j	                  dd	      }|j	                  d
d      }|j	                  dd      }|j	                  dd      }|j	                  dd      }|j	                  dd      }|j                  dd|      }|j	                  dd      }|j	                  dd      }|j                  dd|      }|j	                  dd      }|j	                  dd      }|j	                  dd      }|j	                  dd      }|j                  dd |      }|j                  d!d"|      }|j
                  ||   k7  rB|j                         ||   j                         k(  r|j                  ||   j
                        }|||<    |S )#Nzstem.conv.weightr   znetwork.0.0.mlp.fc1.weightzlayer_scale_([0-9]+)zlayer_scale\1.scalez	network.1zdownsample_layers.1z	network.3zdownsample_layers.2z	network.5zdownsample_layers.3z	network.2z	network.4z	network.6networkr/  zdownsample_layers.([0-9]+)zstages.\1.downsamplezdownsample.projzdownsample.convzpatch_embed.projzpatch_embed.convz([0-9]+).([0-9]+)z\1.blocks.\2zstages.0.downsamplepatch_embedr-  	post_normr6   pre_normz^headhead.fcz^normz	head.norm)re
state_dictitemssubreplacerY   numelr   )rc  modelrb  out_dictis_poolformerv1model_state_dictr   r   s           r:   checkpoint_filter_fnrl    s   Z'H2j@O'')  " 1.0FJA		+'<=A		+'<=A		+'<=A		+{3A		+{3A		+{3A		)X.AFF02I1MII'):;II(*<=FF'!<II+];IImV,IIk6*IIj&)FF8Y*FF8[!,77&q))aggi;KA;N;T;T;V.V		*1-334A34 Or;   c           	          t        d t        |j                  dd            D              }|j                  d|      }t	        t
        | |ft        t        d|      d|}|S )Nc              3   &   K   | ]	  \  }}|  y wr=   r   ).0r   _s      r:   	<genexpr>z%_create_metaformer.<locals>.<genexpr>  s     \da\s   r  r  out_indicesT)flatten_sequentialrr  )pretrained_filter_fnfeature_cfg)r+  rJ  r   popr   r   rl  r.  )variant
pretrainedr   default_out_indicesrr  rh  s         r:   _create_metaformerrz    sl    \i

8\8Z.[\\**],?@K  2DkJ E Lr;   c                 2    | dddddt         t        dddd	|S )
Nr   )r      r|  )r)   r)   r`   bicubicra  z	stem.convz
apache-2.0)urlr   
input_sizer   crop_pctinterpolationmeanr8  
classifier
first_convlicenser   )r~  r   s     r:   _cfgr    s3    =v)%.B{  r;   zpoolformer_s12.sail_in1kztimm/g?)	hf_hub_idr  zpoolformer_s24.sail_in1kzpoolformer_s36.sail_in1kzpoolformer_m36.sail_in1kgffffff?zpoolformer_m48.sail_in1kzpoolformerv2_s12.sail_in1k)r  zpoolformerv2_s24.sail_in1kzpoolformerv2_s36.sail_in1kzpoolformerv2_m36.sail_in1kzpoolformerv2_m48.sail_in1kzconvformer_s18.sail_in1kzhead.fc.fc2)r  r  zconvformer_s18.sail_in1k_384)r     r  )   r  )r  r  r  r   z!convformer_s18.sail_in22k_ft_in1kz%convformer_s18.sail_in22k_ft_in1k_384zconvformer_s18.sail_in22kiQU  )r  r  r   zconvformer_s36.sail_in1kzconvformer_s36.sail_in1k_384z!convformer_s36.sail_in22k_ft_in1kz%convformer_s36.sail_in22k_ft_in1k_384zconvformer_s36.sail_in22kzconvformer_m36.sail_in1kzconvformer_m36.sail_in1k_384z!convformer_m36.sail_in22k_ft_in1kz%convformer_m36.sail_in22k_ft_in1k_384zconvformer_m36.sail_in22kzconvformer_b36.sail_in1kzconvformer_b36.sail_in1k_384z!convformer_b36.sail_in22k_ft_in1kz%convformer_b36.sail_in22k_ft_in1k_384zconvformer_b36.sail_in22kzcaformer_s18.sail_in1kzcaformer_s18.sail_in1k_384zcaformer_s18.sail_in22k_ft_in1kz#caformer_s18.sail_in22k_ft_in1k_384zcaformer_s18.sail_in22kzcaformer_s36.sail_in1kzcaformer_s36.sail_in1k_384zcaformer_s36.sail_in22k_ft_in1kz#caformer_s36.sail_in22k_ft_in1k_384zcaformer_s36.sail_in22kzcaformer_m36.sail_in1kzcaformer_m36.sail_in1k_384zcaformer_m36.sail_in22k_ft_in1kz#caformer_m36.sail_in22k_ft_in1k_384zcaformer_m36.sail_in22kzcaformer_b36.sail_in1kzcaformer_b36.sail_in1k_384zcaformer_b36.sail_in22k_ft_in1kz#caformer_b36.sail_in22k_ft_in1k_384zcaformer_b36.sail_in22kr?  c                 r    t        dg dg dd t        j                  dt        dd dd	|}t	        d	d| i|S )
Nr  r  Th㈵>F	r  r  r   r   r   r  r  r  r  rx  r   )poolformer_s12r.  r2   GELUr   rz  rx  r   model_kwargss      r:   r  r    sO     
  $"
 
L V:VVVr;   c                 r    t        dg dg dd t        j                  dt        dd dd	|}t	        d	d| i|S )
Nr*   r*   r  r*   r  Tr  Fr  rx  r   )poolformer_s24r  r  s      r:   r  r    O     
  $"
 
L V:VVVr;   c                 r    t        dg dg dd t        j                  dt        dd dd	|}t	        d	d| i|S )
Nr  r     r  r  Tr   Fr  rx  r   )poolformer_s36r  r  s      r:   r  r    r  r;   c                 r    t        dg dg dd t        j                  dt        dd dd	|}t	        d	d| i|S )
Nr  `      r     Tr   Fr  rx  r   )poolformer_m36r  r  s      r:   r  r    r  r;   c                 r    t        dg dg dd t        j                  dt        dd dd	|}t	        d	d| i|S )
N   r     r  r  Tr   Fr  rx  r   )poolformer_m48r  r  s      r:   r  r    r  r;   c                 L    t        dg dg dt        dd|}t        dd| i|S )Nr  r  Fr  r  r  r  rx  r   )poolformerv2_s12r.  r   rz  r  s      r:   r  r    s<      $	
 L XZX<XXr;   c                 L    t        dg dg dt        dd|}t        dd| i|S )Nr  r  Fr  rx  r   )poolformerv2_s24r  r  s      r:   r  r    <      $	
 L XZX<XXr;   c                 L    t        dg dg dt        dd|}t        dd| i|S )Nr  r  Fr  rx  r   )poolformerv2_s36r  r  s      r:   r  r  )  r  r;   c                 L    t        dg dg dt        dd|}t        dd| i|S )Nr  r  Fr  rx  r   )poolformerv2_m36r  r  s      r:   r  r  4  r  r;   c                 L    t        dg dg dt        dd|}t        dd| i|S )Nr  r  Fr  rx  r   )poolformerv2_m48r  r  s      r:   r  r  ?  r  r;   c                 T    t        dg dg dt        t        d|}t        dd| i|S )Nr   r   	   r   r  r  r  r  r  rx  r   )convformer_s18r.  r   r   rz  r  s      r:   r  r  J  s<      %	
 L V:VVVr;   c                 T    t        dg dg dt        t        d|}t        dd| i|S )Nr   r  r  r   r  r  rx  r   )convformer_s36r  r  s      r:   r  r  U  <      %	
 L V:VVVr;   c                 T    t        dg dg dt        t        d|}t        dd| i|S )Nr  r  r  r  i@  r  rx  r   )convformer_m36r  r  s      r:   r  r  `  r  r;   c                 T    t        dg dg dt        t        d|}t        dd| i|S )Nr  r     r  r  r  rx  r   )convformer_b36r  r  s      r:   r  r  k  s<     !%	
 L V:VVVr;   c           	          t        dg dg dt        t        t        t        gt        gdz  t        gdz  z   d|}t        dd| i|S )Nr  r  r+   r  rx  r   )caformer_s18r.  r   rz   r   r   rz  r  s      r:   r  r  v  sZ      w	9=&'!+.?!.CC	
 L TT|TTr;   c           	          t        dg dg dt        t        t        t        gt        gdz  t        gdz  z   d|}t        dd| i|S )Nr  r  r+   r  rx  r   )caformer_s36r  r  s      r:   r  r    Z      w	9=&'!+.?!.CC	
 L TT|TTr;   c           	          t        dg dg dt        t        t        t        gt        gdz  t        gdz  z   d|}t        dd| i|S )Nr  r  r+   r  rx  r   )caformer_m36r  r  s      r:   r  r    r  r;   c           	          t        dg dg dt        t        t        t        gt        gdz  t        gdz  z   d|}t        dd| i|S )Nr  r  r+   r  rx  r   )caformer_b36r  r  s      r:   r  r    sZ     !w	9=&'!+.?!.CC	
 L TT|TTr;   rl   )rT  )RrG   collectionsr   	functoolsr   typingr   r   r   r   r	   r[   torch.nnr2   torch.nn.functional
functionalr   r
   	torch.jitr   	timm.datar   r   timm.layersr   r   r   r   r   r   r   r   r   _builderr   	_featuresr   _manipulater   r   	_registryr   r   __all__rI   r!   rM   rQ   rd   rn   rz   r   r   r   r   r   r   r   r   r   rl  rz  r  default_cfgsr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r;   r:   <module>r     s  8 $  5 5       A
 
 
 + + 3 <.299 @299 D/BII /.
*")) 
*:ryy :69		 9~z  bll )bii )Xbii bii D;bii ;|Fbii FRr rl"J 	 % V&!V& !	V& !V& !V& !V&" !$"9#V&$ !$"9%V&& !$"9'V&( !$"9)V&* !$"9+V&.  !"/V&4 #D ]h%P5V&: ( *";V&@ ,T ]h.PAV&F   e"5GV&N  !"OV&T #D ]h%PUV&Z ( *"[V&` ,T ]h.PaV&f   e"5gV&n  !"oV&t #D ]h%PuV&z ( *"{V&@ ,T ]h.PAV&F   e"5GV&N  !"OV&T #D ]h%PUV&Z ( *"[V&` ,T ]h.PaV&f   e"5gV&n d "oV&t !$ ]h#PuV&z &t ("{V&@ *4 ]h,PAV&F t e 5GV&N d "OV&T !$ ]h#PUV&Z &t ("[V&` *4 ]h,PaV&f t e 5gV&n d "oV&t !$ ]h#PuV&z &t ("{V&@ *4 ]h,PAV&F t e 5GV&N d "OV&T !$ ]h#PUV&Z &t ("[V&` *4 ]h,PaV&f t e 5gV& Vr W* W W W* W W W* W W W* W W W* W W YJ Y Y YJ Y Y YJ Y Y YJ Y Y YJ Y Y W* W W W* W W W* W W W* W W U
 U U U
 U U U
 U U U
 U Ur;   