
    ^jar              
       F   d Z dgZddlZddlmZ ddlmZ ddlmZm	Z	m
Z
mZmZmZ ddlZddlmZ ddlmZmZ ddlmZmZmZmZ d	d
lmZ d	dlmZ d	dlmZmZ d	dl m!Z!m"Z"  G d dej                   jF                        Z$ G d dej                   jF                        Z% G d dej                   jL                        Z' G d dej                   jL                        Z( G d dej                   jL                        Z) G d dej                   jL                        Z* G d dej                   jL                        Z+ G d dej                   jL                        Z, G d dej                   jL                        Z- G d  d!ej                   jF                        Z. G d" dejL                        Z/d.d#Z0 e" e0d$%       e0d$%       e0d$%       e0d$%       e0d$%       e0d$%      d&      Z1d/d'Z2e!d/d(       Z3e!d/d)       Z4e!d/d*       Z5e!d/d+       Z6e!d/d,       Z7e!d/d-       Z8y)0z EfficientViT (by MSRA)

Paper: `EfficientViT: Memory Efficient Vision Transformer with Cascaded Group Attention`
    - https://arxiv.org/abs/2305.07027

Adapted from official impl at https://github.com/microsoft/Cream/tree/main/EfficientViT
EfficientVitMsra    N)OrderedDict)partial)DictListOptionalTupleTypeUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)SqueezeExciteSelectAdaptivePool2dtrunc_normal__assert   )build_model_with_cfg)feature_take_indices)
checkpointcheckpoint_seq)register_modelgenerate_default_cfgsc                        e Zd Z	 	 	 	 	 	 	 	 ddedededededededef fd	Z ej                         d
        Z xZ	S )ConvNormin_chsout_chsksstridepaddilationgroupsbn_weight_initc           	      $   |	|
d}t         |           t        j                  |||||||fddi|| _        t        j
                  |fi || _        t        j                  j                  j                  | j                  j                  |       y )NdevicedtypebiasF)super__init__nnConv2dconvBatchNorm2dbntorchinit	constant_weight)selfr   r   r   r   r    r!   r"   r#   r&   r'   dd	__class__s               h/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/efficientvit_msra.pyr*   zConvNorm.__init__   sw     /IIfgr63&cW\c`bc	../B/?    c           	      B   | j                   | j                  }}|j                  |j                  |j                  z   dz  z  }|j                  |d d d d d f   z  }|j
                  |j                  |j                  z  |j                  |j                  z   dz  z  z
  }t        j                  j                  |j                  d      | j                   j                  z  |j                  d      |j                  dd  | j                   j                  | j                   j                  | j                   j                  | j                   j                        }|j                  j                   j#                  |       |j
                  j                   j#                  |       |S )N      ?r   r      )r   paddingr!   r"   )r-   r/   r3   running_varepsr(   running_meanr0   r+   r,   sizer"   shaper   r<   r!   datacopy_)r4   cr/   wbms         r7   fusezConvNorm.fuse.   s5   		4772II"&&0366HHqD$,--GGboo		1^^bff$s*+ +HHOOFF1I		(((!&&)QWWQR[99##TYY->->I[I[dhdmdmdtdt  v 	
A	!r8   )r   r   r   r   r   r   NN)
__name__
__module____qualname__intfloatr*   r0   no_gradrH   __classcell__r6   s   @r7   r   r      s    
 $%@@ @ 	@
 @ @ @ @ "@& U]]_ r8   r   c                   n     e Zd Z	 	 	 	 	 ddededededef
 fdZ ej                         d        Z	 xZ
S )	
NormLinearin_featuresout_featuresr(   stddropc                    ||d}t         	|           t        j                  |fi || _        t        j
                  |      | _        t        j                  ||fd|i|| _        t        | j                  j                  |       | j                  j                  5t        j                  j                  | j                  j                  d       y y )Nr%   r(   )rU   r   )r)   r*   r+   BatchNorm1dr/   DropoutrV   Linearlinearr   r3   r(   r1   r2   )
r4   rS   rT   r(   rU   rV   r&   r'   r5   r6   s
            r7   r*   zNormLinear.__init__>   s     /..33JJt$	ii\KKKdkk((c2;;'GGdkk..2 (r8   c                 J   | j                   | j                  }}|j                  |j                  |j                  z   dz  z  }|j
                  | j                   j                  | j                   j                  z  |j                  |j                  z   dz  z  z
  }|j                  |d d d f   z  }|j
                  $|| j                  j                  j                  z  }n<|j                  |d d d f   z  j                  d      | j                  j
                  z   }t        j                  j                  |j                  d      |j                  d            }|j                  j                  j                  |       |j
                  j                  j                  |       |S )Nr:   r   r   )r/   r[   r3   r=   r>   r(   r?   Tviewr0   r+   rZ   r@   rB   rC   )r4   r/   r[   rE   rF   rG   s         r7   rH   zNormLinear.fuseR   s6   WWdkkFII"&&0366GGdgg**GGNN nnrvv5;< <MMAdAgJ&;;DKK&&(((A1d7+11"58H8HHAHHOOAFF1Iqvvay1	A	!r8   )Tg{Gz?        NN)rI   rJ   rK   rL   boolrM   r*   r0   rN   rH   rO   rP   s   @r7   rR   rR   =   se    
 33 3 	3
 3 3( U]]_ r8   rR   c                   4     e Zd Z	 	 ddedef fdZd Z xZS )PatchMergingdimout_dimc                 4   ||d}t         |           t        |dz        }t        ||dddfi || _        t
        j                  j                         | _        t        ||dddfd|i|| _	        t        |dfi || _        t        ||dddfi || _        y )	Nr%      r   r      r;   r"   g      ?)r)   r*   rL   r   conv1r0   r+   ReLUactconv2r   seconv3)r4   rd   re   r&   r'   r5   hid_dimr6   s          r7   r*   zPatchMerging.__init__d   s     /cAg,c7Aq!:r:
88==?gw1aNN2N
33gw1a>2>
r8   c                     | j                  | j                  | j                  | j                  | j                  | j	                  |                                    }|S N)rn   rm   rk   rl   ri   r4   xs     r7   forwardzPatchMerging.forwardt   sA    JJtwwtxx

488DJJqM3J(KLMNr8   NNrI   rJ   rK   rL   r*   rt   rO   rP   s   @r7   rc   rc   c   s'    
 ?? ? r8   rc   c                   D     e Zd Zddej                  def fdZd Z xZS )ResidualDroprG   rV   c                 >    t         |           || _        || _        y rq   )r)   r*   rG   rV   )r4   rG   rV   r6   s      r7   r*   zResidualDrop.__init__z   s    	r8   c           	      v   | j                   r| j                  dkD  r|| j                  |      t        j                  |j                  d      ddd|j                        j                  | j                        j                  d| j                  z
        j                         z  z   S || j                  |      z   S )Nr   r   )r&   )
trainingrV   rG   r0   randr@   r&   ge_divdetachrr   s     r7   rt   zResidualDrop.forward   s    ==TYY]tvvay5::q	1a188$558S^CCDIIDVW]W]W_` ` ` tvvay= r8   )r`   )	rI   rJ   rK   r+   ModulerM   r*   rt   rO   rP   s   @r7   rx   rx   y   s    ")) 5 
!r8   rx   c                   4     e Zd Z	 	 ddedef fdZd Z xZS )ConvMlpedhc                     ||d}t         |           t        ||fi || _        t        j
                  j                         | _        t        ||fddi|| _        y )Nr%   r#   r   )	r)   r*   r   pw1r0   r+   rj   rk   pw2)r4   r   r   r&   r'   r5   r6   s         r7   r*   zConvMlp.__init__   sV     /B(R(88==?Ar:!:r:r8   c                 d    | j                  | j                  | j                  |                  }|S rq   )r   rk   r   rr   s     r7   rt   zConvMlp.forward   s&    HHTXXdhhqk*+r8   ru   rv   rP   s   @r7   r   r      s'    
 ;; ;r8   r   c                       e Zd ZU eeej                  f   ed<   	 	 	 	 	 	 	 ddededededede	edf   f fd	Z
ddZddZddZ ej                         d fd	       Zdej                   d
ej                  fdZd Z xZS )CascadedGroupAttentionattention_bias_cacherd   key_dim	num_heads
attn_ratio
resolutionkernels.c	                 6   ||d}	t         |           || _        |dz  | _        || _        t        ||z        | _        || _        g }
g }t        |      D ]  }|
j                  t        ||z  | j                  dz  | j                  z   fi |	       |j                  t        | j                  | j                  ||   d||   dz  fd| j                  i|	        t        j                  j                  |
      | _        t        j                  j                  |      | _        t        j                  j!                  t        j                  j#                         t        | j                  |z  |fddi|	      | _        || _        ||z  }||z  }t        j                  j)                  t        j*                  ||fi |	      | _        | j/                  dt        j*                  ||f|t        j0                        d	
       i | _        | j5                          y )Nr%   g      r;   r   r"   r#   r   attention_bias_idxsF)
persistent)r)   r*   r   scaler   rL   val_dimr   rangeappendr   r0   r+   
ModuleListqkvsdws
Sequentialrj   projr   	Parameteremptyattention_biasesregister_bufferlongr   reset_parameters)r4   rd   r   r   r   r   r   r&   r'   r5   r   r   iNnum_offsetsr6   s                  r7   r*   zCascadedGroupAttention.__init__   s    /"_
:/0$y! 	xAKK	!14<<!3Cdll3RYVXYZJJxdllGAJ7ST:YZ?vcgcocovsuvw	x HH''-	88&&s+HH''HHMMOT\\I-sK1KK
	
 %# :- % 2 25;;y+3\Y[3\ ]!KKAvUZZ@ 	 	

 %'! 	r8   returnc                     t         j                  j                  j                  | j                         | j                          y)z"Initialize parameters and buffers.N)r0   r+   r1   zeros_r   _init_buffersr4   s    r7   r   z'CascadedGroupAttention.reset_parameters   s*    T223r8   c                    t        t        j                  t        | j                        t        | j                                    }i }g }|D ]W  }|D ]P  }t        |d   |d   z
        t        |d   |d   z
        f}||vrt        |      ||<   |j                  ||          R Y | j                  j                  t        j                  |t        j                        j                  t        |      t        |                   y)z.Compute and fill non-persistent buffer values.r   r   )r'   N)list	itertoolsproductr   r   abslenr   r   rC   r0   tensorr   r_   )r4   pointsattention_offsetsidxsp1p2offsets          r7   r   z$CascadedGroupAttention._init_buffers   s    i''doo(>doo@VWX 	7B 7bebem,c"Q%"Q%-.@A!22034E0F%f--f56	7	7 	  &&u||D

'K'P'PQTU[Q\^abh^i'jkr8   c                 $    | j                          y)z"Initialize non-persistent buffers.N)r   r   s    r7   init_non_persistent_buffersz2CascadedGroupAttention.init_non_persistent_buffers   s    r8   c                 R    t         |   |       |r| j                  ri | _        y y y rq   )r)   trainr   )r4   moder6   s     r7   r   zCascadedGroupAttention.train   s)    dD--(*D% .4r8   r&   c                 4   t         j                  j                         s| j                  r| j                  d d | j
                  f   S t        |      }|| j                  vr*| j                  d d | j
                  f   | j                  |<   | j                  |   S rq   )r0   jit
is_tracingr{   r   r   strr   )r4   r&   
device_keys      r7   get_attention_biasesz+CascadedGroupAttention.get_attention_biases   s    99!T]]((D,D,D)DEEVJ!:!::8<8M8MaQUQiQiNi8j))*5,,Z88r8   c                    |j                   \  }}}}|j                  t        | j                        d      }g }|d   }| j	                  |j
                        }	t        t        | j                  | j                              D ]%  \  }
\  }}|
dkD  r|||
   z   } ||      }|j                  |d||      j                  | j                  | j                  | j                  gd      \  }}} ||      }|j                  d      |j                  d      |j                  d      }}}|| j                  z  }|j                  dd      |z  }||	|
   z   }|j!                  d      }||j                  dd      z  }|j                  || j                  ||      }|j#                  |       ( | j%                  t'        j(                  |d            }|S )Nr   )rd   r   r]   r;   )rA   chunkr   r   r   r&   	enumeratezipr   r_   splitr   r   flattenr   	transposesoftmaxr   r   r0   cat)r4   rs   BCHWfeats_in	feats_outfeat	attn_biashead_idxqkvr   qkvattns                    r7   rt   zCascadedGroupAttention.forward   s   WW
1a773tyy>q71	{--ahh7	$-c$))TXX.F$G 	# HjsC!|hx00t9Dii2q!,22DLL$,,PTP\P\3]cd2eGAq!AAiilAIIaL!))A,!qADJJA;;r2&*D)H--D<<B<'Dt~~b"--D99Qa3DT"	# IIeii	1-.r8   )   rg         r   r   r   NN)r   NT)rI   rJ   rK   r   r   r0   Tensor__annotations__rL   r	   r*   r   r   r   rN   r   r&   r   rt   rO   rP   s   @r7   r   r      s    sELL011	  '3, ,  ,  	, 
 ,  ,  38_, \
l U]]_+ +
95<< 9ELL 9r8   r   c                   `     e Zd ZdZ	 	 	 	 	 	 	 ddededededededeed	f   f fd
Zd Z xZS )LocalWindowAttentiona   Local Window Attention.

    Args:
        dim (int): Number of input channels.
        key_dim (int): The dimension for query and key.
        num_heads (int): Number of attention heads.
        attn_ratio (int): Multiplier for the query dim for value dimension.
        resolution (int): Input resolution.
        window_resolution (int): Local window resolution.
        kernels (List[int]): The kernel size of the dw conv on query.
    rd   r   r   r   r   window_resolutionr   .c
                     ||	d}
t         |           || _        || _        || _        |dkD  sJ d       || _        t        ||      }t        |||f|||d|
| _        y )Nr%   r   z"window_size must be greater than 0)r   r   r   )	r)   r*   rd   r   r   r   minr   r   r4   rd   r   r   r   r   r   r   r&   r'   r5   r6   s              r7   r*   zLocalWindowAttention.__init__  s     /"$ 1$J&JJ$!2 1:>*)
!(	

 
	r8   c           	         | j                   x}}|j                  \  }}}}t        ||k(  d||f d||f        t        ||k(  d||f d||f        || j                  k  r"|| j                  k  r| j	                  |      }|S |j                  dddd      }| j                  || j                  z  z
  | j                  z  }| j                  || j                  z  z
  | j                  z  }	t        j                  j                  j                  |ddd|	d|f      }||z   ||	z   }}
|
| j                  z  }|| j                  z  }|j                  ||| j                  || j                  |      j                  dd      }|j                  ||z  |z  | j                  | j                  |      j                  dddd      }| j	                  |      }|j                  dddd      j                  |||| j                  | j                  |      }|j                  dd      j                  ||
||      }|d d d |d |f   j                         }|j                  dddd      }|S )Nz%input feature has wrong size, expect z, got r   r;   rh   r   )r   rA   r   r   r   permuter0   r+   
functionalr    r_   r   reshape
contiguous)r4   rs   r   r   r   r   H_W_pad_bpad_rpHpWnHnWs                 r7   rt   zLocalWindowAttention.forward:  sg   Aww1b"R@!QPRTVxjYZR@!QPRTVxjYZ&&&10F0F+F		!A& # 		!Q1%A++a$2H2H.HHDLbLbbE++a$2H2H.HHDLbLbbE##''Aq!UAu+EFAYE	Bt---Bt---Bq"d44b$:P:PRST^^_`bcdA		!b&2+t'='=t?U?UWXYaabcefhiklmA		!A		!Q1%**1b"d6L6LdNdNdfghAAq!))!RQ7A!RaR!)'')A		!Q1%Ar8   )r   rg   r      r   NN)	rI   rJ   rK   __doc__rL   r	   r*   rt   rO   rP   s   @r7   r   r     su    
  %&'3

 
 	

 
 
  #
 38_
8r8   r   c                   `     e Zd ZdZddddg dddfded	ed
ededededee   f fdZd Z xZS )EfficientVitBlocka   A basic EfficientVit building block.

    Args:
        dim (int): Number of input channels.
        key_dim (int): Dimension for query and key in the token mixer.
        num_heads (int): Number of attention heads.
        attn_ratio (int): Multiplier for the query dim for value dimension.
        resolution (int): Input resolution.
        window_resolution (int): Local window resolution.
        kernels (List[int]): The kernel size of the dw conv on query.
    r   rg   r   r   r   Nrd   r   r   r   r   r   r   c
           
         ||	d}
t         |           t        t        ||dddf|dd|
      | _        t        t        |t        |dz        fi |
      | _        t        t        |||f||||d|
      | _	        t        t        ||dddf|dd|
      | _
        t        t        |t        |dz        fi |
      | _        y )Nr%   rh   r   r`   )r"   r#   r;   )r   r   r   r   )r)   r*   rx   r   dw0r   rL   ffn0r   mixerdw1ffn1r   s              r7   r*   zEfficientVitBlock.__init__c  s     /c1a `3WY `]_ `a c#'l!Ab!AB	! Wi%%"3 	

  c1a `3WY `]_ `a c#'l!Ab!AB	r8   c                     | j                  | j                  | j                  | j                  | j	                  |                              S rq   )r   r   r   r   r   rr   s     r7   rt   zEfficientVitBlock.forward  s4    yy$**TYYtxx{-C"DEFFr8   )	rI   rJ   rK   r   rL   r   r*   rt   rO   rP   s   @r7   r   r   W  s{    
  %&!-CC C 	C
 C C  #C #YC@Gr8   r   c                   v     e Zd Zdddddg ddddf	d	ed
ededeeef   dededededee   def fdZd Z xZ	S )EfficientVitStage r   r   rg   r   r   r   r   Nin_dimre   r   
downsampler   r   r   r   r   depthc                    ||d}t         |           |d   dk(  r+|dz
  |d   z  dz   | _        g }|j                  dt        j
                  j                  t        t        ||dddfd|i|      t        t        |t        |dz        fi |            f       |j                  d	t        ||fi |f       |j                  d
t        j
                  j                  t        t        ||dddfd|i|      t        t        |t        |dz        fi |            f       t        j                  t        |            | _        n'||k(  sJ t        j                         | _        || _        g }t        |
      D ]-  }|j                  t!        ||||| j                  ||	fi |       / t        j                  | | _        y )Nr%   r   	subsampler   res1rh   r"   r;   
patchmergeres2)r)   r*   r   r   r0   r+   r   rx   r   r   rL   rc   r   r  Identityr   r   blocks)r4   r  re   r   r  r   r   r   r   r   r  r&   r'   r5   down_blocksr  dr6   s                    r7   r*   zEfficientVitStage.__init__  s    /a=K')A~*Q-?!CDOK## &&!Q!W&!WTV!WX VaZ!GB!GH   l67.Qb.QRS## '7Aq!!ZG!ZWY!Z[ #gk2B!Ib!IJ   !mmK,DEDOW$$$ kkmDO(DOu 
	AMM+!	 	 	
	 mmV,r8   c                 J    | j                  |      }| j                  |      }|S rq   )r  r  rr   s     r7   rt   zEfficientVitStage.forward  s"    OOAKKNr8   )
rI   rJ   rK   rL   r	   r   r   r*   rt   rO   rP   s   @r7   r  r    s     +2 %&!-5-5- 5- 	5-
 c3h5- 5- 5- 5-  #5- #Y5- 5-nr8   r  c                   .     e Zd Z	 	 ddedef fdZ xZS )PatchEmbeddingin_chansrd   c           
      r   t         |           ||d}| j                  dt        ||dz  dddfi |       | j                  dt        j
                  j                                | j                  dt        |dz  |d	z  dddfi |       | j                  d
t        j
                  j                                | j                  dt        |d	z  |dz  dddfi |       | j                  dt        j
                  j                                | j                  dt        |dz  |dddfi |       d| _        y )Nr%   ri   r   rh   r;   r   relu1rl   rg   relu2rn   relu3conv4   )r)   r*   
add_moduler   r0   r+   rj   
patch_size)r4   r  rd   r&   r'   r5   r6   s         r7   r*   zPatchEmbedding.__init__  s    	/(C1HaA!L!LM1#(C1HaA!L!LM1#(C1HaA!L!LM1#(CAq!GB!GHr8   ru   )rI   rJ   rK   rL   r*   rO   rP   s   @r7   r  r    s'    
   r8   r  c                       e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 d(dedededeedf   deedf   deedf   d	eedf   d
eedf   deedf   deeeef   df   dedef fdZd)defdZ	d)de
j                  deddfdZej                  j                  d        Zej                  j                  d*d       Zej                  j                  d)d       Zej                  j                  de
j                  fd       Zd+dedee   fdZ	 	 	 	 	 d,dej,                  deeeee   f      dededededeeej,                     eej,                  eej,                     f   f   fd Z	 	 	 d-deeee   f   d!ed"efd#Zd$ Zd*d%efd&Zd' Z xZS ).r   Nimg_sizer  num_classes	embed_dim.r   r  r   window_sizer   down_opsglobal_pool	drop_ratec                    t         |           ||d}d| _        || _        || _        || _        t        ||d   fi || _        | j                  j                  }|| j                  j                  z  }t        t        |            D cg c]  }||   ||   ||   z  z   }}g | _        g }|d   }t        t        |||||||
            D ]|  \  }\  }}}}}}}t        d|||||||||	|d
|}|}|d   dk(  r|dk7  r||d   z  }|j                  }|j!                  |       | xj                  t#        ||d|       gz  c_        ~ t%        j&                  | | _        |d	k(  rt+        |d
      | _        n |dk(  sJ t%        j.                         | _        |d   x| _        | _        |dkD  r#t5        | j0                  |fd| j
                  i|nt6        j$                  j/                         | _        | j;                  d       y c c}w )Nr%   Fr   )
r  re   r   r  r   r   r   r   r   r  r  r   zstages.)num_chs	reductionmoduleavgT	pool_typer   r]   rV   needs_reset )r)   r*   grad_checkpointingr  r  r#  r  patch_embedr  r   r   feature_infor   r   r  r   r   dictr+   r   stagesr   r"  r  num_featureshead_hidden_sizerR   r0   headinit_weights)r4   r  r  r  r  r   r  r   r   r   r!  r"  r#  r&   r'   r5   r   r   r   r   r2  pre_edr   kddpthnharwddostager6   s                                 r7   r*   zEfficientVitMsra.__init__  s`   " 	/"'& " *(IaLGBG!!,,!1!1!<!<<
JOPST]P^J_`Qilgaj9Q<&?@`
` 11:Iwy*kS[\2^ 	\-A-Bb"b"% %"$ E F!u#Q"Q%))JMM% $rVgVWUXM"Z![[)	\* mmV,%3kSWXD!###!{{}D4=bMAD1JUXY/ {G15GCEG_d_g_g_p_p_r 		 	e,O as   H
r,  c                 P    | j                  t        | j                  |             y )Nr+  )applyr   _init_weights)r4   r,  s     r7   r6  zEfficientVitMsra.init_weights  s    

74--;GHr8   rG   r   c                 D    |rt        |d      r|j                          y y y )Nr   )hasattrr   )r4   rG   r,  s      r7   rA  zEfficientVitMsra._init_weights"  s"    71&89  :;r8   c                 n    | j                         j                         D ch c]	  }d|v s| c}S c c}w )Nr   )
state_dictkeysrr   s     r7   no_weight_decayz EfficientVitMsra.no_weight_decay&  s.    ??,113Oa7IQ7NOOOs   	22c                 ,    t        d|rdnddg      }|S )Nz^patch_embedz^stages\.(\d+))z^stages\.(\d+).downsample)r   )z^stages\.(\d+)\.\w+\.(\d+)N)stemr  )r1  )r4   coarsematchers      r7   group_matcherzEfficientVitMsra.group_matcher*  s'     (.$455
 r8   c                     || _         y rq   )r.  )r4   enables     r7   set_grad_checkpointingz'EfficientVitMsra.set_grad_checkpointing5  s
    "(r8   c                 .    | j                   j                  S rq   )r5  r[   r   s    r7   get_classifierzEfficientVitMsra.get_classifier9  s    yyr8   c                 &   || _         |8|dk(  rt        |d      | _        n |dk(  sJ t        j                         | _        |dkD  r(t        | j                  || j                        | _	        y t        j                  j	                         | _	        y )Nr(  Tr)  r   )rV   )
r  r   r"  r+   r  rR   r3  r#  r0   r5  )r4   r  r"  s      r7   reset_classifierz!EfficientVitMsra.reset_classifier=  s    &"e##7+W[#\ "a'''#%;;= DORSO {A	Y^YaYaYjYjYl 		r8   rs   indicesnorm
stop_early
output_fmtintermediates_onlyc                    |dv sJ d       g }t        t        | j                        |      \  }}	| j                  |      }t        j
                  j                         s|s| j                  }
n| j                  d|	dz    }
t        |
      D ]Z  \  }}| j                  r+t        j
                  j                         st        ||      }n ||      }||v sJ|j                  |       \ |r|S ||fS )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWzOutput shape must be NCHW.Nr   )r   r   r2  r/  r0   r   is_scriptingr   r.  r   r   )r4   rs   rT  rU  rV  rW  rX  intermediatestake_indices	max_indexr2  feat_idxr>  s                r7   forward_intermediatesz&EfficientVitMsra.forward_intermediatesH  s    * Y&D(DD&"6s4;;7G"Qi Q99!!#:[[F[[)a-0F(0 	(OHe&&uyy/E/E/Gua(!H<'$$Q'	(   -r8   
prune_norm
prune_headc                     t        t        | j                        |      \  }}| j                  d|dz    | _        |r| j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r  )r   r   r2  rS  )r4   rT  ra  rb  r]  r^  s         r7   prune_intermediate_layersz*EfficientVitMsra.prune_intermediate_layersv  sM     #7s4;;7G"Qikk.9q=1!!!R(r8   c                     | j                  |      }| j                  r6t        j                  j	                         st        | j                  |      }|S | j                  |      }|S rq   )r/  r.  r0   r   r[  r   r2  rr   s     r7   forward_featuresz!EfficientVitMsra.forward_features  sU    Q""599+A+A+Ct{{A.A  AAr8   
pre_logitsc                 N    | j                  |      }|r|S | j                  |      S rq   )r"  r5  )r4   rs   rg  s      r7   forward_headzEfficientVitMsra.forward_head  s'    Qq0DIIaL0r8   c                 J    | j                  |      }| j                  |      }|S rq   )rf  ri  rr   s     r7   rt   zEfficientVitMsra.forward  s'    !!!$a r8   )   rh     @         )r  r  r  r   r;   rh   rg   rg   rg   r   r   r   r   )r  r  r;   rt  r(  r`   NNr   Frq   )NFFrZ  F)r   FT)rI   rJ   rK   rL   r	   r   rM   r*   ra   r6  r+   r   rA  r0   r   ignorerG  rL  rO  rQ  r   rS  r   r   r   r`  rd  rf  ri  rt   rO   rP   s   @r7   r   r     s     #)7'3%.)2+4'34a$!C-C- C- 	C-
 S#XC- 38_C- c?C- S#XC- sCxC- 38_C- E#s(OS01C- C- C-JI I!ryy !t !t ! YYP P YY  YY) ) YY 		    	mC 	mhsm 	m 8<$$',, ||,  eCcN34,  	, 
 ,  ,  !%,  
tELL!5tELL7I)I#JJ	K, ` ./$#	3S	>*  	1$ 1r8   c           
      .    | dt         t        dddddd	|S )Nrl  zpatch_embed.conv1.convzhead.linearT)rg   rg   mit)	urlr  meanrU   
first_conv
classifierfixed_input_size	pool_sizelicenser   )ry  kwargss     r7   _cfgr    s1    %#.#   r8   ztimm/)	hf_hub_id)zefficientvit_m0.r224_in1kzefficientvit_m1.r224_in1kzefficientvit_m2.r224_in1kzefficientvit_m3.r224_in1kzefficientvit_m4.r224_in1kzefficientvit_m5.r224_in1kc                 h    |j                  dd      }t        t        | |fdt        d|      i|}|S )Nout_indices)r   r   r;   feature_cfgT)flatten_sequentialr  )popr   r   r1  )variant
pretrainedr  r  models        r7   _create_efficientvit_msrar    sG    **]I6K  DkJ	
 E Lr8   c           	      f    t        dg dg dg dg dg d      }t        d	d| it        |fi |S )
Nrk  rm  rq  rr  rs  r   r  r  r  r   r   r   r  )efficientvit_m0r1  r  r  r  
model_argss      r7   r  r    s@     J %l:lQUV`QkdjQkllr8   c           	      f    t        dg dg dg dg dg d      }t        d	d| it        |fi |S )
Nrk  )ro     rp  rq  )r;   rh   rh   rs  r   r   rh   rh   r  r  )efficientvit_m1r  r  s      r7   r  r    @    !J %l:lQUV`QkdjQkllr8   c           	      f    t        dg dg dg dg dg d      }t        d	d| it        |fi |S )
Nrk  )ro  rp  rk  rq  )rg   rh   r;   rs  r  r  r  )efficientvit_m2r  r  s      r7   r  r  	  r  r8   c           	      f    t        dg dg dg dg dg d      }t        d	d| it        |fi |S )
Nrk  )ro     i@  rq  )rg   rh   rg   rs  r   r  r  )efficientvit_m3r  r  s      r7   r  r    r  r8   c           	      f    t        dg dg dg dg dg d      }t        d	d| it        |fi |S )
Nrk  )ro       rq  rr  rs  r  r  r  )efficientvit_m4r  r  s      r7   r  r  #  r  r8   c           	      f    t        dg dg dg dg dg d      }t        d	d| it        |fi |S )
Nrk  )rp  i   r  )r   rh   rg   )rh   rh   rg   rs  r  r  r  )efficientvit_m5r  r  s      r7   r  r  0  r  r8   )r  ru  )9r   __all__r   collectionsr   	functoolsr   typingr   r   r   r	   r
   r   r0   torch.nnr+   	timm.datar   r   timm.layersr   r   r   r   _builderr   	_featuresr   _manipulater   r   	_registryr   r   r   r   rR   r   rc   rx   r   r   r   r   r  r  r   r  default_cfgsr  r  r  r  r  r  r  r-  r8   r7   <module>r     s   
  #  ; ;   A S S * + 3 < uxx""  F#$$ #L588?? ,!588?? !ehhoo &uUXX__ upB588?? BJ-G -G`; ;|UXX(( (zryy z~ %!%" "&" "&" "&" "&" "&"+& 8	 	m 	m 	m 	m 	m 	m 	m 	m 	m 	m 	m 	mr8   