
    ^j                     6   d Z ddgZddlmZmZmZmZmZ ddlm	Z	 ddl
Z
ddlmZ ddlmc mZ ddlmZmZ ddlmZmZmZ d	d
lmZ d	dlmZ d	dlmZ d	dlmZ d	dlm Z m!Z! ddde"xs e#xs e$fdZ%dede"xs e#xs e$de&de&fdZ'de&xs e#e&df   de&xs e#e&df   fdZ( G d dejR                        Z* G d dejR                        Z+ G d dejR                        Z, G d dejR                        Z- G d  d!ejR                        Z. G d" d#ejR                        Z/ ee/        G d$ d%ejR                        Z0 G d& d'ejR                        Z1	 	 	 	 dfd(e&d)e&d*e&d+e2d,e3d-e3d.e4d/e3fd0Z5 G d1 d2ejl                        Z7 G d3 d4ejR                        Z8 G d5 d6ejR                        Z9 G d7 d8ejR                        Z: G d9 dejR                        Z; G d: dejR                        Z<dgd;Z= e!i d< e=d=>      d? e=d=>      d@ e=d=dAdBdCD      dE e=d=dFdGdCD      dH e=d=>      dI e=d=dAdBdCD      dJ e=d=dFdGdCD      dK e=d=>      dL e=d=dAdBdCD      dM e=d=dFdGdCD      dN e=d=dCO      dP e=d=dCO      dQ e=d=dAdBdCD      dR e=d=dFdGdCD      dS e=d=dTdUdCD      dV e=d=dCO      dW e=d=dAdBdCD       e=d=dXdYdCD       e=d=dTdUdCD      dZ      Z>dhd[Z?dhd\Z@e dhd]       ZAe dhd^       ZBe dhd_       ZCe dhd`       ZDe dhda       ZEe dhdb       ZFe dhdc       ZGy)ia   EfficientViT (by MIT Song Han's Lab)

Paper: `Efficientvit: Enhanced linear attention for high-resolution low-computation visual recognition`
    - https://arxiv.org/abs/2205.14756

Adapted from official impl at https://github.com/mit-han-lab/efficientvit
EfficientVitEfficientVitLarge    )ListOptionalTupleTypeUnion)partialNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)SelectAdaptivePool2dcreate_conv2dGELUTanh   )build_model_with_cfg)feature_take_indices)register_notrace_module)checkpoint_seq)register_modelgenerate_default_cfgsxc                     t        | t        t        f      rt        |       S t        |      D cg c]  }|  c}S c c}w N)
isinstancelisttuplerange)r   repeat_time_s      g/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/efficientvit_mit.pyval2listr"      s2    !dE]#Aw[)*!A***s   	;min_len
idx_repeatc                     t        |       } t        |       dkD  r+t        |t        |       z
        D cg c]  }| |   	 c}| || t        |       S c c}w )Nr   )r"   lenr   r   )r   r#   r$   r    s       r!   	val2tupler'       sN    A
1vz;@3q6AQ;R#SaAjM#S*Z 8O $Ts   Akernel_size.returnc                     t        | t              r"t        | D cg c]  }t        |       c}      S | dz  dkD  sJ d       | dz  S c c}w )N   r   z kernel size should be odd number)r   r   get_same_padding)r(   kss     r!   r,   r,   )   sQ    +u%[Ar&r*ABBQ"F$FF"a Bs   Ac                        e Zd Zddddddej                  ej
                  ddf
dededeeeeef   f   d	ed
edede	de
deeej                        deeej                        f fdZd Z xZS )ConvNormAct   r   F        Nin_channelsout_channelsr(   stridedilationgroupsbiasdropout
norm_layer	act_layerc           	      2   ||d}t         |           t        j                  |d      | _        t        ||f|||||d|| _        |	r
 |	dd|i|nt        j                         | _        |
 |
d      | _	        y t        j                         | _	        y )NdevicedtypeFinplace)r(   r4   r5   r6   r7   num_featuresT )
super__init__nnDropoutr8   r   convIdentitynormact)selfr2   r3   r(   r4   r5   r6   r7   r8   r9   r:   r=   r>   dd	__class__s                 r!   rD   zConvNormAct.__init__2   s     /zz'59!	
 $	
 	
	 DNJ?L?B?SUS^S^S`	.7.C9T*    c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }|S r   )r8   rG   rI   rJ   rK   r   s     r!   forwardzConvNormAct.forwardQ   s;    LLOIIaLIIaLHHQKrN   )__name__
__module____qualname__rE   BatchNorm2dReLUintr	   r   boolfloatr   r   ModulerD   rQ   __classcell__rM   s   @r!   r/   r/   1   s    
 8946NN3577WW W sE#s(O34	W
 W W W W W !bii1W  RYY0W>rN   r/   c                   .    e Zd Zdddej                  ej
                  dfddfdedededed	eee	eef   f   d
ee
ej                     e	ee
ej                        df   f   dee
ej                     e	ee
ej                        df   f   f fdZd Z xZS )DSConvr0   r   FNr2   r3   r(   r4   use_biasr9   .r:   c
                     ||	d}
t         |           t        |d      }t        |d      }t        |d      }t        ||||f||d   |d   |d   d|
| _        t        ||df|d   |d   |d   d|
| _        y )Nr<   r+   r   )r6   r9   r:   r7   r   r9   r:   r7   )rC   rD   r'   r/   
depth_conv
point_conv)rK   r2   r3   r(   r4   r_   r9   r:   r=   r>   rL   rM   s              r!   rD   zDSConv.__init__Z   s     /Xq)z1-
i+	%	


 !!}l!

 

 &
 "!}l!
 
rN   c                 J    | j                  |      }| j                  |      }|S r   )rb   rc   rP   s     r!   rQ   zDSConv.forward   s#    OOAOOArN   )rR   rS   rT   rE   rU   ReLU6rW   r	   rX   r   r   rZ   r   rD   rQ   r[   r\   s   @r!   r^   r^   Y   s    
  !7<XZXfXfXZX`X`bfWg%
%
 %
 	%

 %
 D%d
"334%
 d299ouXd299o5NPS5S/TTU%
 T"))_eHT"))_4Ms4R.SST%
NrN   r^   c                   @    e Zd Zdddddej                  ej
                  dfddf	dedededed	ee   d
ede	e
ee
e
f   f   de	eej                     eeeej                        df   f   de	eej                     eeeej                        df   f   f fdZd Z xZS )	ConvBlockr0   r   NFr2   r3   r(   r4   mid_channelsexpand_ratior_   r9   .r:   c                    |
|d}t         |           t        |d      }t        |d      }t        |	d      }	|xs t        ||z        }t	        ||||f|d   |	d   |d   d|| _        t	        |||df|d   |	d   |d   d|| _        y )Nr<   r+   r   ra   r   )rC   rD   r'   roundr/   conv1conv2rK   r2   r3   r(   r4   rh   ri   r_   r9   r:   r=   r>   rL   rM   s                r!   rD   zConvBlock.__init__   s     /Xq)z1-
i+	#Hu[<-G'H 		

 "!}l!	
 	

 !		

 "!}l!	
 	

rN   c                 J    | j                  |      }| j                  |      }|S r   )rl   rm   rP   s     r!   rQ   zConvBlock.forward   s!    JJqMJJqMrN   rR   rS   rT   rE   rU   re   rW   r   rY   r	   rX   r   r   rZ   rD   rQ   r[   r\   s   @r!   rg   rg      s    
  !*."#7<XZXfXfXZX`X`bfWg(
(
 (
 	(

 (
 #3-(
  (
 D%d
"334(
 d299ouXd299o5NPS5S/TTU(
 T"))_eHT"))_4Ms4R.SST(
TrN   rg   c                   V    e Zd Zdddddej                  ej
                  ej
                  dfddf	dededed	ed
ee   dede	e
ee
df   f   de	eej                     eeeej                        df   f   de	eej                     eeeej                        df   f   f fdZd Z xZS )MBConvr0   r   N   Fr2   r3   r(   r4   rh   ri   r_   .r9   r:   c           	      ^   |
|d}t         |           t        |d      }t        |d      }t        |	d      }	|xs t        ||z        }t	        ||dfd|d   |	d   |d   d|| _        t	        |||f|||d   |	d   |d   d|| _        t	        ||df|d   |	d   |d   d|| _        y )	Nr<   r0   r   r   )r4   r9   r:   r7   r4   r6   r9   r:   r7   r+   ra   )rC   rD   r'   rk   r/   inverted_convrb   rc   rn   s                r!   rD   zMBConv.__init__   s    /Xq)z1-
i+	#Hu[<-G'H(	
 !!}l!	
 	
 &

 !!}l!

 

 &
 "!}l!
 
rN   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )rv   rb   rc   rP   s     r!   rQ   zMBConv.forward   s3    q!OOAOOArN   rp   r\   s   @r!   rr   rr      s   
  !*."#6;XZXfXfXZX`X`bdbjbjlpWq2
2
 2
 	2

 2
 #3-2
  2
 D%c	"2232
 d299ouXd299o5NPS5S/TTU2
 T"))_eHT"))_4Ms4R.SST2
hrN   rr   c                   F    e Zd Zddddddej                  ej
                  dfddf
dededed	ed
ee   dedede	e
ee
df   f   de	eej                     eeeej                        df   f   de	eej                     eeeej                        df   f   f fdZd Z xZS )FusedMBConvr0   r   Nrs   Fr2   r3   r(   r4   rh   ri   r6   r_   .r9   r:   c           	         ||d}t         |           t        |d      }t        |	d      }	t        |
d      }
|xs t        ||z        }t	        |||f|||	d   |
d   |d   d|| _        t	        ||df|	d   |
d   |d   d|| _        y )Nr<   r+   r   ru   r   ra   )rC   rD   r'   rk   r/   spatial_convrc   )rK   r2   r3   r(   r4   rh   ri   r6   r_   r9   r:   r=   r>   rL   rM   s                 r!   rD   zFusedMBConv.__init__   s     /Xq)z1-
i+	#Hu[<-G'H'

 !!}l!

 

 &
 "!}l!
 
rN   c                 J    | j                  |      }| j                  |      }|S r   )r{   rc   rP   s     r!   rQ   zFusedMBConv.forward   s%    a OOArN   rp   r\   s   @r!   ry   ry      s   
  !*."#6;XZXfXfXZX`X`bfWg)
)
 )
 	)

 )
 #3-)
  )
 )
 D%c	"223)
 d299ouXd299o5NPS5S/TTU)
 T"))_eHT"))_4Ms4R.SST)
VrN   ry   c                   z    e Zd ZdZdddddej
                  fdej                  ddddfd	ed
edee   de	dede
eeedf   f   de
eej                     eeeej                        df   f   de
eej                     eeeej                        df   f   deej                     deedf   de	f fdZd Zd Z xZS )LiteMLAz(Lightweight multi-scale linear attentionN      ?   FNN)   h㈵>r2   r3   headsheads_ratiodimr_   .r9   r:   kernel_funcscalesepsc                    ||d}t         |           || _        |xs t        ||z  |z        }||z  }t	        |d      }t	        |d      }t	        |d      }|| _        t        |d|z  df|d   |d   |d   d|| _        t        j                  |
D cg c]l  }t        j                  t        j                  d|z  d|z  |ft        |      d|z  |d   d|t        j                  d|z  d|z  dfd|z  |d   d|      n c}      | _         |	d	
      | _        t        |dt        |
      z   z  |df|d   |d   |d   d|| _        y c c}w )Nr<   r+   r0   r   r   )r7   r9   r:   )paddingr6   r7   )r6   r7   Fr?   )rC   rD   r   rW   r'   r   r/   qkvrE   
ModuleList
SequentialConv2dr,   aggregr   r&   proj)rK   r2   r3   r   r   r   r_   r9   r:   r   r   r   r=   r>   rL   	total_dimscalerM   s                    r!   rD   zLiteMLA.__init__)  s     />[C/+=>CK	Xq)z1-
i+		M
 !!!}l
 
 mm  %
  MM			M	M -U3y=!!  		!i-Yd!e)RZ[\R]dacd%
  'u5S[)
 !!!}l
 
	#%
s   A1Ec                    |j                   }|j                         |j                         |j                         }}}|j                  dd      |z  }||z  }|dd df   |ddd f   | j                  z   z  }|j	                  |      S )N.)r>   rY   	transposer   to)rK   qkvr>   kvouts          r!   _attnzLiteMLA._attng  s    '')QWWY	a1[[R 1$"f#ss(ms38}txx78vve}rN   c                 N   |j                   \  }}}}| j                  |      }|g}| j                  D ]  }|j                   ||              t	        j
                  |d      }|j                  |dd| j                  z  ||z        j                  dd      }|j                  dd      \  }	}
}| j                  |	      }	| j                  |
      }
t        j                  |ddd	      }t        j                  j                         sHt	        j                  |j                   j"                  d
      5  | j%                  |	|
|      }d d d        n| j%                  |	|
|      }j                  dd      j                  |d||      }| j'                  |      }|S # 1 sw Y   @xY w)Nr   )r   r   r0   r   )r   r   constantr   )modevalueF)device_typeenabled)shaper   r   appendtorchcatreshaper   r   chunkr   Fpadjitis_scriptingautocastr=   typer   r   )rK   r   Br    HWr   multi_scale_qkvopr   r   r   r   s                r!   rQ   zLiteMLA.forwardo  s{   WW
1a hhqk%++ 	,B""2c7+	,))O;)11!RTXXq1uMWWXZ\^_!''r'21a QQEE!V*B7yy%%'AHHMM5I *jjAq)* * **Q1%C mmB#++Ar1a8iin
* *s   4FF$)rR   rS   rT   __doc__rE   rU   rV   rW   r   rY   r	   rX   r   r   rZ   rD   r   rQ   r[   r\   s   @r!   r~   r~   &  s/   2 $(!$6;Y]_a_m_mXnWc+-77&*<
<
 <
 C=	<

 <
 <
 D%c	"223<
 d299ouXd299o5NPS5S/TTU<
 T"))_eHT"))_4Ms4R.SST<
 bii<
 #s(O<
 <
|rN   r~   c                        e Zd Zdddej                  ej
                  ddfdedededed	eej                     d
eej                     f fdZ
d Z xZS )EfficientVitBlockr          Nr2   r   head_dimri   r9   r:   c	                    ||d}	t         
|           t        t        d||||d |fd|	t	        j
                               | _        t        t        d|||dd d |f||d fd|	t	        j
                               | _        y )Nr<   )r2   r3   r   r   r9   TTF)r2   r3   ri   r_   r9   r:   rB   )	rC   rD   ResidualBlockr~   rE   rH   context_modulerr   local_module)rK   r2   r   r   ri   r9   r:   r=   r>   rL   rM   s             r!   rD   zEfficientVitBlock.__init__  s     /+ '(' *-  KKM

 * '(), $
3$i6  KKM
rN   c                 J    | j                  |      }| j                  |      }|S r   )r   r   rP   s     r!   rQ   zEfficientVitBlock.forward  s'    "a rN   )rR   rS   rT   rE   rU   	HardswishrW   rY   r   rZ   rD   rQ   r[   r\   s   @r!   r   r     sw     "%"#*,..)+#
#
 #
 	#

  #
 RYY#
 BII#
JrN   r   c                        e Zd Z	 	 ddeej
                     deej
                     deej
                     f fdZd Z xZS )r   mainshortcutpre_normc                 x    t         |           ||nt        j                         | _        || _        || _        y r   )rC   rD   rE   rH   r   r   r   )rK   r   r   r   rM   s       r!   rD   zResidualBlock.__init__  s2     	$,$8bkkm	 rN   c                     | j                  | j                  |            }| j                  || j                  |      z   }|S r   )r   r   r   )rK   r   ress      r!   rQ   zResidualBlock.forward  s;    iia()==$a((C
rN   r   )	rR   rS   rT   r   rE   rZ   rD   rQ   r[   r\   s   @r!   r   r     sL     -1,0		!299%	! ryy)	! ryy)		!rN   r   r2   r3   r4   ri   r9   r:   
fewer_norm
block_typec
                 >   ||	d}
|dv sJ |dk(  rE|dk(  r t        d
| |||rdnd|rd |fn||d fd|
}|S t        d
| |||rdnd|rd |fn||d fd|
}|S |dk(  r#t        d
| ||||rdnd|rd d |fn|||d fd	|
}|S t        d
| ||||rdnd|rd |fn||d fd	|
}|S )Nr<   )defaultlargefusedr   r   )TFF)r2   r3   r4   r_   r9   r:   r   )r2   r3   r4   ri   r_   r9   r:   rB   )r^   rg   rr   ry   )r2   r3   r4   ri   r9   r:   r   r   r=   r>   rL   blocks               r!   build_local_blockr     sY    U	+B6666q" ')*4%1;D*-$d+ ET LA  ')*4%1;D*-$d+ E@ L- " 	'))0:,7AD$
3z$i6	 	E* L   	'))*4%1;D*-$d+	 	E LrN   c                   t     e Zd Z	 	 	 ddedededeej                     deej                     def fdZ xZ	S )	Stemin_chsout_chsdepthr9   r:   r   c	                 0   t         |           ||d}	d| _        | j                  dt	        ||fdd||d|	       d}
t        |      D ]I  }| j                  d|
 t        t        d
||dd|||d	|	t        j                                      |
dz  }
K y )Nr<   r+   in_convr0   )r(   r4   r9   r:   r   r   r   )r2   r3   r4   ri   r9   r:   r   rB   )
rC   rD   r4   
add_moduler/   r   r   r   rE   rH   )rK   r   r   r   r9   r:   r   r=   r>   rL   
stem_blockr    rM   s               r!   rD   zStem.__init__  s     	/ %# 	
 
u 	AOOc*.! 	 '!(!")')	 	 1  !OJ	rN   )r   NN)
rR   rS   rT   rW   r   rE   rZ   strrD   r[   r\   s   @r!   r   r     sb     (** * 	*
 RYY* BII* * *rN   r   c                        e Zd Z	 	 	 ddedededeej                     deej                     dededef fd	Z	d
 Z
 xZS )EfficientVitStager   r   r   r9   r:   ri   r   	vit_stagec                    |	|
d}t         |           t        t        d||d||||d|d       g}|}|r1t	        |      D ]"  }|j                  t        d|||||d|       $ nNt	        d|      D ]?  }|j                  t        t        d||d|||d|t        j                                      A t        j                  | | _
        y )Nr<   r+   )r2   r3   r4   ri   r9   r:   r   r2   r   ri   r9   r:   r   )r2   r3   r4   ri   r9   r:   rB   rC   rD   r   r   r   r   r   rE   rH   r   blocks)rK   r   r   r   r9   r:   ri   r   r   r=   r>   rL   r   r    irM   s                  r!   rD   zEfficientVitStage.__init__:  s    / 	"$)%#$	 	 
  5\ 
% $*!)%1#-"+ 	
 1e_ m% $*%, %1#-"+  KKM  mmV,rN   c                 $    | j                  |      S r   r   rP   s     r!   rQ   zEfficientVitStage.forwardw      {{1~rN   )FNN)rR   rS   rT   rW   r   rE   rZ   rY   rX   rD   rQ   r[   r\   s   @r!   r   r   9  sw     $;-;- ;- 	;-
 RYY;- BII;-  ;- ;- ;-zrN   r   c                        e Zd Z	 	 	 	 ddedededeej                     deej                     dededef fd	Zd
 Z	 xZ
S )EfficientVitLargeStager   r   r   r9   r:   r   r   r   c                    |	|
d}t         |           t        t        d||d|rdnd|||xs ||rdndd|d       g}|}|r1t	        |      D ]"  }|j                  t        d||d||d	|       $ nSt	        |      D ]E  }|j                  t        t        d||d
d||||rdndd|t        j                                      G t        j                  | | _
        y )Nr<   r+         r   r   )r2   r3   r4   ri   r9   r:   r   r   rs   r   r   r   rB   r   )rK   r   r   r   r9   r:   r   r   r   r=   r>   rL   r   r    r   rM   s                  r!   rD   zEfficientVitLargeStage.__init__|  s1    / 
"$#,R"%#$2
(29
 
 
  5\ 
% $*!)%&#-"+ 	
 5\ m% 
$*%, %&#-"+#-0:9
 
 KKM   mmV,rN   c                 $    | j                  |      S r   r   rP   s     r!   rQ   zEfficientVitLargeStage.forward  r   rN   )FFNN)rR   rS   rT   rW   r   rE   rZ   rX   rD   rQ   r[   r\   s   @r!   r   r   {  sz     $$>->- >- 	>-
 RYY>- BII>- >- >- >-@rN   r   c                        e Zd Zddej                  ej
                  ddddfdedee   ded	ed
e	ej                     dee	ej                        dedef fdZddedee   fdZddefdZ xZS )ClassifierHead  r1   avgr   Nr2   widthsnum_classesr8   r9   r:   	pool_typenorm_epsc                 <   |	|
d}t         |           || _        |d   | _        |sJ d       t	        ||d   df||d|| _        t        |d      | _        t        j                  t        j                  |d   |d   fd	d
i|t        j                  |d   fd|i||	 |d      nt        j                         t        j                  |d
      |dkD  r&t        j                  |d   |fd	di|      | _        y t        j                               | _        y )Nr<   r   Cannot disable poolingr   r   )r9   r:   Tr   flattenr7   Fr   r?   )rC   rD   r   rA   r/   r   r   global_poolrE   r   Linear	LayerNormrH   rF   
classifier)rK   r2   r   r   r8   r9   r:   r   r   r=   r>   rL   rM   s               r!   rD   zClassifierHead.__init__  s    /"2J222y";q	1o_holno/)TR--IIfQi=="=LL77B7'0'<Id#"++-JJw.BMPQ/BIIfQi>4>2>

 XZWbWbWd
rN   c                     ||sJ d       t        |d      | _        |dkD  r0t        j                  | j                  |d      | j
                  d<   y t        j                         | j
                  d<   y )Nr   Tr   r   )r7   r   )r   r   rE   r   rA   r   rH   )rK   r   r   s      r!   resetzClassifierHead.reset  sc     66693iQUWD?"$))D,=,={QU"VDOOB"$++-DOOBrN   
pre_logitsc                    | j                  |      }| j                  |      }|rV | j                  d   |      } | j                  d   |      } | j                  d   |      } | j                  d   |      }|S | j                  |      }|S )Nr   r   r+   r0   )r   r   r   rK   r   r   s      r!   rQ   zClassifierHead.forward  s    LLOQ""1%A""1%A""1%A""1%A  "ArN   r   F)rR   rS   rT   rE   rU   r   rW   r   rY   r   rZ   r   r   rD   r   rX   rQ   r[   r\   s   @r!   r   r     s    
  $*,..35<<""

 I
 	

 
 RYY
  RYY0
 
 
:0 0# 0T rN   r   c                       e Zd Zdddddej                  ej
                  ddddddfd	ed
eedf   deedf   dedede	ej                     de	ej                     dedeedf   dedef fdZej                  j                  d)d       Zej                  j                  d*d       Zej                  j                  dej                  fd       Zd+dedee   fdZ	 	 	 	 	 d,dej*                  deeeee   f      dededed edeeej*                     eej*                  eej*                     f   f   fd!Z	 	 	 d-deeee   f   d"ed#efd$Zd% Zd)d&efd'Zd( Z xZS ).r   r0   rB   r   r   r   r1   r   Nin_chansr   .depthsr   ri   r9   r:   r   head_widths	drop_rater   c                    ||d}t         |           d| _        || _        || _        || _        t        ||d   |d   ||fi || _        | j                  j                  }g | _	        t        j                         | _        |d   }t        t        |dd  |dd              D ]d  \  }\  }}| j                  j                  t!        ||f||||||dk\  d|       |dz  }|}| xj                  t#        ||d|       gz  c_	        f || _        t'        | j$                  f|	||
| j                  d	|| _        | j(                  j$                  | _        y )
Nr<   Fr   r   r+   )r   r9   r:   ri   r   r   stages.num_chs	reductionmodule)r   r   r8   r   )rC   rD   grad_checkpointingr   r   r  r   stemr4   feature_inforE   r   stages	enumeratezipr   r   dictrA   r   headhead_hidden_size)rK   r  r   r  r   ri   r9   r:   r   r  r  r   r=   r>   rL   r4   r2   r   wdrM   s                       r!   rD   zEfficientVit.__init__  s     /"'&&  6!9fQiYURTU	!! mmoQi"3vabz6!":#>? 	eIAv1KK0
  %#)!q&
  
  
 aKFK${fW^_`^aUb"c!dd	e  ("
#&&
 
	 !%		 6 6rN   c                 ,    t        d|rdnddg      }|S Nz^stemz^stages\.(\d+))z^stages\.(\d+).downsample)r   )z^stages\.(\d+)\.\w+\.(\d+)N)r  r   r  rK   coarsematchers      r!   group_matcherzEfficientVit.group_matcher0  '    (.$455
 rN   c                     || _         y r   r  rK   enables     r!   set_grad_checkpointingz#EfficientVit.set_grad_checkpointing;  
    "(rN   r)   c                 4    | j                   j                  d   S Nr   r  r   rK   s    r!   get_classifierzEfficientVit.get_classifier?      yy##B''rN   c                 J    || _         | j                  j                  ||       y r   r   r  r   rK   r   r   s      r!   reset_classifierzEfficientVit.reset_classifierC      &		[1rN   r   indicesrI   
stop_early
output_fmtintermediates_onlyc                    |dv sJ d       g }t        t        | j                        |      \  }}	| j                  |      }t        j
                  j                         s|s| j                  }
n| j                  d|	dz    }
t        |
      D ]Z  \  }}| j                  r+t        j
                  j                         st        |
|      }n ||      }||v sJ|j                  |       \ |r|S ||fS a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWzOutput shape must be NCHW.Nr   r   r&   r  r  r   r   r   r  r  r   r   rK   r   r2  rI   r3  r4  r5  intermediatestake_indices	max_indexr  feat_idxstages                r!   forward_intermediatesz"EfficientVit.forward_intermediatesG      * Y&D(DD&"6s4;;7G"Qi IIaL99!!#:[[F[[)a-0F(0 	(OHe&&uyy/E/E/G"61-!H<'$$Q'	(   -rN   
prune_norm
prune_headc                     t        t        | j                        |      \  }}| j                  d|dz    | _        |r| j                  dd       |S z@ Prune layers not required for specified intermediates.
        Nr   r    r   r&   r  r0  rK   r2  rB  rC  r<  r=  s         r!   prune_intermediate_layersz&EfficientVit.prune_intermediate_layersu  M     #7s4;;7G"Qikk.9q=1!!!R(rN   c                     | j                  |      }| j                  r6t        j                  j	                         st        | j                  |      }|S | j                  |      }|S r   r  r  r   r   r   r   r  rP   s     r!   forward_featureszEfficientVit.forward_features  R    IIaL""599+A+A+Ct{{A.A  AArN   r   c                 N    |r| j                  ||      S | j                  |      S N)r   r  r  s      r!   forward_headzEfficientVit.forward_head  %    6@tyyzy2RdiiPQlRrN   c                 J    | j                  |      }| j                  |      }|S r   rM  rR  rP   s     r!   rQ   zEfficientVit.forward  '    !!!$a rN   r  Tr   NFFr8  Fr   FT)rR   rS   rT   rE   rU   r   rW   r   rY   r   rZ   r   rD   r   r   ignorer  r%  r+  r   r0  Tensorr	   r   rX   r@  rI  rM  rR  rQ   r[   r\   s   @r!   r   r     s6    &(&("#*,..)+$+-"#8787 #s(O87 #s(O	87
 87  87 RYY87 BII87 87 sCx87 87 87t YY  YY) ) YY(		 ( (2C 2hsm 2 8<$$',, ||,  eCcN34,  	, 
 ,  ,  !%,  
tELL!5tELL7I)I#JJ	K, ` ./$#	3S	>*  	S$ SrN   c                       e Zd Zddddej                  edddddddfd	ed
eedf   deedf   dedeej                     deej                     de
deedf   dededef fdZej                  j                  d)d       Zej                  j                  d*d       Zej                  j                  dej                  fd       Zd+dedee
   fdZ	 	 	 	 	 d,dej*                  deeeee   f      dedede
d edeeej*                     eej*                  eej*                     f   f   fd!Z	 	 	 d-deeee   f   d"ed#efd$Zd% Zd)d&efd'Zd( Z xZS ).r   r0   rB   r   r   r1   r   gHz>Nr  r   .r  r   r9   r:   r   r  r  r   r   c                 
   ||d}t         |           d| _        || _        |
| _        || _        || _        t        || j                        }t        ||d   |d   ||fddi|| _	        | j                  j                  }g | _        t        j                         | _        |d   }t        t!        |dd  |dd              D ]g  \  }\  }}| j                  j#                  t%        ||f|||||dk\  |d	k\  d
|       |d	z  }|}| xj                  t'        ||d|       gz  c_        i || _        t+        | j(                  f||
|	| j                  || j                  d|| _        | j,                  j(                  | _        y )Nr<   F)r   r   r   r   r   r0   r+   )r   r9   r:   r   r   r   r	  r
  )r   r   r8   r   r:   r   )rC   rD   r  r   r   r  r   r
   r   r  r4   r  rE   r   r  r  r  r   r   r  rA   r   r  r  )rK   r  r   r  r   r9   r:   r   r  r  r   r   r=   r>   rL   r4   r2   r   r  r  rM   s                       r!   rD   zEfficientVitLarge.__init__  s     /"'&&  ZT]];
 6!9fQiYi[bifhi	!! mmoQi"3vabz6!":#>? 	eIAv1KK5
  %#!q&6
  
  
 aKFK${fW^_`^aUb"c!dd	e  ("	
#&&]]	
 	
	 !%		 6 6rN   c                 ,    t        d|rdnddg      }|S r  r  r  s      r!   r  zEfficientVitLarge.group_matcher  r   rN   c                     || _         y r   r"  r#  s     r!   r%  z(EfficientVitLarge.set_grad_checkpointing  r&  rN   r)   c                 4    | j                   j                  d   S r(  r)  r*  s    r!   r+  z EfficientVitLarge.get_classifier  r,  rN   c                 J    || _         | j                  j                  ||       y r   r.  r/  s      r!   r0  z"EfficientVitLarge.reset_classifier  r1  rN   r   r2  rI   r3  r4  r5  c                    |dv sJ d       g }t        t        | j                        |      \  }}	| j                  |      }t        j
                  j                         s|s| j                  }
n| j                  d|	dz    }
t        |
      D ]Z  \  }}| j                  r+t        j
                  j                         st        |
|      }n ||      }||v sJ|j                  |       \ |r|S ||fS r7  r9  r:  s                r!   r@  z'EfficientVitLarge.forward_intermediates  rA  rN   rB  rC  c                     t        t        | j                        |      \  }}| j                  d|dz    | _        |r| j                  dd       |S rE  rG  rH  s         r!   rI  z+EfficientVitLarge.prune_intermediate_layers  rJ  rN   c                     | j                  |      }| j                  r6t        j                  j	                         st        | j                  |      }|S | j                  |      }|S r   rL  rP   s     r!   rM  z"EfficientVitLarge.forward_features&  rN  rN   r   c                 N    |r| j                  ||      S | j                  |      S rP  rQ  r  s      r!   rR  zEfficientVitLarge.forward_head.  rS  rN   c                 J    | j                  |      }| j                  |      }|S r   rU  rP   s     r!   rQ   zEfficientVitLarge.forward1  rV  rN   r  rW  r   rX  rY  )rR   rS   rT   rE   rU   r   rW   r   r   rZ   r   rY   rD   r   r   rZ  r  r%  r+  r   r0  r[  r	   r   rX   r@  rI  rM  rR  rQ   r[   r\   s   @r!   r   r     s2    "$"$&(nn%- ')<7<7 c3h<7 c3h	<7
 <7 O<7 		?<7 <7 38_<7 <7 <7 <7| YY  YY) ) YY(		 ( (2C 2hsm 2 8<$$',, ||,  eCcN34,  	, 
 ,  ,  !%,  
tELL!5tELL7I)I#JJ	K, ` ./$#	3S	>*  	S$ SrN   c                 0    | dt         t        ddddddd
|S )	Nr   zstem.in_conv.convzhead.classifier.4gffffff?z
apache-2.0)r0      rh  )   ri  )
urlr   meanstd
first_convr   crop_pctlicense
input_size	pool_sizer   )rj  kwargss     r!   _cfgrs  7  s4    %#))#  rN   zefficientvit_b0.r224_in1kztimm/)	hf_hub_idzefficientvit_b1.r224_in1kzefficientvit_b1.r256_in1k)r0      ru  )r   r   r   )rt  rp  rq  rn  zefficientvit_b1.r288_in1k)r0      rv  )	   rw  zefficientvit_b2.r224_in1kzefficientvit_b2.r256_in1kzefficientvit_b2.r288_in1kzefficientvit_b3.r224_in1kzefficientvit_b3.r256_in1kzefficientvit_b3.r288_in1kzefficientvit_l1.r224_in1k)rt  rn  zefficientvit_l2.r224_in1kzefficientvit_l2.r256_in1kzefficientvit_l2.r288_in1kzefficientvit_l2.r384_in1k)r0     rx  )   ry  zefficientvit_l3.r224_in1kzefficientvit_l3.r256_in1k)r0   @  rz  )
   r{  )zefficientvit_l3.r320_in1kzefficientvit_l3.r384_in1kc                 h    |j                  dd      }t        t        | |fdt        d|      i|}|S Nout_indices)r   r   r+   r0   feature_cfgT)flatten_sequentialr~  )popr   r   r  variant
pretrainedrr  r~  models        r!   _create_efficientvitr    sG    **]L9K  DkJ	
 E LrN   c                 h    |j                  dd      }t        t        | |fdt        d|      i|}|S r}  )r  r   r   r  r  s        r!   _create_efficientvit_larger    sG    **]L9K  DkJ	
 E LrN   c           	      N    t        dddd      }t        dd| it        |fi |S )N)r   r   r   @      )r   r+   r+   r+   r+   r   )   i   r   r  r   r  r  )efficientvit_b0r  r  r  rr  
model_argss      r!   r  r    s6    #ObVbdJgjgDQ[Lf_eLfggrN   c           	      N    t        dddd      }t        dd| it        |fi |S )N)r   r   r  r  ru  )r   r+   r0   r0   r   r   )i   i@  r  r  )efficientvit_b1r  r  s      r!   r  r    6    %oXdfJgjgDQ[Lf_eLfggrN   c           	      N    t        dddd      }t        dd| it        |fi |S )N)r   0   `      rx  )r   r0   r   r   rs   r   i 	  i 
  r  r  )efficientvit_b2r  r  s      r!   r  r    r  rN   c           	      N    t        dddd      }t        dd| it        |fi |S )Nr   r  r  ru     )r   r   rs   rs   rw  r   r  r  r  )efficientvit_b3r  r  s      r!   r  r    s6    &YegJgjgDQ[Lf_eLfggrN   c           	      N    t        dddd      }t        dd| it        |fi |S )Nr  )r   r   r   rs   rs   r   i   i  r  r  )efficientvit_l1r  r  r  s      r!   r  r    7    &YegJ%mJmRVWaRlekRlmmrN   c           	      N    t        dddd      }t        dd| it        |fi |S )Nr  r   r+   r+   r   r   r   r  r  r  )efficientvit_l2r  r  s      r!   r  r    r  rN   c           	      N    t        dddd      }t        dd| it        |fi |S )N)r  r  ru  r  r  r  r   )i   i   r  r  )efficientvit_l3r  r  s      r!   r  r    s7    (2[giJ%mJmRVWaRlekRlmmrN   )r   )r   r   )Fr   NN)rF  r  )Hr   __all__typingr   r   r   r   r	   	functoolsr
   r   torch.nnrE   torch.nn.functional
functionalr   	timm.datar   r   timm.layersr   r   r   _builderr   	_featuresr   _features_fxr   _manipulater   	_registryr   r   r   r   anyr"   rW   r'   r,   rZ   r/   r^   rg   rr   ry   r~   r   r   rY   r   rX   r   r   r   r   r   r   r   r   rs  default_cfgsr  r  r  r  r  r  r  r  r  rB   rN   r!   <module>r     s   .
/ 5 5      A E E * + 1 ' <+$$ +%%#  S  #"8sCx  S=SE#s(O  %")) %P+RYY +\.		 .b9RYY 9x/")) /dcbii cL   )		 )XBII 4 !#::: : 	:
 : : : :z+2== +\?		 ?DBRYY BJ2RYY 2j\299 \~`		 `F  % X&"X&  "	X&   FS"X&   FS"X&  "X&$   FS"%X&,   FS"-X&4  "5X&:   FS";X&B   FS"CX&J  "KX&R  "SX&Z   FS"[X&b   FS"cX&j   Hs"kX&r  "sX&z   FS"{X&B "& Hs" "& Hs"KX& Xv		 h h h h h h h h n n n n n nrN   