
    ^j~                        d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZmZmZmZ ddlmZ dd	l m!Z! dd
l"m#Z# ddl$m%Z%m&Z& dgZ'dddddZ(dddddZ)dddddZ* G d dejV                        Z, G d dej                  jV                        Z- G d dej                  jV                        Z. G d  d!ej                  jV                        Z/ G d" d#ejV                        Z0 G d$ d%ejV                        Z1 G d& d'ejV                        Z2 G d( d)ejf                        Z4 G d* d+ejV                        Z5 G d, dejV                        Z6d7d-Z7 e% e7d./       e7d./       e7d./       e7d./      d0      Z8d8d1Z9e&d8d2e6fd3       Z:e&d8d2e6fd4       Z;e&d8d2e6fd5       Z<e&d8d2e6fd6       Z=y)9aJ   EfficientFormer-V2

@article{
    li2022rethinking,
    title={Rethinking Vision Transformers for MobileNet Size and Speed},
    author={Li, Yanyu and Hu, Ju and Wen, Yang and Evangelidis, Georgios and Salahi, Kamyar and Wang, Yanzhi and Tulyakov, Sergey and Ren, Jian},
    journal={arXiv preprint arXiv:2212.08059},
    year={2022}
}

Significantly refactored and cleaned up for timm from original at: https://github.com/snap-research/EfficientFormer

Original code licensed Apache 2.0, Copyright (c) 2022 Snap Inc.

Modifications and timm support by / Copyright 2023, Ross Wightman
    N)partial)DictListOptionalTupleTypeUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)create_conv2dcreate_norm_layerget_act_layerget_norm_layerConvNormActLayerScale2dDropPathcalculate_drop_path_ratestrunc_normal_	to_2tuple	to_ntuplendgrid   )build_model_with_cfg)feature_take_indices)checkpoint_seq)generate_default_cfgsregister_modelEfficientFormerV2)(   P        )    @      i   )r$   0   x      )r$   r'   `      )LS2S1S0)   r0      
   )   r3         )   r6   	      )   r9   r8   r3   )r3   r3   )r3   r3   r3   r3   r6   r6   r6   r6   r6   r6   r6   r3   r3   r3   r3   )
r3   r3   r3   r6   r6   r6   r6   r3   r3   r3   )r3   r3   )r3   r3   r6   r6   r6   r6   r6   r6   r3   r3   r3   r3   )r3   r3   r6   r6   r6   r6   r3   r3   )r3   r3   )	r3   r3   r6   r6   r6   r6   r3   r3   r3   )r3   r3   r6   r6   r3   r3   )r3   r3   )r3   r6   r6   r6   r3   r3   )r3   r6   r6   r3   c                   t     e Zd Z	 	 	 	 	 	 	 	 	 	 ddededededeeef   dededed	ed
ee   f fdZ	d Z
 xZS )ConvNormin_channelsout_channelskernel_sizestridepaddingdilationgroupsbias
norm_layernorm_kwargsc           	          ||d}|
xs i }
t         |           t        |||f|||||d|| _        t	        |	|fi |
|| _        y )Ndevicedtype)r?   r@   rA   rB   rC   )super__init__r   convr   bn)selfr<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rH   rI   dd	__class__s                 i/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/efficientformer_v2.pyrK   zConvNorm.__init__F   ss     /!'R!

 

 

	 $JRRrR    c                 J    | j                  |      }| j                  |      }|S N)rL   rM   rN   xs     rQ   forwardzConvNorm.forwarde   s!    IIaLGGAJrR   )
r   r    r   r   Tbatchnorm2dNNN)__name__
__module____qualname__intr	   strboolr   r   rK   rW   __classcell__rP   s   @rQ   r;   r;   E   s    
  !')+*.SS S 	S
 S 38_S S S S S "$S>rR   r;   c                   T    e Zd ZU eeej                  f   ed<   dddddej                  dddf	de
d	e
d
e
de
dee
ee
e
f   f   deej                     dee
   f fdZ ej"                         d fd	       ZddZddZddZddZdej.                  dej                  fdZd Z xZS )Attention2dattention_bias_cacher#   r$   r5   r3      Ndimkey_dim	num_heads
attn_ratio
resolution	act_layerr?   c
           	         ||	d}
t         |           || _        |dz  | _        || _        t        |      }|bt        |D cg c]  }t        j                  ||z         c}      }t        ||fd||d|
| _
        t        j                  |d      | _        nd | _
        d | _        || _        | j                  d   | j                  d   z  | _        t!        ||z        | _        t!        ||z        |z  | _        || _        | j                  | j                  z  }t        ||fi |
| _        t        ||fi |
| _        t        || j$                  fi |
| _        t        | j$                  | j$                  fd| j$                  d	|
| _        t        j0                  | j                  | j                  fd
di|
| _        t        j0                  | j                  | j                  fd
di|
| _         |       | _        t        | j$                  |dfi |
| _        t:        j                  j=                  t;        j>                  || j                  fi |
      | _         | jC                  dt;        j>                  | j                  | j                  f|t:        jD                        d       i | _#        | jI                          y c c}w )NrG         r6   r>   r?   rB   bilinear)scale_factormoder   r   )r>   rB   r>   attention_bias_idxsF
persistent)%rJ   rK   rh   scalerg   r   tuplemathceilr;   stride_convnnUpsampleupsamplerj   Nr]   ddhri   qkvv_localConv2dtalking_head1talking_head2actprojtorch	Parameteremptyattention_biasesregister_bufferlongrd   reset_parameters)rN   rf   rg   rh   ri   rj   rk   r?   rH   rI   rO   rkhrP   s                rQ   rK   zAttention2d.__init__n   sc    /"_
z*
zJ!		!f* 5JKJ'SaaWZa^`aDKKV*MDM#D DM$#dooa&88Z')*j7*+i7$\\DNN*#r(R(#r(R(#tww-"-VaVSUVYYt~~t~~[ST[XZ[YYt~~t~~[ST[XZ[;TWWc133	 % 2 25;;y$&&3WTV3W X!KK(uzzJ 	 	

 %'! 	C  Ks   Kc                 R    t         |   |       |r| j                  ri | _        y y y rT   rJ   trainrd   rN   rq   rP   s     rQ   r   zAttention2d.train   )    dD--(*D% .4rR   returnc                 v    t         j                  j                  | j                         | j	                          yz"Initialize parameters and buffers.Nrz   initzeros_r   _init_buffersrN   s    rQ   r   zAttention2d.reset_parameters   $    
t,,-rR   c           
         t        j                  t        t        j                  | j                  d   |t         j
                        t        j                  | j                  d   |t         j
                                    j                  d      }|ddddf   |ddddf   z
  j                         }|d   | j                  d   z  |d   z   }|S )5Compute relative position indices for attention bias.r   rG   r   .Nr   stackr   arangerj   r   flattenabs)rN   rH   posrel_poss       rQ   _compute_attention_bias_idxsz(Attention2d._compute_attention_bias_idxs   s    kk&LL+F%**MLL+F%**M
  71: 	 sAt|$s3a<'88==?1: 22gaj@rR   c                     | j                   j                  | j                  | j                   j                               i | _        yz.Compute and fill non-persistent buffer values.)rH   Nrr   copy_r   rH   rd   r   s    rQ   r   zAttention2d._init_buffers   =      &&--T5M5M5T5T-U	
 %'!rR   c                 $    | j                          yz"Initialize non-persistent buffers.Nr   r   s    rQ   init_non_persistent_buffersz'Attention2d.init_non_persistent_buffers       rR   rH   c                 4   t         j                  j                         s| j                  r| j                  d d | j
                  f   S t        |      }|| j                  vr*| j                  d d | j
                  f   | j                  |<   | j                  |   S rT   r   jit
is_tracingtrainingr   rr   r^   rd   rN   rH   
device_keys      rQ   get_attention_biasesz Attention2d.get_attention_biases       99!T]]((D,D,D)DEEVJ!:!::8<8M8MaQUQiQiNi8j))*5,,Z88rR   c                 $   |j                   \  }}}}| j                  | j                  |      }| j                  |      j                  || j                  d| j
                        j                  dddd      }| j                  |      j                  || j                  d| j
                        j                  dddd      }| j                  |      }| j                  |      }	|j                  || j                  d| j
                        j                  dddd      }||z  | j                  z  }
|
| j                  |j                        z   }
| j                  |
      }
|
j                  d      }
| j                  |
      }
|
|z  j!                  dd      }|j                  || j"                  | j$                  d   | j$                  d         |	z   }| j&                  | j'                  |      }| j)                  |      }| j+                  |      }|S Nr   r   r6   r9   rf   )shapery   r   reshaperh   r}   permuter   r   r   ru   r   rH   r   softmaxr   	transposer   rj   r|   r   r   rN   rV   BCHWr   r   r   r   attns              rQ   rW   zAttention2d.forward   s   WW
1a'  #AFF1IaTVV<DDQ1aPFF1IaTVV<DDQ1aPFF1I,,q/IIaTVV4<<Q1aHA#d//99!!$'|||#!!$'AX  A&IIa$//!"4dooa6HIGS==$a AHHQKIIaLrR   Tr   NrT   )rZ   r[   r\   r   r^   r   Tensor__annotations__rz   GELUr]   r	   r   r   Moduler   rK   no_gradr   r   r   r   r   rH   r   rW   r`   ra   s   @rQ   rc   rc   k   s    sELL011 67)+$(5 5  5  	5 
 5  c5c?235  BII5  SM5 n U]]_+ +

'95<< 9ELL 9rR   rc   c                   4     e Zd Z	 	 ddedef fdZd Z xZS )LocalGlobalQueryin_dimout_dimc                     ||d}t         |           t        j                  ddd      | _        t        j
                  ||fddd|d|| _        t        ||dfi || _        y )NrG   r   r9   r   r6   )r>   r?   r@   rB   )	rJ   rK   rz   	AvgPool2dpoolr   localr;   r   )rN   r   r   rH   rI   rO   rP   s         rQ   rK   zLocalGlobalQuery.__init__   sh     /LLAq)	YYvvg1QPQZ`gdfg
VWa626	rR   c                 v    | j                  |      }| j                  |      }||z   }| j                  |      }|S rT   )r   r   r   )rN   rV   local_qpool_qr   s        rQ   rW   zLocalGlobalQuery.forward   s8    **Q-1fIIaLrR   )NN)rZ   r[   r\   r]   rK   rW   r`   ra   s   @rQ   r   r      s'    
 77 7rR   r   c                   T    e Zd ZU eeej                  f   ed<   ddddddej                  ddf	de
d	e
d
e
de
dee
ee
e
f   f   dee
   deej                     f fdZ ej"                         d fd	       ZddZddZddZddZdej.                  dej                  fdZd Z xZS )Attention2dDownsamplerd   r#      r5   r3   re   Nrf   rg   rh   ri   rj   r   rk   c
           	         ||	d}
t         |           || _        |dz  | _        || _        t        |      | _        t        | j                  D cg c]  }t        j                  |dz         c}      | _
        | j                  d   | j                  d   z  | _        | j                  d   | j                  d   z  | _        t        ||z        | _        t        ||z        |z  | _        || _        |xs || _        | j                  | j                  z  }t%        ||fi |
| _        t)        ||dfi |
| _        t)        || j                  dfi |
| _        t)        | j                  | j                  fdd| j                  d|
| _         |       | _        t)        | j                  | j"                  dfi |
| _        t5        j6                  t9        j:                  || j                  fi |
      | _        | j?                  dt9        j:                  | j                  | j                  f|t8        j@                        d	
       i | _!        | jE                          y c c}w )NrG   rm   r9   r   r   r6   rn   rr   Frs   )#rJ   rK   rh   ru   rg   r   rj   rv   rw   rx   resolution2r}   N2r]   r~   r   ri   r   r   r   r;   r   r   r   r   r   rz   r   r   r   r   r   r   rd   r   )rN   rf   rg   rh   ri   rj   r   rk   rH   rI   rO   r   r   rP   s                rQ   rK   zAttention2dDownsample.__init__  s    /"_
#J/ DOO!Lq$))AE"2!LM#dooa&88""1%(8(8(;;Z')*j7*+i7$~#\\DNN*!#r0R0#r1++#tww0R0`aRVRYRY`]_`;TWWdllA<<	 "U[[DFF-Qb-Q R!KK$&&)&

K 	 	

 %'! 	7 "Ms   Ic                 R    t         |   |       |r| j                  ri | _        y y y rT   r   r   s     rQ   r   zAttention2dDownsample.train2  r   rR   r   c                 v    t         j                  j                  | j                         | j	                          yr   r   r   s    rQ   r   z&Attention2dDownsample.reset_parameters8  r   rR   c                    t        j                  t        t        j                  | j                  d   |t         j
                        t        j                  | j                  d   |t         j
                                    j                  d      }t        j                  t        t        j                  d| j                  d   d|t         j
                        t        j                  d| j                  d   d|t         j
                                    j                  d      }|ddddf   |ddddf   z
  j                         }|d   | j                  d   z  |d   z   }|S )r   r   rG   r   r9   )steprH   rI   .Nr   )rN   rH   k_posq_posr   s        rQ   r   z2Attention2dDownsample._compute_attention_bias_idxs=  s   FLL+F%**MLL+F%**M
  71: 	 FLLDOOA.QvUZZXLLDOOA.QvUZZX
  71: 	 a&sD!|)<<AAC1: 22gaj@rR   c                     | j                   j                  | j                  | j                   j                               i | _        yr   r   r   s    rQ   r   z#Attention2dDownsample._init_buffersK  r   rR   c                 $    | j                          yr   r   r   s    rQ   r   z1Attention2dDownsample.init_non_persistent_buffersR  r   rR   rH   c                 4   t         j                  j                         s| j                  r| j                  d d | j
                  f   S t        |      }|| j                  vr*| j                  d d | j
                  f   | j                  |<   | j                  |   S rT   r   r   s      rQ   r   z*Attention2dDownsample.get_attention_biasesV  r   rR   c                 l   |j                   \  }}}}| j                  |      j                  || j                  d| j                        j                  dddd      }| j                  |      j                  || j                  d| j                        j                  dddd      }| j                  |      }| j                  |      }	|j                  || j                  d| j                        j                  dddd      }||z  | j                  z  }
|
| j                  |j                        z   }
|
j                  d      }
|
|z  j                  dd      }|j                  || j                  | j                   d   | j                   d         |	z   }| j#                  |      }| j%                  |      }|S r   )r   r   r   rh   r   r   r   r}   r   r   ru   r   rH   r   r   r   r   r   r   r   s              rQ   rW   zAttention2dDownsample.forward_  sk   WW
1aFF1IaTWW=EEaAqQFF1IaTVV<DDQ1aPFF1I,,q/IIaTVV4<<Q1aHA#d//99|||#AX  A&IIa$"2"21"5t7G7G7JKgUHHQKIIaLrR   r   r   rT   )rZ   r[   r\   r   r^   r   r   r   rz   r   r]   r	   r   r   r   r   rK   r   r   r   r   r   r   rH   r   rW   r`   ra   s   @rQ   r   r      s    sELL011 67%))+. .  .  	. 
 .  c5c?23.  c].  BII. ` U]]_+ +

'95<< 9ELL 9rR   r   c                       e Zd Zdddddej                  ej
                  ddf	deded	eeeeef   f   d
eeeeef   f   deeeeef   f   deeeeef   f   de	de
ej                     dee
ej                        f fdZd Z xZS )
Downsampler6   r9   r   re   FNin_chsout_chsr>   r?   r@   rj   use_attnrk   rD   c                 
   |
|d}t         |           t        |      }t        |      }t        |      }|	xs t        j                         }	t        ||f||||	d|| _        |rt        d||||d|| _        y d | _        y )NrG   )r>   r?   r@   rD   )rf   r   rj   rk    )	rJ   rK   r   rz   Identityr;   rL   r   r   )rN   r   r   r>   r?   r@   rj   r   rk   rD   rH   rI   rO   rP   s                rQ   rK   zDownsample.__init__t  s     /,6"G$02;;=

 $!
 
	 - %#	
 DI DIrR   c                 h    | j                  |      }| j                  | j                  |      |z   S |S rT   )rL   r   )rN   rV   outs      rQ   rW   zDownsample.forward  s1    iil99 99Q<#%%
rR   )rZ   r[   r\   rz   r   BatchNorm2dr]   r	   r   r_   r   r   r   rK   rW   r`   ra   s   @rQ   r   r   s  s    
 89233467")+46NN(( ( sE#s(O34	(
 #uS#X./( 3c3h/0( c5c?23( ( BII( !bii1(TrR   r   c                        e Zd ZdZddej
                  ej                  ddddfdedee   dee   de	ej                     d	e	ej                     d
edef fdZd Z xZS )ConvMlpWithNormz`
    Implementation of MLP with 1*1 convolutions.
    Input: tensor with shape [B, C, H, W]
    N        Fin_featureshidden_featuresout_featuresrk   rD   dropmid_convc
                 l   ||	d}
t         |           |xs |}|xs |}t        ||dfd||d|
| _        |rt        ||df|d||d|
| _        nt        j                         | _        t        j                  |      | _        t        ||dfd|i|
| _
        t        j                  |      | _        y )NrG   r   T)rC   rD   rk   r6   )rB   rC   rD   rk   rD   )rJ   rK   r   fc1midrz   r   Dropoutdrop1r;   fc2drop2)rN   r   r   r   rk   rD   r   r   rH   rI   rO   rP   s              rQ   rK   zConvMlpWithNorm.__init__  s     /#2{)8[
 !
 
 "	 '%#	 	DH {{}DHZZ%
O\1ZZWYZZZ%
rR   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }|S rT   )r  r  r  r  r  rU   s     rQ   rW   zConvMlpWithNorm.forward  sH    HHQKHHQKJJqMHHQKJJqMrR   )rZ   r[   r\   __doc__rz   r   r   r]   r   r   r   floatr_   rK   rW   r`   ra   s   @rQ   r   r     s     .2*.)+*,.."(&(& &c](& #3-	(&
 BII(& RYY(& (& (&TrR   r   c                        e Zd Zdej                  ej
                  ddddddddfdeded	eej                     d
eej                     dedede
e   deeeeef   f   de
e   def fdZd Z xZS )EfficientFormerV2Block      @r   h㈵>re   NTrf   	mlp_ratiork   rD   	proj_drop	drop_pathlayer_scale_init_valuerj   r?   r   c           
         ||d}t         |           |
rgt        |f|||	d|| _        |t	        ||fi |nt        j                         | _        |dkD  rt        |      nt        j                         | _	        nd | _        d | _        d | _	        t        d|t        ||z        |||dd|| _        |t	        ||fi |nt        j                         | _        |dkD  rt        |      | _        y t        j                         | _        y )NrG   )rj   rk   r?   r   T)r   r   rk   rD   r   r   r   )rJ   rK   rc   token_mixerr   rz   r   ls1r   
drop_path1r   r]   mlpls2
drop_path2)rN   rf   r  rk   rD   r  r  r  rj   r?   r   rH   rI   rO   rP   s                 rQ   rK   zEfficientFormerV2Block.__init__  s1    /* %#	 
  D 7M6X $+3/13^`^i^i^k H5>^hy1DO#DDH"DO" 
i0!
 
 3I2T  '/+-/Z\ZeZeZg 	1:R(9-R[[]rR   c                     | j                   2|| j                  | j                  | j                  |                  z   }|| j                  | j	                  | j                  |                  z   }|S rT   )r  r  r  r  r  r  rU   s     rQ   rW   zEfficientFormerV2Block.forward  s^    'DOODHHT-=-=a-@$ABBA! 566rR   )rZ   r[   r\   rz   r   r   r]   r
  r   r   r   r	   r   r_   rK   rW   r`   ra   s   @rQ   r  r    s      ")+*,..!!6:67$(!-S-S -S BII	-S
 RYY-S -S -S %-UO-S c5c?23-S SM-S -S^rR   r  c            
            e Zd Zej                  ej
                  ddfdededeej                     deej                     f fdZ	 xZ
S )Stem4Nr   r   rk   rD   c           
          ||d}t         |           d| _        t        ||dz  fdddd||d|| _        t        |dz  |fdddd||d|| _        y )NrG   r3   r9   r6   r   T)r>   r?   r@   rC   rD   rk   )rJ   rK   r?   r   conv1conv2)	rN   r   r   rk   rD   rH   rI   rO   rP   s	           rQ   rK   zStem4.__init__  s     / qL	
 a!	
 	

 !qL

 !

 


rR   )rZ   r[   r\   rz   r   r   r]   r   r   rK   r`   ra   s   @rQ   r  r    sY    
 *,*,.. 
 
  
 BII	 

 RYY 
  
rR   r  c                         e Zd Zddddddddddej                  ej
                  ddfd	ed
ededeeeeef   f   de	de
e   de	de	dedeeeedf   f   dedeeee   f   de
e   deej                     deej                     f fdZd Z xZS )EfficientFormerV2Stagere   TNFr   r  r   r  rf   dim_outdepthrj   
downsampleblock_stridedownsample_use_attnblock_use_attnnum_vitr  .r  r  r  rk   rD   c                    ||d}t         |           d| _         t        |      |
      }
t	        |      }|rIt        ||f||||d|| _        |}t        |D cg c]  }t        j                  |dz         c}      }n ||k(  sJ t        j                         | _        g }t        |      D ]3  }||	z
  dz
  }t        |f|||
|   |xr ||kD  |||   |||d	|}||gz  }5 t        j                  | | _        y c c}w )NrG   F)r   rj   rD   rk   r9   r   )	rj   r?   r  r   r  r  r  rk   rD   )rJ   rK   grad_checkpointingr   r   r   r$  rv   rw   rx   rz   r   ranger  
Sequentialblocks)rN   rf   r"  r#  rj   r$  r%  r&  r'  r(  r  r  r  r  rk   rD   rH   rI   rO   r   r-  	block_idx
remain_idxbrP   s                           rQ   rK   zEfficientFormerV2Stage.__init__;  sG   ( /"'$Ie$Y/	z*
( -%%# DO C*EQ		!a% 0EFJ'>!> kkmDOu 	I1,J&%##I.'BI
,B##I.'=#% A qcMF	  mmV,-  Fs   C>c                     | j                  |      }| j                  r6t        j                  j	                         st        | j                  |      }|S | j                  |      }|S rT   )r$  r*  r   r   is_scriptingr   r-  rU   s     rQ   rW   zEfficientFormerV2Stage.forwardx  sS    OOA""599+A+A+Ct{{A.A  AArR   )rZ   r[   r\   rz   r   r   r]   r	   r   r_   r   r
  r   r   r   rK   rW   r`   ra   s   @rQ   r!  r!  9  s!    78#*.(-#(9;!356:)+*,..%;-;- ;- 	;-
 c5c?23;- ;- #3-;- "&;- !;- ;- UE%*$556;- ;- UDK/0;- %-UO;- BII;-  RYY!;-zrR   r!  c            #       `    e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d,deedf   dedeeeeef   f   dedeeedf      deeedf      dee	ee	df   eee	df   df   f   d	ed
e	dedede	de	de	dee	   dedef" fdZ
d-defdZd-defdZej                  j                  d        Zej                  j                  d.d       Zej                  j                  d-d       Zej                  j                  dej(                  fd       Zd/dedee   fdZej                  j                  d-d       Z	 	 	 	 	 d0dej0                  deeeee   f      d ed!ed"ed#edeeej0                     eej0                  eej0                     f   f   fd$Z	 	 	 d1deeee   f   d%ed&efd'Zd( Zd.d)efd*Zd+ Z xZS )2r   depths.in_chansimg_sizeglobal_pool
embed_dimsdownsamples
mlp_ratiosrD   norm_epsrk   num_classes	drop_rateproj_drop_ratedrop_path_rater  r(  distillationc                    t         |           ||d}|dv sJ || _        || _        || _        g | _        t        |      }t        t        |      |	      }t        |
      }
t        ||d   f|
|d|| _        |d   }d}t        |      }t        ||d      }|xs d	d
t        |      dz
  z  z   } t        |      |      }g }t        |      D ]  }t!        |D cg c]  }t#        j$                  ||z         c}      }t'        |||   f||   |||   |dk(  rdnd |dk\  |dk\  |||   |||   ||
|d|}||   r|dz  }||   }| xj
                  t)        ||d|       gz  c_        |j+                  |        t-        j.                  | | _        |d   x| _        | _         ||d   fi || _        t-        j8                  |      | _        |dkD  rt-        j<                  |d   |fi |nt-        j>                         | _         || _!        | jB                  r9|dkD  rt-        j<                  |d   |fi |nt-        j>                         | _"        nd | _"        | jG                  d       d| _$        y c c}w )NrG   )avgrX   )epsr   )rk   rD   r3   T)	stagewiseFr   r   r9   r6   )r#  rj   r$  r%  r&  r'  r(  r  r  r  r  rk   rD   zstages.)num_chs	reductionmoduler   Fneeds_reset)%rJ   rK   r<  r5  r7  feature_infor   r   r   r   r  stemlenr   r   r+  rv   rw   rx   r!  dictappendrz   r,  stagesnum_featureshead_hidden_sizenormr  	head_dropLinearr   headdist	head_distinit_weightsdistilled_training)rN   r4  r5  r6  r7  r8  r9  r:  rD   r;  rk   r<  r=  r>  r?  r  r(  r@  rH   rI   rO   prev_dimr?   
num_stagesdprrP  iscurr_resolutionstagerP   s                                 rQ   rK   zEfficientFormerV2.__init__  s   , 	/k)))& &X&^J7XF
!),	(JqMdYS]dacd	a=[
'$O!KX3v;?0K%K*Yz*:6
z" 	!A#H$MqTYYq6z%:$MNO*1 Qi*&q>"#q&Qd$%F Av$Q-(a&'=#%  !E$ 1~!!!}H$x6T[\][^R_"`!aaMM% 1	!2 mmV, 5?rNBD1z"~44	I.DORSOBIIjnk@R@Y[YdYdYf	 	99MX[\_RYYz"~{IbIbdbmbmboDN!DN 	e,"'Q %Ns   I9
rJ  c                    t        |t        j                        rOt        |j                  d       |j
                  +t        j                  j                  |j
                  d       y y |rt        |d      r|j                          y y y )N{Gz?)stdr   r   )

isinstancerz   rU  r   weightrC   r   	constant_hasattrr   )rN   mrJ  s      rQ   _init_weightszEfficientFormerV2._init_weights  se    a#!((,vv!!!!&&!, "WQ(:;  <[rR   c                 P    | j                  t        | j                  |             y )NrI  )applyr   rj  )rN   rJ  s     rQ   rY  zEfficientFormerV2.init_weights  s    

74--;GHrR   c                 ^    | j                         D ch c]  \  }}d|v s| c}}S c c}}w )Nr   )named_parameters)rN   r   _s      rQ   no_weight_decayz!EfficientFormerV2.no_weight_decay  s+    "335Qda9Kq9PQQQs   ))c                 $    t        dddg      }|S )Nz^stem)z^stages\.(\d+)N)z^norm)i )rL  r-  )rN  )rN   coarsematchers      rQ   group_matcherzEfficientFormerV2.group_matcher  s    -/CD
 rR   c                 4    | j                   D ]	  }||_         y rT   )rP  r*  )rN   enabler_  s      rQ   set_grad_checkpointingz(EfficientFormerV2.set_grad_checkpointing  s     	*A#)A 	*rR   r   c                 2    | j                   | j                  fS rT   rV  rX  r   s    rQ   get_classifierz EfficientFormerV2.get_classifier  s    yy$..((rR   c                 (   || _         ||| _        |dkD  r t        j                  | j                  |      nt        j
                         | _        |dkD  r&t        j                  | j                  |      | _        y t        j
                         | _        y )Nr   )r<  r7  rz   rU  rQ  r   rV  rX  )rN   r<  r7  s      rQ   reset_classifierz"EfficientFormerV2.reset_classifier  sq    &"*DALqBIId//=VXVaVaVc	FQTUo4#4#4kB[][f[f[hrR   c                     || _         y rT   )rZ  )rN   rv  s     rQ   set_distilled_trainingz(EfficientFormerV2.set_distilled_training  s
    "(rR   rV   indicesrS  
stop_early
output_fmtintermediates_onlyc                 &   |dv sJ d       g }t        t        | j                        |      \  }}	| j                  |      }t        | j                        dz
  }
t        j
                  j                         s|s| j                  }n| j                  d|	dz    }t        |      D ]O  \  }} ||      }||v s||
k(  r'|r| j                  |      n|}|j                  |       ?|j                  |       Q |r|S |
k(  r| j                  |      }||fS )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWzOutput shape must be NCHW.r   N)
r   rM  rP  rL  r   r   r2  	enumeraterS  rO  )rN   rV   r  rS  r  r  r  intermediatestake_indices	max_indexlast_idxrP  feat_idxra  x_inters                  rQ   forward_intermediatesz'EfficientFormerV2.forward_intermediates  s   * Y&D(DD&"6s4;;7G"Qi IIaLt{{#a'99!!#:[[F[[)a-0F(0 	,OHeaA<'x'.2diilG!((1!((+	,   x		!A-rR   
prune_norm
prune_headc                     t        t        | j                        |      \  }}| j                  d|dz    | _        |rt        j                         | _        |r| j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   rX   )r   rM  rP  rz   r   rS  r|  )rN   r  r  r  r  r  s         rQ   prune_intermediate_layersz+EfficientFormerV2.prune_intermediate_layers5  s]     #7s4;;7G"Qikk.9q=1DI!!!R(rR   c                 l    | j                  |      }| j                  |      }| j                  |      }|S rT   )rL  rP  rS  rU   s     rQ   forward_featuresz"EfficientFormerV2.forward_featuresE  s.    IIaLKKNIIaLrR   
pre_logitsc                 6   | j                   dk(  r|j                  d      }| j                  |      }|r|S | j                  |      | j	                  |      }}| j
                  r.| j                  r"t        j                  j                         s||fS ||z   dz  S )NrB  )r9   r6   r   r9   )
r7  meanrT  rV  rX  rZ  r   r   r   r2  )rN   rV   r  x_dists       rQ   forward_headzEfficientFormerV2.forward_headK  s    u$6"ANN1HIIaL$.."36""t}}UYY=S=S=Uf9 J!##rR   c                 J    | j                  |      }| j                  |      }|S rT   )r  r  rU   s     rQ   rW   zEfficientFormerV2.forwardY  s'    !!!$a rR   )r6   r)   rB  NNr3   rY   r  gelu  r   r   r   r  r   TNNr   rE  rT   )NFFr  F)r   FT) rZ   r[   r\   r   r]   r	   r^   r   r_   r
  rK   rj  rY  r   r   ignorerp  rt  rw  rz   r   rz  r|  r~  r   r   r  r  r  r  rW   r`   ra   s   @rQ   r   r     s    47$486:YZ+"##!$&$&6:!%)S(#s(OS( S( CsCx01	S(
 S( !sCx1S( "%c	"23S( eU5#:%6eE3J>OQT>T8UUVS( S( S( S( S( S( "S( "S(  %-UO!S(" #S($ %S(j!D !I I YYR R YY  YY* * YY)		 ) )iC ihsm i YY) ) 8<$$',1 ||1  eCcN341  	1 
 1  1  !%1  
tELL!5tELL7I)I#JJ	K1 j ./$#	3S	>*  	 $$ $rR   c                 4    | ddd dddt         t        dddd	|S )
Nr  )r6   r)   r)   Tgffffff?bicubicry  zstem.conv1.convz
apache-2.0)urlr<  
input_size	pool_sizefixed_input_sizecrop_pctinterpolationr  rd  
classifier
first_convlicenser
   )r  kwargss     rQ   _cfgr  _  s7    =tae)%.B+;L  rR   ztimm/)	hf_hub_id)z#efficientformerv2_s0.snap_dist_in1kz#efficientformerv2_s1.snap_dist_in1kz#efficientformerv2_s2.snap_dist_in1kz"efficientformerv2_l.snap_dist_in1kc                 h    |j                  dd      }t        t        | |fdt        d|      i|}|S )Nout_indices)r   r   r9   r6   feature_cfgT)flatten_sequentialr  )popr   r   rN  )variant
pretrainedr  r  models        rQ   _create_efficientformerv2r  {  sC    **]L9K 7JDkJ E LrR   r   c           	      z    t        t        d   t        d   ddt        d         }t	        dd| it        |fi |S )Nr/   r9   r   r4  r8  r(  r?  r:  r  )efficientformerv2_s0rN  EfficientFormer_depthEfficientFormer_width EfficientFormer_expansion_ratiosr  r  r  
model_argss      rQ   r  r    L    $T*(.3D9J %q
qVZ[eVpioVpqqrR   c           	      z    t        t        d   t        d   ddt        d         }t	        dd| it        |fi |S )Nr.   r9   r   r  r  )efficientformerv2_s1r  r  s      rQ   r  r    r  rR   c           	      z    t        t        d   t        d   ddt        d         }t	        dd| it        |fi |S )Nr-   r3   rc  r  r  )efficientformerv2_s2r  r  s      rQ   r  r    sL    $T*(.3D9J %q
qVZ[eVpioVpqqrR   c           	      z    t        t        d   t        d   ddt        d         }t	        dd| it        |fi |S )Nr,   r8   g?r  r  )efficientformerv2_lr  r  s      rQ   r  r    sL    $S)(-3C8J %pzpUYZdUohnUopprR   )rX   rE  )>r	  rw   	functoolsr   typingr   r   r   r   r   r	   r   torch.nnrz   	timm.datar   r   timm.layersr   r   r   r   r   r   r   r   r   r   r   r   _builderr   	_featuresr   _manipulater   	_registryr   r   __all__r  r  r  r   r;   rc   r   r   r   r   r  r,  r  r!  r   r  default_cfgsr  r  r  r  r  r   rR   rQ   <module>r     s      ; ;   A    + + ' < 
 



	  



	  
_
P
A
2	$  #ryy #L{%((// {|uxx ,qEHHOO qh/ /d6bii 6r4RYY 4n!
BMM !
HERYY EP[		 [|	 %+/, ,0, ,0, +/+&   r8I r r r8I r r r8I r r q7H q qrR   