
    ^j                        d Z ddlmZ ddlmZ ddlmZmZmZm	Z	m
Z
mZ ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZmZ dd	lmZ dd
lmZ ddlmZmZ ddl m!Z!m"Z" dgZ# G d dejH                        Z% G d dejH                        Z& G d dejH                        Z' G d dejP                        Z) G d dejP                        Z* G d dejH                        Z+ G d dejH                        Z, G d dejH                        Z- G d dejH                        Z. G d  d!ejH                        Z/ G d" d#ejH                        Z0 G d$ d%ejH                        Z1 G d& dejH                        Z2 G d' d(e2      Z3d) Z4 e5 e5d*d+d,d-.       e5d*d+d/d0.       e5d1d2d3d0.       e5d4d2d,d0.       e5d5d2d6d0.       e5d5d2d6d0d7d89       e5d:d;d<d0d7d89       e5d=d;d>d0d7?       e5d4d2d,d@d7?       e5dAd;dBd@d7?      C
      Z6dsdDZ7dtdEZ8 e!i dF e8dGH      dI e8dGH      dJ e8dGH      dK e8dGH      dL e8dGH      dM e8dGdNO      dP e8dGdNO      dQ e8dGdNO      dR e8dGdNO      dS e8dGdNO      dT e8dUV      dW e8dUV      dX e8dUV      dY e8dUV      dZ e8dUV      d[ e8dUV      d\ e8dUV       e8dUV       e8dUV       e8dUV      d]      Z9e"dud^e2fd_       Z:e"dud^e2fd`       Z;e"dud^e2fda       Z<e"dud^e2fdb       Z=e"dud^e2fdc       Z>e"dud^e2fdd       Z?e"dud^e2fde       Z@e"dud^e2fdf       ZAe"dud^e2fdg       ZBe"dud^e2fdh       ZCe"dud^e2fdi       ZDe"dud^e2fdj       ZEe"dud^e2fdk       ZFe"dud^e2fdl       ZGe"dud^e2fdm       ZHe"dud^e2fdn       ZIe"dud^e2fdo       ZJe"dud^e2fdp       ZKe"dud^e2fdq       ZLe"dud^e2fdr       ZMy)va   LeViT

Paper: `LeViT: a Vision Transformer in ConvNet's Clothing for Faster Inference`
    - https://arxiv.org/abs/2104.01136

@article{graham2021levit,
  title={LeViT: a Vision Transformer in ConvNet's Clothing for Faster Inference},
  author={Benjamin Graham and Alaaeldin El-Nouby and Hugo Touvron and Pierre Stock and Armand Joulin and Herv'e J'egou and Matthijs Douze},
  journal={arXiv preprint arXiv:22104.01136},
  year={2021}
}

Adapted from official impl at https://github.com/facebookresearch/LeViT, original copyright bellow.

This version combines both conv/linear models and fixes torchscript compatibility.

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
    )OrderedDict)partial)DictListOptionalTupleTypeUnionN)IMAGENET_DEFAULT_STDIMAGENET_DEFAULT_MEAN)	to_ntuple	to_2tupleget_act_layerDropPathtrunc_normal_ndgrid   )build_model_with_cfg)feature_take_indices)
checkpointcheckpoint_seq)generate_default_cfgsregister_modelLevitc                        e Zd Z	 	 	 	 	 	 	 	 ddedededededededef fd	Z ej                         d
        Zd Z	 xZ
S )ConvNormin_chsout_chskernel_sizestridepaddingdilationgroupsbn_weight_initc           	         |	|
d}t         |           t        j                  |||||||fddi|| _        t        j
                  |fi || _        t        j                  j                  | j                  j                  |       y NdevicedtypebiasF)
super__init__nnConv2dlinearBatchNorm2dbninit	constant_weight)selfr   r   r   r    r!   r"   r#   r$   r(   r)   dd	__class__s               \/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/levit.pyr,   zConvNorm.__init__,   st     /iifgxY_rfkroqr../B/
$''...9    c           	          | j                   | j                  }}|j                  |j                  |j                  z   dz  z  }|j                  |d d d d d f   z  }|j
                  |j                  |j                  z  |j                  |j                  z   dz  z  z
  }t        j                  |j                  d      |j                  d      |j                  dd  | j                   j                  | j                   j                  | j                   j                  | j                   j                        }|j                  j                  j!                  |       |j
                  j                  j!                  |       |S )N      ?r   r      )r    r!   r"   r#   )r/   r1   r4   running_varepsr*   running_meanr-   r.   sizeshaper    r!   r"   r#   datacopy_)r5   cr1   wbms         r8   fusezConvNorm.fuse@   s   TWW2II"&&0S88HHqD$,--GGboo		1R^^bff5LQT4TTTIIFF1Iqvvay!''!"+dkk6H6HKK''$++2F2Ft{{OaOac 	
A	!r9   c                 B    | j                  | j                  |            S N)r1   r/   r5   xs     r8   forwardzConvNorm.forwardM   s    wwt{{1~&&r9   )r   r   r   r   r   r   NN__name__
__module____qualname__intfloatr,   torchno_gradrH   rM   __classcell__r7   s   @r8   r   r   +   s    
  !$%:: : 	:
 : : : : ":( U]]_
 
'r9   r   c                   h     e Zd Z	 	 	 ddededef fdZ ej                         d        Zd Z	 xZ
S )
LinearNormin_featuresout_featuresr$   c                    ||d}t         |           t        j                  ||fddi|| _        t        j
                  |fi || _        t        j                  j                  | j                  j                  |       y r&   )
r+   r,   r-   Linearr/   BatchNorm1dr1   r2   r3   r4   )r5   rZ   r[   r$   r(   r)   r6   r7   s          r8   r,   zLinearNorm.__init__R   sg     /ii\LLL..44
$''...9r9   c                 6   | j                   | j                  }}|j                  |j                  |j                  z   dz  z  }|j                  |d d d f   z  }|j
                  |j                  |j                  z  |j                  |j                  z   dz  z  z
  }t        j                  |j                  d      |j                  d            }|j                  j                  j                  |       |j
                  j                  j                  |       |S )Nr;   r   r   )r/   r1   r4   r=   r>   r*   r?   r-   r]   r@   rB   rC   )r5   lr1   rE   rF   rG   s         r8   rH   zLinearNorm.fusea   s    TWW2II"&&0S88HHqDz!GGboo		1R^^bff5LQT4TTTIIaffQi+	A	!r9   c                     | j                  |      }| j                  |j                  dd            j                  |      S )Nr   r   )r/   r1   flatten
reshape_asrK   s     r8   rM   zLinearNorm.forwardl   s3    KKNwwqyyA'22155r9   )r   NNrN   rW   s   @r8   rY   rY   Q   sP    
 %&:: : "	: U]]_ 6r9   rY   c                   t     e Zd Z	 	 	 	 	 d	dededededef
 fdZ ej                         d        Z	d Z
 xZS )

NormLinearrZ   r[   r*   stddropc                    ||d}t         	|           t        j                  |fi || _        t        j
                  |      | _        t        j                  ||fd|i|| _        t        | j                  j                  |       | j                  j                  5t        j                  j                  | j                  j                  d       y y )Nr'   r*   )rf   r   )r+   r,   r-   r^   r1   Dropoutrg   r]   r/   r   r4   r*   r2   r3   )
r5   rZ   r[   r*   rf   rg   r(   r)   r6   r7   s
            r8   r,   zNormLinear.__init__r   s     /..33JJt$	ii\KKKdkk((c2;;'GGdkk..2 (r9   c                 6   | j                   | j                  }}|j                  |j                  |j                  z   dz  z  }|j
                  | j                   j                  | j                   j                  z  |j                  |j                  z   dz  z  z
  }|j                  |d d d f   z  }|j
                  $|| j                  j                  j                  z  }n<|j                  |d d d f   z  j                  d      | j                  j
                  z   }t        j                  |j                  d      |j                  d            }|j                  j                  j                  |       |j
                  j                  j                  |       |S )Nr;   r   r   )r1   r/   r4   r=   r>   r*   r?   Tviewr-   r]   r@   rB   rC   )r5   r1   r`   rE   rF   rG   s         r8   rH   zNormLinear.fuse   s-   AII"&&0S88GGdgg**TWW^^;r~~PRPVPV?V[^>^^^HHqqz!66>DKK&&(((AAagJ&,,R04;;3C3CCAIIaffQi+	A	!r9   c                 `    | j                  | j                  | j                  |                  S rJ   )r/   rg   r1   rK   s     r8   rM   zNormLinear.forward   s"    {{499TWWQZ011r9   )Tg{Gz?        NN)rO   rP   rQ   rR   boolrS   r,   rT   rU   rH   rM   rV   rW   s   @r8   re   re   q   sj    
 33 3 	3
 3 3( U]]_ 2r9   re   c                   L     e Zd Z	 	 ddededeej                     f fdZ xZS )Stem8r   r   	act_layerc           	      t   ||d}t         |           d| _        | j                  dt	        ||dz  dfddd|       | j                  d	 |              | j                  d
t	        |dz  |dz  dfddd|       | j                  d |              | j                  dt	        |dz  |dfddd|       y )Nr'      conv1      r<   r   r    r!   act1conv2act2conv3r+   r,   r    
add_moduler   r5   r   r   rs   r(   r)   r6   r7   s          r8   r,   zStem8.__init__   s     /&'Q,!]!UV!]Z\!]^	,'Q,1a!cPQ[\!c`b!cd	,'Q,!^1VW!^[]!^_r9   NN	rO   rP   rQ   rR   r	   r-   Moduler,   rV   rW   s   @r8   rr   rr      s>     `` ` BII	` `r9   rr   c                   L     e Zd Z	 	 ddededeej                     f fdZ xZS )Stem16r   r   rs   c           	         ||d}t         |           d| _        | j                  dt	        ||dz  dfddd|       | j                  d	 |              | j                  d
t	        |dz  |dz  dfddd|       | j                  d |              | j                  dt	        |dz  |dz  dfddd|       | j                  d |              | j                  dt	        |dz  |dfddd|       y )Nr'      rv   ru   rx   r<   r   ry   rz   r{   rw   r|   r}   act3conv4r~   r   s          r8   r,   zStem16.__init__   s     /&'Q,!]!UV!]Z\!]^	,'Q,1a!cPQ[\!c`b!cd	,'Q,1a!cPQ[\!c`b!cd	,'Q,!^1VW!^[]!^_r9   r   r   rW   s   @r8   r   r      s>     `` ` BII	` `r9   r   c            	       N     e Zd Z	 	 	 ddedeeeeef   f   def fdZd Z xZ	S )
Downsampler    
resolutionuse_poolc                     t         |           || _        t        |      | _        |rt        j                  d|dd      | _        y d | _        y )Nrx   r   F)r    r!   count_include_pad)r+   r,   r    r   r   r-   	AvgPool2dpool)r5   r    r   r   r(   r)   r7   s         r8   r,   zDownsample.__init__   sC     	#J/ZbBLL61PUV	hl	r9   c                 p   |j                   \  }}}|j                  || j                  d   | j                  d   |      }| j                  6| j                  |j	                  dddd            j	                  dddd      }n$|d d d d | j
                  d d | j
                  f   }|j                  |d|      S )Nr   r   rx   r<   rk   )rA   rm   r   r   permuter    reshape)r5   rL   BNCs        r8   rM   zDownsample.forward   s    ''1aFF1dooa($//!*<a@99 		!))Aq!Q/088Aq!DA!]t{{]MdkkM12AyyB""r9   )FNN)
rO   rP   rQ   rR   r
   r   rp   r,   rM   rV   rW   s   @r8   r   r      sJ    
 #mm c5c?23m 	m#r9   r   c                   J    e Zd ZU eeej                  f   ed<   ddddej                  ddfde
de
d	e
d
edee
ee
e
f   f   dedeej                      f fdZ ej$                         d fd	       ZddZddZddZddZdej0                  dej                  fdZd Z xZS )	Attentionattention_bias_cacheru         @   FNdimkey_dim	num_heads
attn_ratior   use_convrs   c
                    ||	d}
t         |           |rt        nt        }t	        |      }|| _        || _        |dz  | _        || _        ||z  | _	        t        ||z        | _        t        ||z        |z  | _        || _         ||| j                  | j                  dz  z   fi |
| _        t        j                   t#        d |       fd || j                  |fddi|
fg            | _        |d   |d   z  }t        j&                  t)        j*                  ||fi |
      | _        | j/                  d	t)        j*                  ||f|t(        j0                        d
       i | _        | j5                          y )Nr'         r<   actlnr$   r   r   attention_bias_idxsF
persistent)r+   r,   r   rY   r   r   r   scaler   key_attn_dimrR   val_dimval_attn_dimr   qkvr-   
Sequentialr   proj	ParameterrT   emptyattention_biasesregister_bufferlongr   reset_parameters)r5   r   r   r   r   r   r   rs   r(   r)   r6   ln_layerr   r7   s                r8   r,   zAttention.__init__   ss    /'8Zz*
 "_
#i/:/0
W 45	A$C!2!2T5F5F5J!JQbQMM+IK 8D--sK1KKL/
 # 	
 qMJqM) "U[[A-L-L M!5;;1vfEJJ#Wdi 	 	k$&! 	r9   c                 R    t         |   |       |r| j                  ri | _        y y y rJ   r+   trainr   r5   moder7   s     r8   r   zAttention.train  )    dD--(*D% .4r9   returnc                 v    t         j                  j                  | j                         | j	                          yz"Initialize parameters and buffers.Nr-   r2   zeros_r   _init_buffersr5   s    r8   r   zAttention.reset_parameters  $    
t,,-r9   c           
         t        j                  t        t        j                  | j                  d   |t         j
                        t        j                  | j                  d   |t         j
                                    j                  d      }|ddddf   |ddddf   z
  j                         }|d   | j                  d   z  |d   z   }|S )5Compute relative position indices for attention bias.r   r'   r   .N)rT   stackr   aranger   r   rb   abs)r5   r(   posrel_poss       r8   _compute_attention_bias_idxsz&Attention._compute_attention_bias_idxs  s    kk&LL+F%**MLL+F%**M
  71: 	 sAt|$s3a<'88==?1: 22gaj@r9   c                     | j                   j                  | j                  | j                   j                               i | _        yz.Compute and fill non-persistent buffer values.)r(   Nr   rC   r   r(   r   r   s    r8   r   zAttention._init_buffers  =      &&--T5M5M5T5T-U	
 %'!r9   c                 $    | j                          yz"Initialize non-persistent buffers.Nr   r   s    r8   init_non_persistent_buffersz%Attention.init_non_persistent_buffers#      r9   r(   c                 4   t         j                  j                         s| j                  r| j                  d d | j
                  f   S t        |      }|| j                  vr*| j                  d d | j
                  f   | j                  |<   | j                  |   S rJ   rT   jit
is_tracingtrainingr   r   strr   r5   r(   
device_keys      r8   get_attention_biaseszAttention.get_attention_biases'      99!T]]((D,D,D)DEEVJ!:!::8<8M8MaQUQiQiNi8j))*5,,Z88r9   c                 ^   | j                   r|j                  \  }}}}| j                  |      j                  || j                  d||z        j                  | j                  | j                  | j                  gd      \  }}}|j                  dd      |z  | j                  z  | j                  |j                        z   }	|	j                  d      }	||	j                  dd      z  j                  |d||      }n|j                  \  }}
}| j                  |      j                  ||
| j                  d      j                  | j                  | j                  | j                  gd      \  }}}|j                  dddd      }|j                  dddd      }|j                  dddd      }||z  | j                  z  | j                  |j                        z   }	|	j                  d      }	|	|z  j                  dd      j                  ||
| j                        }| j!                  |      }|S )Nrk   r<   r   rx   r   r   )r   rA   r   rm   r   splitr   r   	transposer   r   r(   softmaxr   r   r   r   )r5   rL   r   r   HWqkvattnr   s              r8   rM   zAttention.forward0  s   ==JAq!Qhhqk&&4>>2q1u..3eT\\4<<QUQ]Q]4^dee.f Aq! KKB'!+tzz9D<U<UVWV^V^<__D<<B<'DT^^B++11!RA>AggGAq!hhqk&&1dnnb**/%t||T\\0Z`a%*b Aq!		!Q1%A		!Q1%A		!Q1%Aq54::%(A(A!(((KKD<<B<'D$$Q*221a9J9JKAIIaLr9   Tr   NrJ   rO   rP   rQ   r   r   rT   Tensor__annotations__r-   SiLUrR   rS   r
   r   rp   r	   r   r,   rU   r   r   r   r   r   r(   r   rM   rV   rW   s   @r8   r   r      s    sELL011  "68")+' '  '  	' 
 '  c5c?23'  '  BII' R U]]_+ +

'95<< 9ELL 9r9   r   c                   Z    e Zd ZU eeej                  f   ed<   ddddddej                  ddf	de
d	e
d
e
de
dede
dee
ee
e
f   f   dededeej                      f fdZ ej$                         d fd	       ZddZddZddZddZdej0                  dej                  fdZd Z xZS )AttentionDownsampler   ru          @r<   r   FNin_dimout_dimr   r   r   r    r   r   r   rs   c                 D   ||d}t         |           t        |      }|| _        || _        || _        || _        ||z  | _        t        ||z        | _	        | j                  | j
                  z  | _
        |dz  | _        || _        | j                  r,t        }t        t        j                   |	rdnd|	rdndd      }nt"        }t        t$        f||	d|} ||| j                  | j                  z   fi || _        t        j(                  t+        d	 ||
      fd ||| j                  fi |fg            | _        t        j(                  t+        d |
       fd || j                  |fi |fg            | _        |d   |d   z  }|d    |z   |d    |z   z  }t        j0                  t3        j4                  ||fi |      | _        | j9                  dt3        j4                  ||f|t2        j:                        d       i | _        | j?                          y )Nr'   r   rx   r   r   F)r   r!   r   )r   r   down)r    r   r   r   r   ) r+   r,   r   r    r   r   r   r   rR   r   r   r   r   r   r   r-   r   rY   r   kvr   r   r   r   r   rT   r   r   r   r   r   r   )r5   r   r   r   r   r   r    r   r   r   rs   r(   r)   r6   r   	sub_layerN_kN_qr7   s                     r8   r,   zAttentionDownsample.__init__M  s    /z*
$"#i/:/0 LL4>>9_
 ==H!)Aqx!QbgiI "H
[zH[XZ[I64#4#4t7H7H#HOBO{Yf-.8FD$5$5<<=,
    MM+IK 8D--w="=>/
 # 	
 mjm+A&()z!}n.F,GG "U[[C-N2-N O2EKKc
SYafakak4ly~$&! 	r9   c                 R    t         |   |       |r| j                  ri | _        y y y rJ   r   r   s     r8   r   zAttentionDownsample.train  r   r9   r   c                 v    t         j                  j                  | j                         | j	                          yr   r   r   s    r8   r   z$AttentionDownsample.reset_parameters  r   r9   c                    t        j                  t        t        j                  | j                  d   |t         j
                        t        j                  | j                  d   |t         j
                                    j                  d      }t        j                  t        t        j                  d| j                  d   | j                  |t         j
                        t        j                  d| j                  d   | j                  |t         j
                                    j                  d      }|ddddf   |ddddf   z
  j                         }|d   | j                  d   z  |d   z   }|S )r   r   r'   r   )stepr(   r)   .N)	rT   r   r   r   r   r   rb   r    r   )r5   r(   k_posq_posr   s        r8   r   z0AttentionDownsample._compute_attention_bias_idxs  s$   FLL+F%**MLL+F%**M
  71: 	 FLLDOOA.T[[W\WaWabLLDOOA.T[[W\WaWab
  71: 	 a&sD!|)<<AAC1: 22gaj@r9   c                     | j                   j                  | j                  | j                   j                               i | _        yr   r   r   s    r8   r   z!AttentionDownsample._init_buffers  r   r9   c                 $    | j                          yr   r   r   s    r8   r   z/AttentionDownsample.init_non_persistent_buffers  r   r9   r(   c                 4   t         j                  j                         s| j                  r| j                  d d | j
                  f   S t        |      }|| j                  vr*| j                  d d | j
                  f   | j                  |<   | j                  |   S rJ   r   r   s      r8   r   z(AttentionDownsample.get_attention_biases  r   r9   c                 p   | j                   rO|j                  \  }}}}|dz
  | j                  z  dz   |dz
  | j                  z  dz   }}| j                  |      j	                  || j
                  d||z        j                  | j                  | j                  gd      \  }}	| j                  |      j	                  || j
                  | j                  d      }
|
j                  dd      |z  | j                  z  | j                  |j                        z   }|j                  d      }|	|j                  dd      z  j                  || j                   ||      }nH|j                  \  }}}| j                  |      j	                  ||| j
                  d      j                  | j                  | j                  gd      \  }}	|j#                  dddd      }|	j#                  dddd      }	| j                  |      j	                  |d| j
                  | j                        j#                  dddd      }
|
|z  | j                  z  | j                  |j                        z   }|j                  d      }||	z  j                  dd      j                  |d| j                         }| j%                  |      }|S )Nr   rk   r<   r   r   rx   r   )r   rA   r    r   rm   r   r   r   r   r   r   r   r   r(   r   r   r   r   r   )r5   rL   r   r   r   r   HHWWr   r   r   r   r   s                r8   rM   zAttentionDownsample.forward  sK   ==JAq!Q!e+a/!a%DKK1G!1KB771:??1dnnb!a%@FFVZVbVbGcijFkDAqq	q$..$,,CAKKB'!+tzz9D<U<UVWV^V^<__D<<B<'DT^^B++44Q8I8I2rRAggGAq!771:??1a<BBDLLRVR^R^C_efBgDAq		!Q1%A		!Q1%Aq	q"dnndllCKKAqRSUVWAq54::%(A(A!(((KKD<<B<'D$$Q*221b$:K:KLAIIaLr9   r   r   rJ   r   rW   s   @r8   r   r   J  s
   sELL011  #68"")+7 7  7  	7 
 7  7  7  c5c?237  7  7  BII7 r U]]_+ +

'95<< 9ELL 9r9   r   c                        e Zd ZdZdddej
                  dddfdedee   dee   ded	e	ej                     d
ef fdZd Z xZS )LevitMlpzL MLP for Levit w/ normalization + ability to switch btw conv and linear
    NFro   rZ   hidden_featuresr[   r   rs   rg   c	                     ||d}	t         |           |xs |}|xs |}|rt        nt        }
 |
||fi |	| _         |       | _        t        j                  |      | _         |
||fddi|	| _	        y )Nr'   r$   r   )
r+   r,   r   rY   ln1r   r-   ri   rg   ln2)r5   rZ   r  r[   r   rs   rg   r(   r)   r6   r   r7   s              r8   r,   zLevitMlp.__init__  s{     /#2{)8['8ZK?B?;JJt$	O\R!RrRr9   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }|S rJ   )r  r   rg   r  rK   s     r8   rM   zLevitMlp.forward  s;    HHQKHHQKIIaLHHQKr9   )rO   rP   rQ   __doc__r-   r   rR   r   rp   r	   r   rS   r,   rM   rV   rW   s   @r8   r
  r
    s~    
 .2*.")+SS &c]S #3-	S
 S BIIS S,r9   r
  c                        e Zd Zdddej                  dddddddfded	ed
ededededeej                     de	eej                        de
eeeef   f   dededef fdZd Z xZS )LevitDownsampleru   r   r   Nr   Fro   r   r   r   r   r   	mlp_ratiors   attn_act_layerr   r   r   	drop_pathc                    ||d}t         |           |xs |}t        d|||||||	|
|d	|| _        t	        |t        ||z        f|
|d|| _        |dkD  rt        |      | _
        y t        j                         | _
        y )Nr'   )	r   r   r   r   r   rs   r   r   r   r   rs   ro    )r+   r,   r   attn_downsampler
  rR   mlpr   r-   Identityr  )r5   r   r   r   r   r   r  rs   r  r   r   r   r  r(   r)   r6   r7   s                   r8   r,   zLevitDownsample.__init__  s    " /'492  
!$! 
  
 )#$
 	

 
 1:B),BKKMr9   c                 n    | j                  |      }|| j                  | j                  |            z   }|S rJ   )r  r  r  rK   s     r8   rM   zLevitDownsample.forward  s2      #txx{++r9   )rO   rP   rQ   r-   r   rR   rS   r	   r   r   r
   r   rp   r,   rM   rV   rW   s   @r8   r  r    s      "!)+8<68""!)R)R )R 	)R
 )R )R )R BII)R %T"))_5)R c5c?23)R )R )R )RVr9   r  c                        e Zd Zdddddej                  ddddf
ded	ed
edededeeeeef   f   de	de
ej                     dee
ej                        def fdZd Z xZS )
LevitBlockru   r   r   r   FNro   r   r   r   r   r  r   r   rs   r  r  c                 Z   ||d}t         |           |	xs |}	t        d|||||||	d|| _        |
dkD  rt	        |
      nt        j                         | _        t        |t        ||z        f||d|| _
        |
dkD  rt	        |
      | _        y t        j                         | _        y )Nr'   )r   r   r   r   r   r   rs   ro   r  r  )r+   r,   r   r   r   r-   r  
drop_path1r
  rR   r  
drop_path2)r5   r   r   r   r   r  r   r   rs   r  r  r(   r)   r6   r7   s                 r8   r,   zLevitBlock.__init__!  s     /'49 	!!$	 		 2;R(9-R[[]i 
 	

 
 2;R(9-R[[]r9   c                     || j                  | j                  |            z   }|| j                  | j                  |            z   }|S rJ   )r   r   r!  r  rK   s     r8   rM   zLevitBlock.forwardI  s=    		!--,,r9   )rO   rP   rQ   r-   r   rR   rS   r
   r   rp   r	   r   r   r,   rM   rV   rW   s   @r8   r  r     s    
  "!68")+8<!&S&S &S 	&S
 &S &S c5c?23&S &S BII&S %T"))_5&S &SPr9   r  c                        e Zd Zddddej                  dddddddfd	ed
ededededededeej                     de	eej                        de
eeeef   f   dededef fdZd Z xZS )
LevitStagerw   ru   r   Nr    Fro   r   r   r   depthr   r   r  rs   r  r   
downsampler   r  c                    ||d}t         |           t        |
      }
|r8t        ||f|||z  dd||	|
||d	|| _        |
D cg c]  }|dz
  dz  dz    }
}n ||k(  sJ t        j                         | _        g }t        |      D ]  }|t        ||f|||||	|
||d|gz  } t        j                  | | _
        y c c}w )Nr'   r   r   )	r   r   r   r  rs   r  r   r   r  r   r<   )r   r   r  rs   r  r   r   r  )r+   r,   r   r  r'  r-   r  ranger  r   blocks)r5   r   r   r   r&  r   r   r  rs   r  r   r'  r   r  r(   r)   r6   rr*  _r7   s                       r8   r,   zLevitStage.__init__P  s   $ /z*
-   G+#-%!# DO 5??q1q5Q,*?J?W$$$ kkmDOu 	Az $%##-%!#   F	 mmV,) @s   B>c                 J    | j                  |      }| j                  |      }|S rJ   )r'  r*  rK   s     r8   rM   zLevitStage.forward  s"    OOAKKNr9   )rO   rP   rQ   r-   r   rR   rS   r	   r   r   r
   r   r   rp   r,   rM   rV   rW   s   @r8   r$  r$  O  s      #")+8<68 "!!9-9- 9- 	9-
 9- 9- 9- 9- BII9- %T"))_59- c5c?239- 9- 9- 9-vr9   r$  c            '       p    e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d0deeeeef   f   dededeedf   ded	eedf   d
eeeedf   f   deeeedf   f   deeeedf   f   dee	j                     dee   dedededee   dedededef& fdZd1defdZd1de	j                  deddfdZej"                  j$                  d        Zej"                  j$                  d2d       Zej"                  j$                  d1d       Zej"                  j$                  de	j                  fd        Zd3dedee   fd!Z	 	 	 	 	 d4d"ej0                  d#eeeee   f      d$ed%ed&ed'edeeej0                     eej0                  eej0                     f   f   fd(Z	 	 	 d5d#eeee   f   d)ed*efd+Zd, Zd2d-efd.Zd/ Z xZS )6r   z Vision Transformer with support for patch or hybrid CNN input stage

    NOTE: distillation is defaulted to True since pretrained weights use it, will cause problems
    w/ train scripts that don't take tuple outputs,
    Nimg_sizein_chansnum_classes	embed_dim.r   r&  r   r   r  stem_backbonestem_stride	stem_typedown_oprs   r  r   global_pool	drop_ratedrop_path_ratec                    t          |           ||d}t        |      }t        |xs |      }|| _        || _        || _        || _        |d   x| _        | _        || _	        || _
        d| _        g | _        t        |      }t        |      |k(  sJ  t        |      |      } t        |      |      } t        |      |	      }	|
|dk\  sJ |
| _        |}nP|dv sJ |dk(  rt!        ||d   fd|i|| _        nt#        ||d   fd|i|| _        | j                  j$                  }t'        t)        t+        |      t+        |            D cg c]
  \  }}||z   c}}      }|d   }g }t-        |      D ]  }|dkD  rdnd	}|t/        |||   |f||   ||   ||   |	|   |||||dk(  r|nd
|d
|gz  }||z  }t'        |D cg c]  }|d	z
  |z  d	z    c}      }| xj                  t1        ||   |d|       gz  c_        ||   } t3        j4                  | | _        |dkD  rt9        |d   |fd|i|nt3        j:                         | _        | j?                  d       y c c}}w c c}w )Nr'   rk   Fr<   )s16s8r;  r   rs   r   r%  )
r&  r   r   r  rs   r  r   r   r'  r  zstages.)num_chs	reductionmodulerg   needs_reset) r+   r,   r   r   r1  r0  r7  num_featureshead_hidden_sizer2  r8  grad_checkpointingfeature_infolenr   stemr   rr   r    tuplezipr   r)  r$  dictr-   r   stagesre   r  headinit_weights)!r5   r/  r0  r1  r2  r   r&  r   r   r  r3  r4  r5  r6  rs   r  r   r7  r8  r9  r(   r)   r6   
num_stagesr    ipr   r   rK  stage_strider+  r7   s!                                   r8   r,   zLevit.__init__  s   0 	/!),	&~'BC & &4=bMAD1"""'^
5zZ''')Ij))4	*Yz*:6
)Ij))4	$!###%DI F---E!"8Yq\UYURTU	!(IaLTITQST	YY%%Fs9X3F	RXHY/Z[tq!AF[\
1z" 	"A !A11Lz! Ah#A,%a=#A,#-%!&2a&77R(   F  l"FZPQ< 7! ;PQJ$y|vX_`a_bVc"d!eeq\F+	", mmV, U`bcTcJy}kP	PRPikititiv	 	e,A \.  Qs   I'
I-
rA  c                 P    | j                  t        | j                  |             y )Nr@  )applyr   _init_weights)r5   rA  s     r8   rM  zLevit.init_weights  s    

74--;GHr9   rG   r   c                 D    |rt        |d      r|j                          y y y )Nr   )hasattrr   )r5   rG   rA  s      r8   rT  zLevit._init_weights  s"    71&89  :;r9   c                 n    | j                         j                         D ch c]	  }d|v s| c}S c c}w )Nr   )
state_dictkeysrK   s     r8   no_weight_decayzLevit.no_weight_decay  s.    ??,113Oa7IQ7NOOOs   	22c                 $    t        dddg      }|S )Nz ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)i )rG  r*  )rJ  )r5   coarsematchers      r8   group_matcherzLevit.group_matcher  s    4-/CD
 r9   c                     || _         y rJ   )rD  r5   enables     r8   set_grad_checkpointingzLevit.set_grad_checkpointing  
    "(r9   c                     | j                   S rJ   )rL  r   s    r8   get_classifierzLevit.get_classifier  s    yyr9   c                     || _         ||| _        |dkD  r(t        | j                  || j                        | _        y t        j                         | _        y Nr   )rg   )r1  r7  re   rB  r8  r-   r  rL  r5   r1  r7  s      r8   reset_classifierzLevit.reset_classifier  sS    &"*DDORSO {A	Y[YdYdYf 		r9   rL   indicesnorm
stop_early
output_fmtintermediates_onlyc           	      
   |dv sJ d       g }t        t        | j                        |      \  }}	| j                  |      }|j                  \  }
}}}| j
                  s!|j                  d      j                  dd      }t        j                  j                         s|s| j                  }n| j                  d|	dz    }t        |      D ]  \  }}| j                  r+t        j                  j                         st        ||      }n ||      }||v rS| j
                  r|j                  |       n5|j                  |j                  |
||d      j!                  dddd             |dz   dz
  dz  }|dz   dz
  dz  } |r|S ||fS )	a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWzOutput shape must be NCHW.r<   r   Nrk   r   rx   )r   rF  rK  rG  rA   r   rb   r   rT   r   is_scripting	enumeraterD  r   appendr   r   )r5   rL   rj  rk  rl  rm  rn  intermediatestake_indices	max_indexr   r   r   r   rK  feat_idxstages                    r8   forward_intermediateszLevit.forward_intermediates  sr   * Y&D(DD&"6s4;;7G"Qi IIaLWW
1a}}		!&&q!,A99!!#:[[F[[)a-0F(0 	!OHe&&uyy/E/E/Gua(!H<'==!((+!((1aB)?)G)G1aQR)STQq AQq A	!   -r9   
prune_norm
prune_headc                     t        t        | j                        |      \  }}| j                  d|dz    | _        |r| j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r%  )r   rF  rK  ri  )r5   rj  rz  r{  ru  rv  s         r8   prune_intermediate_layerszLevit.prune_intermediate_layersH  sM     #7s4;;7G"Qikk.9q=1!!!R(r9   c                 (   | j                  |      }| j                  s!|j                  d      j                  dd      }| j                  r6t
        j                  j                         st        | j                  |      }|S | j                  |      }|S )Nr<   r   )
rG  r   rb   r   rD  rT   r   rq  r   rK  rK   s     r8   forward_featureszLevit.forward_featuresV  ss    IIaL}}		!&&q!,A""599+A+A+Ct{{A.A  AAr9   
pre_logitsc                     | j                   dk(  r0| j                  r|j                  d      n|j                  d      }|r|S | j                  |      S )Navgr   rk   r   r   )r7  r   meanrL  )r5   rL   r  s      r8   forward_headzLevit.forward_head`  sG    u$(,8$166a6=Aq0DIIaL0r9   c                 J    | j                  |      }| j                  |      }|S rJ   )r  r  rK   s     r8   rM   zLevit.forwarde  s'    !!!$a r9   )   rx     )   @   )   )rx   r   r   NNr;  	subsample
hard_swishNFr  ro   ro   NNr   FrJ   )NFFrp  F)r   FT) rO   rP   rQ   r  r
   rR   r   rS   r   r-   r   r   rp   r,   rM  rT  rT   r   ignorerZ  r^  rb  re  ri  r   r   ry  r}  r  r  rM   rV   rW   s   @r8   r   r     s?    58#)/%*59:<9;15)-"&),0"$!$&-W-CsCx01W- W- 	W-
 S#XW- W- c?W- S%S/12W- eU5#:%667W- UE%*$556W- $BII.W- "#W- W- W- W-  %SM!W-" #W-$ %W-& 'W-( ")W-rI I!ryy !t !t ! YYP P YY  YY) ) YY		  gC gx} g 8<$$',3 ||3  eCcN343  	3 
 3  3  !%3  
tELL!5tELL7I)I#JJ	K3 n ./$#	3S	>*  	1$ 1
r9   c                        e Zd Z fdZej
                  j                  dej                  fd       Z	d
de
dee   fdZej
                  j                  dd       Zddefd	Z xZS )LevitDistilledc                    t        |   |i | |j                  dd       |j                  dd       d}| j                  dkD  r!t	        | j
                  | j                  fi |nt        j                         | _        d| _	        y )Nr(   r)   r'   r   F)
r+   r,   getr1  re   rB  r-   r  	head_distdistilled_training)r5   argskwargsr6   r7   s       r8   r,   zLevitDistilled.__init__l  sw    $)&)

8T2VZZQU=VWRVRbRbefRfD$5$5t7G7GN2Nlnlwlwly"'r9   r   c                 2    | j                   | j                  fS rJ   )rL  r  r   s    r8   re  zLevitDistilled.get_classifierr  s    yy$..((r9   r1  r7  c                    || _         ||| _        |dkD  r"t        | j                  || j                        nt        j                         | _        |dkD  rt        | j                  |      | _        y t        j                         | _        y rg  )	r1  r7  re   rB  r8  r-   r  rL  r  rh  s      r8   ri  zLevitDistilled.reset_classifierv  sz    &"*DDORSO {AY[YdYdYf 		GRUVD$5$5{C\^\g\g\ir9   c                     || _         y rJ   )r  r`  s     r8   set_distilled_trainingz%LevitDistilled.set_distilled_training~  rc  r9   r  c                 P   | j                   dk(  r0| j                  r|j                  d      n|j                  d      }|r|S | j                  |      | j	                  |      }}| j
                  r.| j                  r"t        j                  j                         s||fS ||z   dz  S )Nr  r  r   r   r<   )
r7  r   r  rL  r  r  r   rT   r   rq  )r5   rL   r  x_dists       r8   r  zLevitDistilled.forward_head  s    u$(,8$166a6=AHIIaL$.."36""t}}UYY=S=S=Uf9 J!##r9   rJ   r   r  )rO   rP   rQ   r,   rT   r   r  r-   r   re  rR   r   r   ri  r  rp   r  rV   rW   s   @r8   r  r  k  su    ( YY)		 ) )jC jhsm j YY) )$$ $r9   r  c                 t    d| v r| d   } | j                         D ci c]  \  }}d|vs|| } }}| S c c}}w )Nmodelr   )items)rX  r  r   r   s       r8   checkpoint_filter_fnr    sQ    *(
 $.#3#3#5X41a9NVW9W!Q$XJX  Ys   44)        r   )rw      ru   )r<   rx   rw   )r2  r   r   r&  )rw   ru   r  )rw   rw   rw   )r  i   r      )rx      r  )r  r     )r  r     )r  	   r  silur<  )r2  r   r   r&  rs   r5  )r    i  r  )ru   
   r   )r  r  i   )ru   r  r   )r2  r   r   r&  rs   )rw   ru   r  )r  r  r  )ru   r  r  )

levit_128s	levit_128	levit_192	levit_256	levit_384levit_384_s8levit_512_s8	levit_512
levit_256d
levit_512dc                 >   d| v }|j                  dd      }|j                  dd      r|s|j                  dd       || t        v r| }n|r| j	                  dd      }t        t        |   fi |}t        |rt        nt        | |ft        t        d	|
      d|}|S )N_convout_indices)r   r   r<   features_onlyFfeature_clsgetterr%  T)flatten_sequentialr  )pretrained_filter_fnfeature_cfg)
popr  
setdefault
model_cfgsreplacerJ  r   r  r   r  )	variantcfg_variant
pretrained	distilledr  is_convr  	model_cfgr  s	            r8   create_levitr    s     G**]I6Kzz/5)'-2j !K!//'26KZ,77I # 2DkJ E Lr9   c                 4    | ddd dddt         t        dddd	|S )
Nr  )rx   r  r  g?bicubicTzstem.conv1.linear)head.linearzhead_dist.linearz
apache-2.0)urlr1  
input_size	pool_sizecrop_pctinterpolationfixed_input_sizer  rf   
first_conv
classifierlicense)r   r   )r  r  s     r8   _cfgr    s6    =t%.B)9\  r9   zlevit_128s.fb_dist_in1kztimm/)	hf_hub_idzlevit_128.fb_dist_in1kzlevit_192.fb_dist_in1kzlevit_256.fb_dist_in1kzlevit_384.fb_dist_in1kzlevit_conv_128s.fb_dist_in1k)rw   rw   )r  r  zlevit_conv_128.fb_dist_in1kzlevit_conv_192.fb_dist_in1kzlevit_conv_256.fb_dist_in1kzlevit_conv_384.fb_dist_in1kzlevit_384_s8.untrainedr  )r  zlevit_512_s8.untrainedzlevit_512.untrainedzlevit_256d.untrainedzlevit_512d.untrainedzlevit_conv_384_s8.untrainedzlevit_conv_512_s8.untrained)zlevit_conv_512.untrainedzlevit_conv_256d.untrainedzlevit_conv_512d.untrainedr   c                     t        dd| i|S )Nr  )r  r  r  r  s     r8   r  r    s    FFvFFr9   c                     t        dd| i|S )Nr  )r  r  r  s     r8   r  r  #      E
EfEEr9   c                     t        dd| i|S )Nr  )r  r  r  s     r8   r  r  (  r  r9   c                     t        dd| i|S )Nr  )r  r  r  s     r8   r  r  -  r  r9   c                     t        dd| i|S )Nr  )r  r  r  s     r8   r  r  2  r  r9   c                     t        dd| i|S )Nr  )r  r  r  s     r8   r  r  7  s    H:HHHr9   c                     t        d| dd|S )NFr  r  )r  r  r  s     r8   r  r  <  s    Y:YRXYYr9   c                     t        d| dd|S )NFr  )r  r  r  s     r8   r  r  A  s    V
eVvVVr9   c                     t        d| dd|S )NFr  )r  r  r  s     r8   r  r  F      WuWPVWWr9   c                     t        d| dd|S )NFr  )r  r  r  s     r8   r  r  K  r  r9   c                     t        d| dd|S )NTr  r   )levit_conv_128sr  r  s     r8   r  r  P  s    Zj4ZSYZZr9   c                     t        d| dd|S )NTr  )levit_conv_128r  r  s     r8   r  r  U      YZ$YRXYYr9   c                     t        d| dd|S )NTr  )levit_conv_192r  r  s     r8   r  r  Z  r  r9   c                     t        d| dd|S )NTr  )levit_conv_256r  r  s     r8   r  r  _  r  r9   c                     t        d| dd|S )NTr  )levit_conv_384r  r  s     r8   r  r  d  r  r9   c                     t        d| dd|S )NTr  )levit_conv_384_s8r  r  s     r8   r  r  i  s    \
T\U[\\r9   c                      t        d| ddd|S )NTFr  r   r  )levit_conv_512_s8r  r  s     r8   r  r  n  s    m
T]bmflmmr9   c                      t        d| ddd|S )NTFr  )levit_conv_512r  r  s     r8   r  r  s  s    jZ$Z_jcijjr9   c                      t        d| ddd|S )NTFr  )levit_conv_256dr  r  s     r8   r  r  x      kj4[`kdjkkr9   c                      t        d| ddd|S )NTFr  )levit_conv_512dr  r  s     r8   r   r   }  r  r9   )NFT)r%  r  )Nr  collectionsr   	functoolsr   typingr   r   r   r   r	   r
   rT   torch.nnr-   	timm.datar   r   timm.layersr   r   r   r   r   r   _builderr   	_featuresr   _manipulater   r   	_registryr   r   __all__r   r   rY   re   r   rr   r   r   r   r   r
  r  r  r$  r   r  r  rJ  r  r  r  default_cfgsr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r9   r8   <module>r     s  2 $  ; ;   A \ \ * + 3 <)#'ryy #'L6 6@%2 %2P`BMM `(`R]] `,# #0l		 l^@")) @Fryy B/bii /d, ,^? ?DWBII Wt"$U "$J* !2)U!29V!2)U!2)U!29V !29D* !2ID*
 "B+Ybhj !2)_eg!2Iagi7
@.	 % 3&t 3&
 d3& d3& d3& d3&& #D%'3&. "4$/3&6 "4$73&> "4$?3&F "4$G3&P dm<Q3&R dm<S3&T 4=9U3&V DM:W3&X DM:Y3&\ "4=#A]3&^ "4=#A_3&` !% >!%!?!%!?e3& 3l Ge G G FU F F FU F F FU F F FU F F I I I Z Z Z WU W W Xe X X Xe X X [5 [ [ Z% Z Z Z% Z Z Z% Z Z Z% Z Z ]U ] ] nU n n k% k k l5 l l l5 l lr9   