
    ^j                     t   d Z ddlZddlmZ ddlmZ ddlmZmZ ddl	m
Z
mZmZmZmZmZ ddlZddlmZ ddlmZmZ dd	lmZmZmZmZmZmZ d
dlmZ d
dlmZ d
dl m!Z! d
dl"m#Z# d
dl$m%Z%m&Z& ddgZ'e G d d             Z(d Z) G d dejT                        Z+e!	 dVdee,   de-deej\                  eej\                     f   fd       Z/e!	 dWde,deej\                     deej\                  ee,   f   fd       Z0e!dej\                  dej\                  de-d ee,   d!ee,   d"ej\                  d#ej\                  fd$       Z1 G d% d&ejT                        Z2 G d' d(ejT                        Z3 G d) d*ejT                        Z4 G d+ d,ejT                        Z5 G d- dejT                        Z6d. Z7 e8 e(d/0       e(d10       e(d20       e(d3d4d5d67       e(d1d8       e(d2d8       e(d3d4d5dd9       e(d:d;d<dd9      =      Z9dXd>Z:dYd@Z; e& e;dAdBC       e;dDdBC       e;dEdBC       e;dFdBC       e;d?G       e;dHdBdIJ       e;dKdBdIJ       e;dLdBdIJ      dM      Z<e%dZde6fdN       Z=e%dZde6fdO       Z>e%dZde6fdP       Z?e%dZde6fdQ       Z@e%dZde6fdR       ZAe%dZde6fdS       ZBe%dZde6fdT       ZCe%dZde6fdU       ZDy)[a   Multi-Scale Vision Transformer v2

@inproceedings{li2021improved,
  title={MViTv2: Improved multiscale vision transformers for classification and detection},
  author={Li, Yanghao and Wu, Chao-Yuan and Fan, Haoqi and Mangalam, Karttikeya and Xiong, Bo and Malik, Jitendra and Feichtenhofer, Christoph},
  booktitle={CVPR},
  year={2022}
}

Code adapted from original Apache 2.0 licensed impl at https://github.com/facebookresearch/mvit
Original copyright below.

Modifications and timm support by / Copyright 2022, Ross Wightman
    N)OrderedDict)	dataclass)partialreduce)UnionListTupleOptionalAnyType)nnIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)MlpDropPathcalculate_drop_path_ratestrunc_normal_tf_get_norm_layer	to_2tuple   )build_model_with_cfg)feature_take_indices)register_notrace_function)
checkpoint)register_modelgenerate_default_cfgsMultiScaleVitMultiScaleVitCfgc                      e Zd ZU dZeedf   ed<   dZeeeedf   f   ed<   dZ	eeeedf   f   ed<   dZ
eed	<   d
Zeed<   dZeed<   dZeed<   d
Zeed<   d
Zeed<   dZeed<   dZeed<   dZeeef   ed<   dZeeeeef         ed<   dZeeeeef         ed<   dZeeeef      ed<   dZeeef   ed<   dZeeef   ed<   dZeeef   ed<   d Zeed!<   d"Zeed#<   d$Zeeeeef   f   ed%<   d&Z eeeeef   f   ed'<   d(Z!eed)<   d* Z"y)+r            r#   .depths`   	embed_dimr   	num_heads      @	mlp_ratioF
pool_firstTexpand_attnqkv_biasuse_cls_tokenuse_abs_posresidual_poolingconvmoder#   r#   
kernel_qkv)r   r   r"   r"   r6   r6   stride_qN	stride_kv   r:   stride_kv_adaptive   r=   patch_kernelpatch_stridepatch_paddingmax	pool_typespatialrel_pos_typegelu	act_layer	layernorm
norm_layergư>norm_epsc           	      &    t         j                        }t         j                  t        t
        f      s"t	         fdt        |      D               _        t         j                        |k(  sJ t         j                  t        t
        f      s"t	         fdt        |      D               _        t         j                        |k(  sJ  j                   j                   j                  }g }t        |      D ]x  }t         j                  |         dkD  rAt        t        |            D cg c]$  }t        ||    j                  |   |   z  d      & }}|j                  t	        |             z t	        |       _	        y y y c c}w )Nc              3   B   K   | ]  }j                   d |z  z    ywr"   N)r'   .0iselfs     ]/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/mvitv2.py	<genexpr>z1MultiScaleVitCfg.__post_init__.<locals>.<genexpr>A        "Vq4>>AF#:"V   c              3   B   K   | ]  }j                   d |z  z    ywrL   )r(   rM   s     rQ   rR   z1MultiScaleVitCfg.__post_init__.<locals>.<genexpr>E   rS   rT   r   )lenr%   
isinstancer'   tuplelistranger(   r;   r8   minr7   rA   append)rP   
num_stages
_stride_kvpool_kv_striderO   ds   `     rQ   __post_init__zMultiScaleVitCfg.__post_init__>   sW   %
$..5$-8""VE*DU"VVDN4>>"j000$..5$-8""VE*DU"VVDN4>>"j000"".4>>3I00JN:& 9t}}Q'(1, "'s:!7" JqMT]]1-=a-@@!D"J " %%eJ&789 #>2DN 4J.
"s   4)F)#__name__
__module____qualname__r%   r	   int__annotations__r'   r   r(   r*   floatr+   boolr,   r-   r.   r/   r0   r2   strr4   r7   r
   r8   r;   r>   r?   r@   rB   rD   rF   rH   rI   ra        rQ   r   r   $   s   +FE#s(O+-/IuS%S/)*/-.IuS%S/)*.IuJKHdM4K!d!D#"(Jc3h(1QHhuU38_-.Q26IxeCHo./64:sCx1:$*L%S/*$*L%S/*%+M5c?+Is!L#!-3IuS%S/)*3.9Jc5c?*+9He3rk   c                 8    t        t        j                  | d      S )Nr   )r   operatormul)iterables    rQ   prodrp   U   s    (,,!,,rk   c                        e Zd ZdZ	 	 	 	 	 	 	 d
dededeeef   deeef   deeef   f
 fdZdeej                  e	e   f   fd	Z
 xZS )
PatchEmbedz
    PatchEmbed.
    dim_indim_outkernelstridepaddingc                 l    t         	|           ||d}t        j                  ||f|||d|| _        y )Ndevicedtype)kernel_sizerv   rw   )super__init__r   Conv2dproj)
rP   rs   rt   ru   rv   rw   rz   r{   dd	__class__s
            rQ   r~   zPatchEmbed.__init__^   sJ     	/II
 
 
	rk   returnc                     | j                  |      }|j                  d      j                  dd      |j                  dd  fS )Nr"   r   )r   flatten	transposeshaperP   xs     rQ   forwardzPatchEmbed.forwardt   s9    IIaLyy|%%a+QWWRS\99rk   )r#   i   r<   r9   r3   NN)rb   rc   rd   __doc__re   r	   r~   torchTensorr   r   __classcell__r   s   @rQ   rr   rr   Y   s     &,&,'-

 
 #s(O	

 #s(O
 38_
,:E%,,S	"9: :rk   rr   T	feat_sizehas_cls_tokenr   c                     |\  }}|r#| d d d d d dd d f   | d d d d dd d d f   } }nd }| j                  d||| j                  d         j                  dddd      j                         } | |fS )Nr   r   r#   r"   )reshaper   permute
contiguous)r   r   r   HWcls_toks         rQ   reshape_pre_poolr   z   s|     DAqq!RaR{^Qq!QR{^			"aAGGBK(00Aq!<GGIAg:rk   r(   r   c                    | j                   d   | j                   d   g}| j                   d   | j                   d   z  }| j                  d|| j                   d   |      j                  dd      } |t        j                  || fd      } | |fS )Nr"   r#   r   r   dim)r   r   r   r   cat)r   r(   r   r   L_pooleds        rQ   reshape_post_poolr      s     QWWQZ(IwwqzAGGAJ&H			"iX6@@AFAIIwl*i<rk   attnqq_sizek_size	rel_pos_h	rel_pos_wc                    |rdnd}|\  }}	|\  }
}t        |
|z  d      }t        ||
z  d      }t        j                  ||j                  t        j                        j                  d      |z  t        j                  |
|j                  t        j                        j                  d      |z  z
  }||
dz
  |z  z  }t        ||	z  d      }t        |	|z  d      }t        j                  |	|j                  t        j                        j                  d      |z  t        j                  ||j                  t        j                        j                  d      |z  z
  }||dz
  |z  z  }||j	                            }||j	                            }|j                  \  }}}}|dddd|df   j                  ||||	|      }t        j                  d||      }t        j                  d||      }| dddd|d|df   j                  |d||	|
|      |j                  d      z   |j                  d	      z   j                  |d||	z  |
|z        | dddd|d|df<   | S )
z1
    Spatial Relative Positional Embeddings.
    r   r   g      ?ry   r   Nzbyhwc,hkc->byhwkzbyhwc,wkc->byhwkr   )
rA   r   arangerz   long	unsqueezer   r   einsumview)r   r   r   r   r   r   r   sp_idxq_hq_wk_hk_w	q_h_ratio	k_h_ratiodist_h	q_w_ratio	k_w_ratiodist_wrel_hrel_wBn_headq_Nr   r_qs                            rQ   cal_rel_pos_typer      sD     QQFHCHC C#Is#IC#Is#IS<FFrJYVS<FFqIIU	V  sQw)##FC#Is#IC#Is#IS<FFrJYVS<FFqIIU	V  sQw)##Ffkkm$Efkkm$E''AvsC
Aq&'M

"
"1fc3
<CLL+S%8ELL+S%8E 	Q67FG#$))!Rc3D
//"
	
//"
	 d1b#)S3Y'	 	Avw	  Krk   c                        e Zd Zddddddddddej                  ddfdeded	eeef   d
edededeeef   deeef   deeef   deeef   dededede	ej                     f fdZd	ee   fdZ xZS )MultiScaleAttentionPoolFirst   Tr1   r5   rC   Nr   rt   r   r(   r-   r2   kernel_q	kernel_kvr7   r8   r   rD   r0   rH   c           	      ^   ||d}t         |           || _        || _        ||z  | _        | j                  dz  | _        || _        t        |D cg c]  }t        |dz         c}      }t        |D cg c]  }t        |dz         c}      }t        j                  ||fd|i|| _        t        j                  ||fd|i|| _        t        j                  ||fd|i|| _        t        j                  ||fi || _        t        |      dk(  rt        |	      dk(  rd }t        |      dk(  rt        |
      dk(  rd }|| _        |dk(  | _        d\  | _        | _        | _        d\  | _        | _        | _        |dv rW|d	k(  rt        j0                  nt        j2                  }|r |||	|      | _        |r |||
|      | _         |||
|      | _        n|d
k(  s|dk(  r|d
k(  r||z  n|}|r0t        j4                  |||f|	||dd|| _         ||fi || _        |rot        j4                  |||f|
||dd|| _         ||fi || _        t        j4                  |||f|
||dd|| _         ||fi || _        nt7        d|       || _        | j8                  dk(  r|d   |d   k(  sJ |d   }t;        |	      dkD  r||	d   z  n|}t;        |
      dkD  r||
d   z  n|}dt=        ||      z  dz
  }t        j>                  tA        jB                  || j                  fi |      | _"        t        j>                  tA        jB                  || j                  fi |      | _#        tI        | jD                  d       tI        | jF                  d       || _%        y c c}w c c}w )Nry         r"   biasr   conv_unsharedNNNavgrA   rA   r1   Frv   rw   groupsr   Unsupported model rC   r   {Gz?std)&r}   r~   r(   rt   head_dimscaler   rX   re   r   Linearr   kvr   rp   r2   unsharedpool_qpool_kpool_vnorm_qnorm_knorm_v	MaxPool2d	AvgPool2dr   NotImplementedErrorrD   rV   rA   	Parameterr   zerosr   r   r   r0   rP   r   rt   r   r(   r-   r2   r   r   r7   r8   r   rD   r0   rH   rz   r{   r   r   	padding_qkv
padding_kvpool_opdim_convsizer   kv_size
rel_sp_dimr   s                               rQ   r~   z%MultiScaleAttentionPoolFirst.__init__   s   & /"9,]]d*
*913qAv;9:	9=RCaL=>
3=h="=3=h="=3=h="=IIgw5"5	 >Q4>Q#6H	?aDOq$8I	/0@-T[$+0@-T[$+>!&*embllG%h)D%iJG%iJGV^t6+/6>si'sH ii	 $%#	 	 )8R8 ii	 %&#	 	 )8R8 ii	 %&#	 	 )8R8%(:4&&ABB )	)Q<9Q<///Q<D,/MA,=TXa[(4F.1)nq.@dil*dGS11A5J\\%++j$--*VSU*VWDN\\%++j$--*VSU*VWDNT^^6T^^6 0] :=s   N%6N*c           	         |j                   \  }}}| j                  rdn| j                  }|j                  |||d      j	                  dddd      }|x}x}}	| j
                  Ft        ||| j                        \  }}
| j                  |      }t        || j                  |
      \  }}n|}| j                  | j                  |      }| j                  Ft        ||| j                        \  }}| j                  |      }t        || j                  |      \  }}n|}| j                  | j                  |      }| j                  Ft        |	|| j                        \  }	}| j                  |	      }	t        |	| j                  |      \  }	}n|}| j                  | j                  |	      }	|d   |d   z  t        | j                        z   }|j                  dd      j                  ||d      }| j!                  |      j                  ||| j                  d      j                  dd      }|d   |d   z  t        | j                        z   }|j                  dd      j                  ||d      }| j#                  |      j                  ||| j                  d      }|d   |d   z  t        | j                        z   }|	j                  dd      j                  ||d      }	| j%                  |	      j                  ||| j                  d      j                  dd      }	|| j&                  z  |z  }| j(                  dk(  r/t+        ||| j                  ||| j,                  | j.                        }|j1                  d      }||	z  }| j2                  r||z   }|j                  dd      j                  |d| j4                        }| j7                  |      }||fS )Nr   r   r   r"   r#   rC   r   )r   r   r(   r   r   r   r   r   r   r   r   r   r   r   re   r   r   r   r   r   rD   r   r   r   softmaxr0   rt   r   )rP   r   r   r   N_fold_dimr   r   r   q_tokr   k_tokr   v_tokv_sizer   k_Nv_Nr   s                       rQ   r   z$MultiScaleAttentionPoolFirst.forward5  sn   ''1a14>>IIaHb)11!Q1=A;;"'9d6H6HIHAuAA)!T^^UCIAvF;;"AA;;"'9d6H6HIHAuAA)!T^^UCIAvF;;"AA;;"'9d6H6HIHAuAA)!T^^UCIAvF;;"AAQi&)#c$*<*<&==KK1%%ab1FF1Iadnnb9CCAqIQi&)#c$*<*<&==KK1%%ab1FF1Iadnnb9Qi&)#c$*<*<&==KK1%%ab1FF1Iadnnb9CCAqIDJJ!#	)#""D |||#1H  AAKK1%%aT\\:IIaL&yrk   rb   rc   rd   r   	LayerNormre   r	   rh   ri   r   Moduler~   r   r   r   r   s   @rQ   r   r      s    !(.)/(.)/"& )%)*,,,#h1h1 h1 S#X	h1
 h1 h1 h1 CHoh1 S#Xh1 CHoh1 S#Xh1  h1 h1 #h1 RYYh1TBDI Brk   r   c                        e Zd Zddddddddddej                  ddfdeded	eeef   d
edededeeef   deeef   deeef   deeef   dededede	ej                     f fdZd	ee   fdZ xZS )MultiScaleAttentionr   Tr1   r5   rC   Nr   rt   r   r(   r-   r2   r   r   r7   r8   r   rD   r0   rH   c           	         ||d}t         |           || _        || _        ||z  | _        | j                  dz  | _        || _        t        |D cg c]  }t        |dz         c}      }t        |D cg c]  }t        |dz         c}      }t        j                  ||dz  fd|i|| _        t        j                  ||fi || _        t        |      dk(  rt        |	      dk(  rd }t        |      dk(  rt        |
      dk(  rd }|| _        |dk(  | _        d\  | _        | _        | _        d\  | _        | _        | _        |d	v rW|d
k(  rt        j,                  nt        j.                  }|r |||	|      | _        |r |||
|      | _         |||
|      | _        n|dk(  s|dk(  r|dk(  r||z  n|}|r0t        j0                  |||f|	||dd|| _         ||fi || _        |rot        j0                  |||f|
||dd|| _         ||fi || _        t        j0                  |||f|
||dd|| _         ||fi || _        nt3        d|       || _        | j4                  dk(  r|d   |d   k(  sJ |d   }t7        |	      dkD  r||	d   z  n|}t7        |
      dkD  r||
d   z  n|}dt9        ||      z  dz
  }t        j:                  t=        j>                  || j                  fi |      | _         t        j:                  t=        j>                  || j                  fi |      | _!        tE        | j@                  d       tE        | jB                  d       || _#        y c c}w c c}w )Nry   r   r"   r#   r   r   r   r   r   rA   r1   Fr   r   rC   r   r   r   )$r}   r~   r(   rt   r   r   r   rX   re   r   r   qkvr   rp   r2   r   r   r   r   r   r   r   r   r   r   r   rD   rV   rA   r   r   r   r   r   r   r0   r   s                               rQ   r~   zMultiScaleAttention.__init__{  s   & /"9,]]d*
*913qAv;9:	9=RCaL=>
99S'A+CHCCIIgw5"5	 >Q4>Q#6H	?aDOq$8I	/0@-T[$+0@-T[$+>!&*embllG%h)D%iJG%iJGV^t6/3v~w)+7H ii	 $%#	 	 )8R8 ii	 %&#	 	 )8R8 ii	 %&#	 	 )8R8%(:4&&ABB )	)Q<9Q<///Q<D,/MA,=TXa[(4F.1)nq.@dil*dGS11A5J\\%++j$--*VSU*VWDN\\%++j$--*VSU*VWDNT^^6T^^6 0Y :=s   M,6M1c           	         |j                   \  }}}| j                  |      j                  ||d| j                  d      j	                  ddddd      }|j                  d      \  }}}	| j                  Ft        ||| j                        \  }}
| j                  |      }t        || j                  |
      \  }}n|}| j                  | j                  |      }| j                  Ft        ||| j                        \  }}| j                  |      }t        || j                  |      \  }}n|}| j                  | j                  |      }| j                  Et        |	|| j                        \  }	}| j                  |	      }	t        |	| j                  |      \  }	}| j                  | j                  |	      }	|| j                  z  |j!                  dd      z  }| j"                  d	k(  r/t%        ||| j                  ||| j&                  | j(                        }|j+                  d      }||	z  }| j,                  r||z   }|j!                  dd      j                  |d| j.                        }| j1                  |      }||fS )
Nr#   r   r"   r   r   r:   r   r   rC   )r   r   r   r(   r   unbindr   r   r   r   r   r   r   r   r   r   r   rD   r   r   r   r   r0   rt   r   )rP   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   s                   rQ   r   zMultiScaleAttention.forward  s=   ''1ahhqk!!!Q4>>2>FFq!QPQSTU***#1a;;"'9d6H6HIHAuAA)!T^^UCIAvF;;"AA;;"'9d6H6HIHAuAA)!T^^UCIAvF;;"AA;;"'9d6H6HIHAuAA$Q>DAq;;"AADJJ!++b""55	)#""D |||#1H  AAKK1%%aT\\:IIaL&yrk   r   r   s   @rQ   r   r   z  s    !(.)/(.)/"& )%)*,,,#f1f1 f1 S#X	f1
 f1 f1 f1 CHof1 S#Xf1 CHof1 S#Xf1  f1 f1 #f1 RYYf1P3DI 3rk   r   c            %           e Zd Zdddej                  ddddddddddddfd	ed
ededeeef   dededede	ej                     deeef   deeef   deeef   deeef   dedededededef$ fdZdee   fdZdee   fdZ xZS )MultiScaleBlockr)   T        r5   r1   FrC   Nr   rt   r(   r   r*   r-   	drop_pathrH   r   r   r7   r8   r2   r   r,   r+   rD   r0   c                 \   ||d}t         |           ||k7  }|| _        || _        || _         ||fi || _        |r|rt        j                  ||fi |nd | _        |rat        |      dkD  rS|D cg c]  }|dkD  r|dz   n| }}|}|D cg c]  }t        |dz         }}t        j                  |||      | _        nd | _        |r|n|}|rt        nt        } |||f||||	|
|||||||d|| _        |dkD  rt!        |      nt        j"                         | _         ||fi || _        |}|r|st        j                  ||fi |nd | _        t+        d|t        ||z        |d|| _        |dkD  rt!        |      | _        y t        j"                         | _        y c c}w c c}w )Nry   r   r"   )r(   r   r-   r   r   r7   r8   rH   r   r2   rD   r0   r  )in_featureshidden_featuresout_featuresrj   )r}   r~   r   rt   r   norm1r   r   shortcut_proj_attnrp   re   r   shortcut_pool_attnr   r   r   r   Identity
drop_path1norm2shortcut_proj_mlpr   mlp
drop_path2) rP   r   rt   r(   r   r*   r-   r  rH   r   r   r7   r8   r2   r   r,   r+   rD   r0   rz   r{   r   proj_neededskernel_skipstride_skipskippadding_skipatt_dim
attn_layermlp_dim_outr   s                                   rQ   r~   zMultiScaleBlock.__init__  s   . /Wn**r*
CNS^"))C"?B"?dhX*:BCQAE1q5q0CKC"K7BCtC	NCLC&(ll;\&ZD#&*D#('c5?1EX

  !'%-
 
	" 2;S(9-bkkm.2.
BMVa3!>2!>gk 
) 34$
 	
 2;S(9-bkkmK DCs   3F$F)c                    | j                   |S | j                  r|d d d dd d f   |d d dd d d f   }}nd }|j                  \  }}}|\  }}|j                  ||||      j	                  dddd      j                         }| j                  |      }|j                  ||d      j                  dd      }|t        j                  ||fd      }|S )Nr   r   r#   r"   r   r   )	r
  r   r   r   r   r   r   r   r   )	rP   r   r   r   r   LCr   r   s	            rQ   _shortcut_poolzMultiScaleBlock._shortcut_poolc  s    ""*H1bqb!8a12qkQGG''1a1IIaAq!))!Q15@@B##A&IIaB))!Q/		7A,A.Ark   c                    | j                  |      }| j                  |n| j                  |      }| j                  ||      }| j                  ||      \  }}|| j	                  |      z   }| j                  |      }| j                  |n| j                  |      }|| j                  | j                  |            z   }||fS N)	r  r	  r  r   r  r  r  r  r  )rP   r   r   x_norm
x_shortcutfeat_size_news         rQ   r   zMultiScaleBlock.forwards  s    A119Qt?V?VW]?^
((Y?
99VY7=++A008Qd>T>TU[>\
&)9::-rk   )rb   rc   rd   r   r   re   r	   rg   rh   r   r   ri   r~   r   r  r   r   r   s   @rQ   r  r    sg     #!"*,,,(.)/(.)/"& %$ )%)+GTGT GT 	GT
 S#XGT GT GT GT RYYGT CHoGT S#XGT CHoGT S#XGT GT  GT  !GT" #GT$ %GT& #'GTR49   DI  rk   r  c            *           e Zd Zddddddddddddej                  dddfd	ed
edededeeef   dededeeef   deeef   deeef   deeef   de	dededede	dede
ej                     deeee   f   f& fdZdee   fdZ xZS )MultiScaleVitStager)   Tr5   r1   FrC   r  Nr   rt   depthr(   r   r*   r-   r   r   r7   r8   r2   r   r,   r+   rD   r0   rH   r  c                 @   ||d}t         |           d| _        t        j                         | _        |r|f|z  }n|f|dz
  z  |fz   }t        |      D ]  }t        di d|d||   d|d|d|d	|d
|d|	d|dk(  r|
ndd|d|d|d|d|d|d|d|dt        |t        t        f      r||   n||}||   }| j
                  j                  |       |dk(  st        t        ||
      D cg c]
  \  }}||z   c}}      } || _        y c c}}w )Nry   Fr   r   rt   r(   r   r*   r-   r   r   r7   r   r5   r8   r2   r   r+   rD   r0   r,   rH   r  rj   )r}   r~   grad_checkpointingr   
ModuleListblocksrZ   r  rW   rY   rX   r\   zipr   )rP   r   rt   r%  r(   r   r*   r-   r   r   r7   r8   r2   r   r,   r+   rD   r0   rH   r  rz   r{   r   out_dimsrO   attention_blockr   rv   r   s                               rQ   r~   zMultiScaleVitStage.__init__  s   0 /"'mmozE)Hv+wj8Hu 	aA-   $ $	
 $ " " $ &'!V $  , & * "2  (!" &#$ +5Yu*N)A,T]'O* 1+CKK/Av!c)U]F^"_ldF46>"_`	3	a6 # #`s   9Dc                     | j                   D ]I  }| j                  r/t        j                  j	                         st        |||      \  }}> |||      \  }}K ||fS r  )r)  r'  r   jitis_scriptingr   )rP   r   r   blks       rQ   r   zMultiScaleVitStage.forward  s[    ;; 	1C&&uyy/E/E/G)#q)<9"1i09		1
 )|rk   )rb   rc   rd   r   r   re   r	   rg   rh   ri   r   r   r   r   r~   r   r   r   s   @rQ   r$  r$    sI     #!(.)/(.)/"& %$ )%)*,,,36-=#=# =# 	=#
 =# S#X=# =# =# CHo=# S#X=# CHo=# S#X=# =#  =# =#  !=#" #=#$ #%=#& RYY'=#( UDK/0)=#~DI rk   r$  c                   z    e Zd ZdZ	 	 	 	 	 	 	 	 ddedeeef   dedee   dede	de	f fd	Z
d
 Zej                  j                  d        Zej                  j                  d d       Zej                  j                  d!d       Zej                  j                  dej&                  fd       Zd"dedee   fdZ	 	 	 	 	 d#dej,                  deeeee   f      dededededeeej,                     eej,                  eej,                     f   f   fdZ	 	 	 d$deeee   f   dedefdZd Zd defdZd Z xZS )%r   a  
    Improved Multiscale Vision Transformers for Classification and Detection
    Yanghao Li*, Chao-Yuan Wu*, Haoqi Fan, Karttikeya Mangalam, Bo Xiong, Jitendra Malik,
        Christoph Feichtenhofer*
    https://arxiv.org/abs/2112.01526

    Multiscale Vision Transformers
    Haoqi Fan*, Bo Xiong*, Karttikeya Mangalam*, Yanghao Li*, Zhicheng Yan, Jitendra Malik,
        Christoph Feichtenhofer*
    https://arxiv.org/abs/2104.11227
    cfgimg_sizein_chansglobal_poolnum_classesdrop_path_rate	drop_ratec
                 Z	   t         |           ||	d}
t        |      }t        t	        |j
                        |j                        }|| _        || _        || _	        ||j                  rdnd}|| _        t        |j                        | _        |j                  | _        |j                  d   }t!        d#|||j"                  |j$                  |j&                  d|
| _        |d   |j$                  d   z  |d   |j$                  d   z  f}t+        |      }|j                  r=t-        j.                  t1        j2                  dd|fi |
      | _        d| _        |dz   }nd| _        d | _        |}|j8                  r1t-        j.                  t1        j2                  d||fi |
      | _        nd | _        t=        |j                        }|}t?        |j$                        }tA        ||j                  d	      }t-        jB                         | _"        g | _#        tI        |      D ]  }|j                  r|j                  |   }n|j                  tK        |dz   |dz
           }tM        d#i d
|d|d|j                  |   d|jN                  |   d|d|jP                  d|jR                  d|jT                  d|jV                  d|j                  d|jX                  d|jX                  d|jZ                  |   d|j\                  |   d|j                  d|j^                  d|j`                  d|d||   |
}|t?        |jZ                  |         z  }| xjF                  tc        d| ||      gz  c_#        |}|jd                  }| jD                  jg                  |        |x| _4        | _5         ||fi |
| _6        t-        jn                  tq        dt-        jr                  | j                        fd |dkD  r!t-        jt                  | jh                  |fi |
nt-        jv                         fg            | _<        | j:                  t{        | j:                  d!"       | j4                  t{        | j4                  d!"       | j}                  | j~                         y )$Nry   )epstokenr   r   )rs   rt   ru   rv   rw   r   T)	stagewiser   rt   r%  r(   r   r*   r-   r2   r+   r,   r   r   r7   r8   r   rD   r0   rH   r  zblock.)modulenum_chs	reductiondropfcr   r   rj   )@r}   r~   r   r   r   rH   rI   r6  r4  r8  r.   r5  rX   r%   r,   r'   rr   r>   r?   r@   patch_embedrp   r   r   r   r   	cls_tokennum_prefix_tokensr/   	pos_embedrV   rA   r   r(  stagesfeature_inforZ   r[   r$  r(   r*   r-   r2   r+   r4   r7   r8   rD   r0   dictr   r\   num_featureshead_hidden_sizenorm
Sequentialr   Dropoutr   r  headr   apply_init_weights)rP   r2  r3  r4  r5  r6  r7  r8  rz   r{   r   rH   r'   
patch_dimsnum_patchespos_embed_dimr]   r   curr_stridedprrO   rt   stager   s                          rQ   r~   zMultiScaleVit.__init__  si    	/X&^CNN;N
& "%(%6%6'EK&CJJ'??MM!$	% 
####%%
 
 qkS%5%5a%88(1+IYIYZ[I\:\]
:&\\%++aI*L*LMDN%&D"'!OM%&D"!DN'M??\\%++a	*XUW*XYDN!DN'
	#**+'

dSmmoz" 	&A--*--AE:>(BC&  jjm --*	
 $ --  XX >>  OO  .. a --* "//  !--!" "%!5!5#$ &%& a&)E, 3s||A//K$qclGWb"c!ddIIKKu%?	&B 5>=D1y/B/	MM+RZZ/0kTUo299T..BrB[][f[f[hi/
 # 	
 >>%T^^6>>%T^^6

4%%&rk   c                    t        |t        j                        rjt        |j                  d       t        |t        j                        r8|j
                  +t        j                  j                  |j
                  d       y y y y )Nr   r   r  )rW   r   r   r   weightr   init	constant_)rP   ms     rQ   rP  zMultiScaleVit._init_weights?  sZ    a#QXX40!RYY'AFF,>!!!&&#. -?' $rk   c                     | j                         D ch c]  \  }t        fddD              r c}}S c c}}w )Nc              3   &   K   | ]  }|v  
 y wr  rj   )rN   nr   s     rQ   rR   z0MultiScaleVit.no_weight_decay.<locals>.<genexpr>H  s     \!qAv\s   )rE  r   r   rC  )named_parametersany)rP   r   r   s    ` rQ   no_weight_decayzMultiScaleVit.no_weight_decayE  sH    "335 ^ ^da\'[\\  ^ 	^ ^s    :c                 $    t        dddg      }|S )Nz^patch_embed)z^stages\.(\d+)N)z^norm)i )stemr)  )rH  )rP   coarsematchers      rQ   group_matcherzMultiScaleVit.group_matcherJ  s     -/CD
 rk   c                 4    | j                   D ]	  }||_         y r  )rF  r'  )rP   enabler  s      rQ   set_grad_checkpointingz$MultiScaleVit.set_grad_checkpointingR  s     	*A#)A 	*rk   r   c                 .    | j                   j                  S r  )rN  rA  )rP   s    rQ   get_classifierzMultiScaleVit.get_classifierW  s    yy||rk   c                 T   || _         ||| _        t        | j                  j                  d      r*| j                  j                  j
                  j                  nd }t        | j                  j                  d      r*| j                  j                  j
                  j                  nd }t        j                  t        dt        j                  | j                        fd|dkD  r#t        j                  | j                  |||      nt        j                         fg            | _        y )NrX  r@  rA  r   ry   )r6  r5  hasattrrN  rA  rX  rz   r{   r   rL  r   rM  r8  r   rI  r  )rP   r6  r5  rz   r{   s        rQ   reset_classifierzMultiScaleVit.reset_classifier[  s    &"*D/6tyy||X/N$$++TX-4TYY\\8-L		##))RVMM+RZZ/0\gjk\k299T..FRWXqsq|q|q~/
 # 	rk   r   indicesrK  
stop_early
output_fmtintermediates_onlyc                    |dv sJ d       |dk(  }g }t        t        | j                        |      \  }	}
| j                  |      \  }}|j                  d   }| j
                  6| j
                  j                  |dd      }t        j                  ||fd      }| j                  || j                  z   }t        | j                        dz
  }t        | j                        D ]  \  }} |||      \  }}||	v s|r||k(  r| j                  |      }n|}|rC| j
                  |ddddf   }|j                  ||d   |d   d      j                  dd	dd
      }|j                  |        |r|S |k(  r| j                  |      }||fS )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWNLCz!Output shape must be NCHW or NLC.rt  r   Nr   r   r   r#   r"   )r   rV   rF  rB  r   rC  expandr   r   rE  	enumeraterK  r   r   r\   )rP   r   ro  rK  rp  rq  rr  r   intermediatestake_indices	max_indexr   r   
cls_tokenslast_idxfeat_idxrV  x_inters                     rQ   forward_intermediatesz#MultiScaleVit.forward_intermediatesf  s   * _,Q.QQ,&"6s4;;7G"Qi ''*9GGAJ>>%..q"b9J		:q/q1A>>%DNN"At{{#a'(5 	.OHe I.LAy<'H0"iilGG~~1")!QR%.%ooa1y|RPXXYZ\]_`bcdG$$W-	.   x		!A-rk   
prune_norm
prune_headc                     t        t        | j                        |      \  }}|rt        j                         | _        |r| j                  dd       |S )z@ Prune layers not required for specified intermediates.
        r    )r   rV   rF  r   r  rK  rn  )rP   ro  r  r  ry  rz  s         rQ   prune_intermediate_layersz'MultiScaleVit.prune_intermediate_layers  sI     #7s4;;7G"Qi DI!!!R(rk   c                 d   | j                  |      \  }}|j                  \  }}}| j                  6| j                  j                  |dd      }t	        j
                  ||fd      }| j                  || j                  z   }| j                  D ]  } |||      \  }} | j                  |      }|S )Nr   r   r   )	rB  r   rC  rv  r   r   rE  rF  rK  )rP   r   r   r   r   r  r{  rV  s           rQ   forward_featureszMultiScaleVit.forward_features  s    ''*9''1a>>%..q"b9J		:q/q1A>>%DNN"A[[ 	/E I.LAy	/ IIaLrk   
pre_logitsc                     | j                   r=| j                   dk(  r%|d d | j                  d f   j                  d      }n	|d d df   }|r|S | j                  |      S )Nr   r   r   )r5  rD  meanrN  )rP   r   r  s      rQ   forward_headzMultiScaleVit.forward_head  s^    5(a//00166q9adGq0DIIaL0rk   c                 J    | j                  |      }| j                  |      }|S r  )r  r  r   s     rQ   r   zMultiScaleVit.forward  s'    !!!$a rk   ))   r  r#   N  r  r  NNFTr  )NFFrt  F)r   FT) rb   rc   rd   r   r   r	   re   r
   ri   rg   r~   rP  r   r.  ignorera  rf  ri  r   r   rk  rn  r   r   r   rh   r  r  r  r  r   r   r   s   @rQ   r   r     s   
 )3)-#$&!e'!e' CHoe' 	e'
 "#e' e' "e' e'N/ YY^ ^ YY  YY* * YY		  	C 	hsm 	 8<$$',9 ||9  eCcN349  	9 
 9  9  !%9  
tELL!5tELL7I)I#JJ	K9 z ./$#	3S	>*  	""1$ 1rk   c           
         d| v r| j                         D ]  }d|v s| |   }|j                         |   j                  }|j                  d   |d   k7  s@t        j                  j
                  j                  |j                  d|j                  d   d      j                  ddd      |d   d      }|j                  d|d         j                  dd      | |<    | S dd l	}d	| v r| d	   } t        |d
d       }t        |dd      }|J d       i d}	t        |      D ]<  \  }
}j                  t        |	|	|z         D ci c]
  }||
||	z
  f c}       |	|z  }	> i }| j                         D ]_  \  }}|j                  dfd|      }|r|j                  dd|      }n|j                  dd|      }d|v r|j!                  dd      }|||<   a |S c c}w )Nzstages.0.blocks.0.norm1.weightrel_posr   r   r   r"   linear)r   r2   model_stater%   r,   Tz3model requires depth attribute to remap checkpointszblocks\.(\d+)c           	          dt        | j                  d               d    dt        | j                  d               d    S )Nzstages.r   r   z.blocks.)re   group)r   	depth_maps    rQ   <lambda>z&checkpoint_filter_fn.<locals>.<lambda>  sJ    	#aggaj/ :1 =>hyQTUVU\U\]^U_Q`GabcGdFef rk   z stages\.(\d+).blocks\.(\d+).projz&stages.\1.blocks.\2.shortcut_proj_attnz%stages.\1.blocks.\2.shortcut_proj_mlprN  zhead.projectionhead.fc)keys
state_dictr   r   r   
functionalinterpolater   r   regetattrrw  updaterZ   itemssubreplace)r  modelr   r  dest_rel_pos_shaperel_pos_resizedr  r%   r,   	block_idx	stage_idxr`   rO   out_dictr   r  s                  @rQ   checkpoint_filter_fnr    s'   ':5" 
	eAA~$Q-%*%5%5%7%:%@%@"==#'9!'<<&+hh&9&9&E&E7==+;R@HHAqQ/2% 'F 'O
 %4$;$;B@RST@U$V$^$^_`bc$dJqM
	e 
".
UHd+F%5KTTTII!&) 	1yR[^_R_A`aA!iY77abQ	 H  " 1FFf
 :>fijkA:>ehijAQ;		+Y7A O% bs   G
)r   r"      r"   )r%   )r   r"      r"   r!   )r"      $   r:      r"   F)r%   r'   r(   r,   )r%   r.   )r%   r'   r(   r.   r,   )r:   r   <   r      r#   )mvitv2_tinymvitv2_smallmvitv2_basemvitv2_largemvitv2_small_clsmvitv2_base_clsmvitv2_large_clsmvitv2_huge_clsc           	          |j                  dd      }t        t        | |f|s	t        |    nt        |   t        t        |d      d|S )Nout_indicesr:   getter)r  feature_cls)	model_cfgpretrained_filter_fnfeature_cfg)popr   r   
model_cfgsr  rH  )variantcfg_variant
pretrainedkwargsr  s        rQ   _create_mvitv2r  *  sW    **]A.K .9*W%j>U1[hG  rk   r  c                 4    | ddd ddt         t        ddddd	|S )
Nr  )r#   r  r  g?bicubiczpatch_embed.projr  Tz
apache-2.0)urlr6  
input_size	pool_sizecrop_pctinterpolationr  r   
first_conv
classifierfixed_input_sizelicenser   )r  r  s     rQ   _cfgr  7  s6    =t%.B(	 	 	 	rk   zDhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_T_in1k.pythztimm/)r  	hf_hub_idzDhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_S_in1k.pythzDhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_B_in1k.pythzDhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_L_in1k.pyth)r  zEhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_B_in21k.pythiJ  )r  r  r6  zEhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_L_in21k.pythzEhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_H_in21k.pyth)zmvitv2_tiny.fb_in1kzmvitv2_small.fb_in1kzmvitv2_base.fb_in1kzmvitv2_large.fb_in1kr  zmvitv2_base_cls.fb_inw21kzmvitv2_large_cls.fb_inw21kzmvitv2_huge_cls.fb_inw21kc                     t        dd| i|S )Nr  )r  r  r  r  s     rQ   r  r  _      IJI&IIrk   c                     t        dd| i|S )Nr  )r  r  r  s     rQ   r  r  d      JZJ6JJrk   c                     t        dd| i|S )Nr  )r  r  r  s     rQ   r  r  i  r  rk   c                     t        dd| i|S )Nr  )r  r  r  s     rQ   r  r  n  r  rk   c                     t        dd| i|S )Nr  )r  r  r  s     rQ   r  r  s      NNvNNrk   c                     t        dd| i|S )Nr  )r  r  r  s     rQ   r  r  x      M
MfMMrk   c                     t        dd| i|S )Nr  )r  r  r  s     rQ   r  r  }  r  rk   c                     t        dd| i|S )Nr  )r  r  r  s     rQ   r  r    r  rk   r  r  )NF)r  r  )Er   rm   collectionsr   dataclassesr   	functoolsr   r   typingr   r   r	   r
   r   r   r   r   	timm.datar   r   timm.layersr   r   r   r   r   r   _builderr   	_featuresr   _features_fxr   _manipulater   	_registryr   r   __all__r   rp   r   rr   re   rh   r   r   r   r   r   r   r  r$  r   r  rH  r  r  r  default_cfgsr  r  r  r  r  r  r  r  rj   rk   rQ   <module>r     s    # ! % : :   A m m * + 3 # <.
/ -3 -3 -3`-: :B  #9  5<<%,,//0	   +/

 %,,'
 5<<c"#	
 
 /ll/<</ / S		/
 S	/ <</ <</ /dm299 m`\")) \~e bii e PG GTCBII CL,^   " ! "	 & % & %A'
T

 %R !%k$j %k !%S" #'S# "&S")& 6 J} J J K K K J} J J K K K OM O O N= N N OM O O N= N Nrk   