Ë
    éÓ^j—¡  ã                   ót  — d Z ddlZddlmZ ddlmZ ddlmZmZ ddl	m
Z
mZmZmZmZmZ ddlZddlmZ ddlmZmZ dd	lmZmZmZmZmZmZ d
dlmZ d
dlmZ d
dl m!Z! d
dl"m#Z# d
dl$m%Z%m&Z& ddgZ'e G d„ d«      «       Z(d„ Z) G d„ dejT                  «      Z+e!	 dVdee,   de-deej\                  eej\                     f   fd„«       Z/e!	 dWde,deej\                     deej\                  ee,   f   fd„«       Z0e!dej\                  dej\                  de-d ee,   d!ee,   d"ej\                  d#ej\                  fd$„«       Z1 G d%„ d&ejT                  «      Z2 G d'„ d(ejT                  «      Z3 G d)„ d*ejT                  «      Z4 G d+„ d,ejT                  «      Z5 G d-„ dejT                  «      Z6d.„ Z7 e8 e(d/¬0«       e(d1¬0«       e(d2¬0«       e(d3d4d5d6¬7«       e(d1d¬8«       e(d2d¬8«       e(d3d4d5dd¬9«       e(d:d;d<dd¬9«      ¬=«      Z9dXd>„Z:dYd@„Z; e& e;dAdB¬C«       e;dDdB¬C«       e;dEdB¬C«       e;dFdB¬C«       e;d?¬G«       e;dHdBdI¬J«       e;dKdBdI¬J«       e;dLdBdI¬J«      dMœ«      Z<e%dZde6fdN„«       Z=e%dZde6fdO„«       Z>e%dZde6fdP„«       Z?e%dZde6fdQ„«       Z@e%dZde6fdR„«       ZAe%dZde6fdS„«       ZBe%dZde6fdT„«       ZCe%dZde6fdU„«       ZDy)[a   Multi-Scale Vision Transformer v2

@inproceedings{li2021improved,
  title={MViTv2: Improved multiscale vision transformers for classification and detection},
  author={Li, Yanghao and Wu, Chao-Yuan and Fan, Haoqi and Mangalam, Karttikeya and Xiong, Bo and Malik, Jitendra and Feichtenhofer, Christoph},
  booktitle={CVPR},
  year={2022}
}

Code adapted from original Apache 2.0 licensed impl at https://github.com/facebookresearch/mvit
Original copyright below.

Modifications and timm support by / Copyright 2022, Ross Wightman
é    N)ÚOrderedDict)Ú	dataclass)ÚpartialÚreduce)ÚUnionÚListÚTupleÚOptionalÚAnyÚType)Únn©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚMlpÚDropPathÚcalculate_drop_path_ratesÚtrunc_normal_tf_Úget_norm_layerÚ	to_2tupleé   )Úbuild_model_with_cfg)Úfeature_take_indices)Úregister_notrace_function)Ú
checkpoint)Úregister_modelÚgenerate_default_cfgsÚMultiScaleVitÚMultiScaleVitCfgc                   ó  — e Zd ZU dZeedf   ed<   dZeeeedf   f   ed<   dZ	eeeedf   f   ed<   dZ
eed	<   d
Zeed<   dZeed<   dZeed<   d
Zeed<   d
Zeed<   dZeed<   dZeed<   dZeeef   ed<   dZeeeeef         ed<   dZeeeeef         ed<   dZeeeef      ed<   dZeeef   ed<   dZeeef   ed<   dZeeef   ed<   d Zeed!<   d"Zeed#<   d$Zeeeeef   f   ed%<   d&Z eeeeef   f   ed'<   d(Z!eed)<   d*„ Z"y)+r   ©é   é   é   r#   .Údepthsé`   Ú	embed_dimr   Ú	num_headsç      @Ú	mlp_ratioFÚ
pool_firstTÚexpand_attnÚqkv_biasÚuse_cls_tokenÚuse_abs_posÚresidual_poolingÚconvÚmode©r#   r#   Ú
kernel_qkv)©r   r   ©r"   r"   r6   r6   Ústride_qNÚ	stride_kv©é   r:   Ústride_kv_adaptive©é   r=   Úpatch_kernelÚpatch_strideÚpatch_paddingÚmaxÚ	pool_typeÚspatialÚrel_pos_typeÚgeluÚ	act_layerÚ	layernormÚ
norm_layerg�íµ ÷Æ°>Únorm_epsc           	      ó&  ‡ — t        ‰ j                  «      }t        ‰ j                  t        t
        f«      s"t	        ˆ fd„t        |«      D «       «      ‰ _        t        ‰ j                  «      |k(  sJ ‚t        ‰ j                  t        t
        f«      s"t	        ˆ fd„t        |«      D «       «      ‰ _        t        ‰ j                  «      |k(  sJ ‚‰ j                  �²‰ j                  €¥‰ j                  }g }t        |«      D ]x  }t        ‰ j                  |   «      dkD  rAt        t        |«      «      D �cg c]$  }t        ||   ‰ j                  |   |   z  d«      ‘Œ& }}|j                  t	        |«      «       Œz t	        |«      ‰ _	        y y y c c}w )Nc              3   óB   •K  — | ]  }‰j                   d |z  z  –— Œ y­w©r"   N)r'   ©Ú.0ÚiÚselfs     €ú]/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/mvitv2.pyú	<genexpr>z1MultiScaleVitCfg.__post_init__.<locals>.<genexpr>A   ó   øè ø€ Ò"V¸q 4§>¡>°A¸±FÕ#:Ñ"Vùó   ƒc              3   óB   •K  — | ]  }‰j                   d |z  z  –— Œ y­wrL   )r(   rM   s     €rQ   rR   z1MultiScaleVitCfg.__post_init__.<locals>.<genexpr>E   rS   rT   r   )Úlenr%   Ú
isinstancer'   ÚtupleÚlistÚranger(   r;   r8   Úminr7   rA   Úappend)rP   Ú
num_stagesÚ
_stride_kvÚpool_kv_striderO   Úds   `     rQ   Ú__post_init__zMultiScaleVitCfg.__post_init__>   sW  ø€ Ü˜Ÿ™Ó%ˆ
Ü˜$Ÿ.™.¬5´$¨-Ô8Ü"Ó"VÄEÈ*ÓDUÔ"VÓVˆDŒNÜ�4—>‘>Ó" jÒ0Ð0Ð0ä˜$Ÿ.™.¬5´$¨-Ô8Ü"Ó"VÄEÈ*ÓDUÔ"VÓVˆDŒNÜ�4—>‘>Ó" jÒ0Ð0Ð0à×"Ñ"Ð.°4·>±>Ð3IØ×0Ñ0ˆJØˆNÜ˜:Ó&ò 9�Ü�t—}‘} QÑ'Ó(¨1Ò,ô "'¤s¨:£Ó!7ö"àô ˜J q™M¨T¯]©]¸1Ñ-=¸aÑ-@Ñ@À!ÕDð"�Jð "ð ×%Ñ%¤e¨JÓ&7Õ8ð9ô # >Ó2ˆD�Nð 4JÐ.ùò
"s   Ä4)F)#Ú__name__Ú
__module__Ú__qualname__r%   r	   ÚintÚ__annotations__r'   r   r(   r*   Úfloatr+   Úboolr,   r-   r.   r/   r0   r2   Ústrr4   r7   r
   r8   r;   r>   r?   r@   rB   rD   rF   rH   rI   ra   © ó    rQ   r   r   $   s�  … à+€FˆE�#�s�(‰OÓ+Ø-/€Iˆu�S˜%  S ™/Ð)Ñ*Ó/Ø-.€Iˆu�S˜%  S ™/Ð)Ñ*Ó.Ø€IˆuÓØ€J�ÓØ€K�ÓØ€HˆdÓØ€M�4ÓØ€K�ÓØ!Ð�dÓ!Ø€Dˆ#ÓØ"(€J��c˜3�h‘Ó(Ø1Q€Hˆh�u˜U 3¨ 8™_Ñ-Ñ.ÓQØ26€Iˆx˜˜e C¨ H™oÑ.Ñ/Ó6Ø4:Ð˜  s¨C x¡Ñ1Ó:Ø$*€L�%˜˜S˜‘/Ó*Ø$*€L�%˜˜S˜‘/Ó*Ø%+€M�5˜˜c˜‘?Ó+Ø€IˆsÓØ!€L�#Ó!Ø-3€Iˆu�S˜%  S ™/Ð)Ñ*Ó3Ø.9€J��c˜5  c ™?Ð*Ñ+Ó9Ø€HˆeÓó3rk   c                 ó8   — t        t        j                  | d«      S )Nr   )r   ÚoperatorÚmul)Úiterables    rQ   Úprodrp   U   s   € Ü”(—,‘, ¨!Ó,Ð,rk   c                   ó–   ‡ — e Zd ZdZ	 	 	 	 	 	 	 d
dededeeef   deeef   deeef   f
ˆ fd„Zdeej                  e	e   f   fd	„Z
ˆ xZS )Ú
PatchEmbedz
    PatchEmbed.
    Údim_inÚdim_outÚkernelÚstrideÚpaddingc                 ól   •— t         ‰	| �  «        ||dœ}t        j                  ||f|||dœ|¤Ž| _        y )N©ÚdeviceÚdtype)Úkernel_sizerv   rw   )ÚsuperÚ__init__r   ÚConv2dÚproj)
rP   rs   rt   ru   rv   rw   rz   r{   ÚddÚ	__class__s
            €rQ   r~   zPatchEmbed.__init__^   sJ   ø€ ô 	‰ÑÔØ¨Ñ/ˆä—I‘IØØð
ð ØØñ
ð ñ
ˆ�	rk   Úreturnc                 ó„   — | j                  |«      }|j                  d«      j                  dd«      |j                  dd  fS )Nr"   r   éþÿÿÿ)r€   ÚflattenÚ	transposeÚshape©rP   Úxs     rQ   ÚforwardzPatchEmbed.forwardt   s9   € Ø�I‰I�a‹Lˆà�y‰y˜‹|×%Ñ% a¨Ó+¨Q¯W©W°R°S¨\Ð9Ð9rk   )r#   i   r<   r9   r3   NN)rb   rc   rd   Ú__doc__re   r	   r~   ÚtorchÚTensorr   r‹   Ú__classcell__©r‚   s   @rQ   rr   rr   Y   sŠ   ø„ ñð ØØ&,Ø&,Ø'-ØØñ
àð
ð ð
ð ˜#˜s˜(‘Oð	
ð
 ˜#˜s˜(‘Oð
ð ˜3 ˜8‘_õ
ð,:˜E %§,¡,°°S±	Ð"9Ñ:÷ :rk   rr   TÚ	feat_sizeÚhas_cls_tokenrƒ   c                 óä   — |\  }}|r#| d d …d d …d d…d d …f   | d d …d d …dd …d d …f   } }nd }| j                  d||| j                  d   «      j                  dddd«      j                  «       } | |fS )Nr   éÿÿÿÿr   r#   r"   )Úreshaperˆ   ÚpermuteÚ
contiguous)rŠ   r‘   r’   ÚHÚWÚcls_toks         rQ   Úreshape_pre_poolr›   z   s|   € ð �D€A€qÙØ’qš!˜R˜a˜R¢�{‘^ Q¢qª!¨Q©R² {¡^�‰àˆØ	�	‰	�"�a˜˜AŸG™G B™KÓ(×0Ñ0°°A°q¸!Ó<×GÑGÓI€AØˆgˆ:Ðrk   r(   rš   c                 ó  — | j                   d   | j                   d   g}| j                   d   | j                   d   z  }| j                  d|| j                   d   |«      j                  dd«      } |�t        j                  || fd¬«      } | |fS )Nr"   r#   r”   r   ©Údim)rˆ   r•   r‡   r�   Úcat)rŠ   r(   rš   r‘   ÚL_pooleds        rQ   Úreshape_post_poolr¡   ‰   s…   € ð —‘˜‘˜QŸW™W Q™ZÐ(€IØ�w‰w�q‰z˜AŸG™G A™JÑ&€HØ	�	‰	�"�i §¡¨¡¨XÓ6×@Ñ@ÀÀAÓF€AØÐÜ�I‰I�w �l¨Ô*ˆØˆiˆ<Ðrk   ÚattnÚqÚq_sizeÚk_sizeÚ	rel_pos_hÚ	rel_pos_wc                 óÊ  — |rdnd}|\  }}	|\  }
}t        |
|z  d«      }t        ||
z  d«      }t        j                  ||j                  t        j                  ¬«      j                  d«      |z  t        j                  |
|j                  t        j                  ¬«      j                  d«      |z  z
  }||
dz
  |z  z  }t        ||	z  d«      }t        |	|z  d«      }t        j                  |	|j                  t        j                  ¬«      j                  d«      |z  t        j                  ||j                  t        j                  ¬«      j                  d«      |z  z
  }||dz
  |z  z  }||j	                  «          }||j	                  «          }|j                  \  }}}}|dd…dd…|d…f   j                  ||||	|«      }t        j                  d||«      }t        j                  d||«      }| dd…dd…|d…|d…f   j                  |d||	|
|«      |j                  d«      z   |j                  d	«      z   j                  |d||	z  |
|z  «      | dd…dd…|d…|d…f<   | S )
z1
    Spatial Relative Positional Embeddings.
    r   r   g      ð?ry   r”   Nzbyhwc,hkc->byhwkzbyhwc,wkc->byhwkr…   )
rA   r�   Úarangerz   ÚlongÚ	unsqueezerˆ   r•   ÚeinsumÚview)r¢   r£   r’   r¤   r¥   r¦   r§   Úsp_idxÚq_hÚq_wÚk_hÚk_wÚ	q_h_ratioÚ	k_h_ratioÚdist_hÚ	q_w_ratioÚ	k_w_ratioÚdist_wÚrel_hÚrel_wÚBÚn_headÚq_Nrž   Úr_qs                            rQ   Úcal_rel_pos_typer¿   —   sD  € ñ  ‰Q Q€FØ�H€CˆØ�H€Cˆô �C˜#‘I˜sÓ#€IÜ�C˜#‘I˜sÓ#€Iä�‰�S §¡´·±Ô<×FÑFÀrÓJÈYÑVÜ�‰�S §¡´·±Ô<×FÑFÀqÓIÈIÑUñ	Vð ð ˆs�Q‰w˜)Ñ#Ñ#€FÜ�C˜#‘I˜sÓ#€IÜ�C˜#‘I˜sÓ#€Iä�‰�S §¡´·±Ô<×FÑFÀrÓJÈYÑVÜ�‰�S §¡´·±Ô<×FÑFÀqÓIÈIÑUñ	Vð ð ˆs�Q‰w˜)Ñ#Ñ#€Fà�f—k‘k“mÑ$€EØ�f—k‘k“mÑ$€EàŸ'™'Ñ€A€vˆs�Cà
ŠAŠq�&‘'ˆMÑ
×
"Ñ
" 1 f¨c°3¸Ó
<€CÜ�L‰LÐ+¨S°%Ó8€EÜ�L‰LÐ+¨S°%Ó8€Eð 	ŠQ’�6‘7˜F™GÐ#Ñ$×)Ñ)¨!¨R°°c¸3ÀÓDØ
�/‰/˜"Ó
ñ	à
�/‰/˜"Ó
ñ	÷ �dˆ1ˆb�#˜‘)˜S 3™YÓ'ð	 	ŠŠAˆv‰w˜™Ð	Ñ ð €Krk   c                   óæ   ‡ — e Zd Zddddddddddej                  ddfdeded	eeef   d
edededeeef   deeef   deeef   deeef   dededede	ej                     fˆ fd„Zd	ee   fd„Zˆ xZS )ÚMultiScaleAttentionPoolFirsté   Tr1   r5   rC   Nrž   rt   r‘   r(   r-   r2   Úkernel_qÚ	kernel_kvr7   r8   r’   rD   r0   rH   c           	      ó^  •— ||dœ}t         ‰| �  «        || _        || _        ||z  | _        | j                  dz  | _        || _        t        |D �cg c]  }t        |dz  «      ‘Œ c}«      }t        |D �cg c]  }t        |dz  «      ‘Œ c}«      }t        j                  ||fd|i|¤Ž| _        t        j                  ||fd|i|¤Ž| _        t        j                  ||fd|i|¤Ž| _        t        j                  ||fi |¤Ž| _        t        |«      dk(  rt        |	«      dk(  rd }t        |«      dk(  rt        |
«      dk(  rd }|| _        |dk(  | _        d\  | _        | _        | _        d\  | _        | _        | _        |dv rW|d	k(  rt        j0                  nt        j2                  }|r |||	|«      | _        |rØ |||
|«      | _         |||
|«      | _        n¹|d
k(  s|dk(  r¡|d
k(  r||z  n|}|r0t        j4                  |||f|	||ddœ|¤Ž| _         ||fi |¤Ž| _        |rot        j4                  |||f|
||ddœ|¤Ž| _         ||fi |¤Ž| _        t        j4                  |||f|
||ddœ|¤Ž| _         ||fi |¤Ž| _        nt7        d|› �«      ‚|| _        | j8                  dk(  rô|d   |d   k(  sJ ‚|d   }t;        |	«      dkD  r||	d   z  n|}t;        |
«      dkD  r||
d   z  n|}dt=        ||«      z  dz
  }t        j>                  tA        jB                  || j                  fi |¤Ž«      | _"        t        j>                  tA        jB                  || j                  fi |¤Ž«      | _#        tI        | jD                  d¬«       tI        | jF                  d¬«       || _%        y c c}w c c}w )Nry   ç      à¿r"   Úbiasr   Úconv_unshared©NNN©ÚavgrA   rA   r1   F©rv   rw   ÚgroupsrÇ   úUnsupported model rC   r   ç{®Gáz”?©Ústd)&r}   r~   r(   rt   Úhead_dimÚscaler’   rX   re   r   ÚLinearr£   ÚkÚvr€   rp   r2   ÚunsharedÚpool_qÚpool_kÚpool_vÚnorm_qÚnorm_kÚnorm_vÚ	MaxPool2dÚ	AvgPool2dr   ÚNotImplementedErrorrD   rV   rA   Ú	Parameterr�   Úzerosr¦   r§   r   r0   ©rP   rž   rt   r‘   r(   r-   r2   rÃ   rÄ   r7   r8   r’   rD   r0   rH   rz   r{   r�   r£   Ú	padding_qÚkvÚ
padding_kvÚpool_opÚdim_convÚsizer¤   Úkv_sizeÚ
rel_sp_dimr‚   s                               €rQ   r~   z%MultiScaleAttentionPoolFirst.__init__Ë   sÃ  ø€ ð& ¨Ñ/ˆÜ‰ÑÔØ"ˆŒØˆŒØ 9Ñ,ˆŒØ—]‘] dÑ*ˆŒ
Ø*ˆÔÜ°Ö9¨1œ3˜q A™v�;Ò9Ó:ˆ	Ü°9Ö=¨RœC  a¡�LÒ=Ó>ˆ
ä—‘˜3 Ñ=¨hÐ=¸"Ñ=ˆŒÜ—‘˜3 Ñ=¨hÐ=¸"Ñ=ˆŒÜ—‘˜3 Ñ=¨hÐ=¸"Ñ=ˆŒÜ—I‘I˜g wÑ5°"Ñ5ˆŒ	ô �‹>˜QÒ¤4¨£>°QÒ#6ØˆHÜ�	‹?˜aÒ¤D¨£O°qÒ$8ØˆIØˆŒ	Ø Ñ/ˆŒØ0@Ñ-ˆŒ�T”[ $¤+Ø0@Ñ-ˆŒ�T”[ $¤+Ø�>Ñ!Ø&*¨e¢m”b—l’l¼¿¹ˆGÙÙ% h°¸)ÓD�”ÙÙ% i°¸JÓG�”Ù% i°¸JÓG�•Ø�VŠ^˜t Ò6Ø+/°6ª>�s˜iÒ'¸sˆHÙÜ Ÿi™iØØØð	ð $Ø%Ø#Øñ	ð ñ	�”ñ )¨Ñ8°RÑ8�”ÙÜ Ÿi™iØØØð	ð %Ø&Ø#Øñ	ð ñ	�”ñ )¨Ñ8°RÑ8�”Ü Ÿi™iØØØð	ð %Ø&Ø#Øñ	ð ñ	�”ñ )¨Ñ8°RÑ8�•ä%Ð(:¸4¸&Ð&AÓBÐBð )ˆÔØ×Ñ 	Ò)Ø˜Q‘< 9¨Q¡<Ò/Ð/Ð/Ø˜Q‘<ˆDÜ,/°«M¸AÒ,=�T˜X a™[Ò(À4ˆFÜ.1°)«n¸qÒ.@�d˜i¨™lÒ*ÀdˆGØœS ¨Ó1Ñ1°AÑ5ˆJäŸ\™\¬%¯+©+°jÀ$Ç-Á-Ñ*VÐSUÑ*VÓWˆDŒNÜŸ\™\¬%¯+©+°jÀ$Ç-Á-Ñ*VÐSUÑ*VÓWˆDŒNÜ˜TŸ^™^°Õ6Ü˜TŸ^™^°Õ6à 0ˆÕùò] :ùÚ=s   ÁN%Á6N*c           	      ó®  — |j                   \  }}}| j                  rdn| j                  }|j                  |||d«      j	                  dddd«      }|x}x}}	| j
                  �Ft        ||| j                  «      \  }}
| j                  |«      }t        || j                  |
«      \  }}n|}| j                  �| j                  |«      }| j                  �Ft        ||| j                  «      \  }}| j                  |«      }t        || j                  |«      \  }}n|}| j                  �| j                  |«      }| j                  �Ft        |	|| j                  «      \  }	}| j                  |	«      }	t        |	| j                  |«      \  }	}n|}| j                  �| j                  |	«      }	|d   |d   z  t        | j                  «      z   }|j                  dd«      j                  ||d«      }| j!                  |«      j                  ||| j                  d«      j                  dd«      }|d   |d   z  t        | j                  «      z   }|j                  dd«      j                  ||d«      }| j#                  |«      j                  ||| j                  d«      }|d   |d   z  t        | j                  «      z   }|	j                  dd«      j                  ||d«      }	| j%                  |	«      j                  ||| j                  d«      j                  dd«      }	|| j&                  z  |z  }| j(                  dk(  r/t+        ||| j                  ||| j,                  | j.                  «      }|j1                  d¬«      }||	z  }| j2                  r||z   }|j                  dd«      j                  |d| j4                  «      }| j7                  |«      }||fS )Nr   r”   r   r"   r#   rC   r�   )rˆ   r×   r(   r•   r–   rØ   r›   r’   r¡   rÛ   rÙ   rÜ   rÚ   rÝ   re   r‡   r£   rÕ   rÖ   rÓ   rD   r¿   r¦   r§   Úsoftmaxr0   rt   r€   )rP   rŠ   r‘   r»   ÚNÚ_Úfold_dimr£   rÕ   rÖ   Úq_tokr¤   Úk_tokr¥   Úv_tokÚv_sizer½   Úk_NÚv_Nr¢   s                       rQ   r‹   z$MultiScaleAttentionPoolFirst.forward5  sn  € Ø—'‘'‰ˆˆ1ˆaàŸš‘1¨4¯>©>ˆØ�I‰I�a˜˜H bÓ)×1Ñ1°!°Q¸¸1Ó=ˆØˆˆˆˆA�à�;‰;Ð"Ü'¨¨9°d×6HÑ6HÓI‰HˆAˆuØ—‘˜A“ˆAÜ)¨!¨T¯^©^¸UÓC‰IˆA‰vàˆFØ�;‰;Ð"Ø—‘˜A“ˆAà�;‰;Ð"Ü'¨¨9°d×6HÑ6HÓI‰HˆAˆuØ—‘˜A“ˆAÜ)¨!¨T¯^©^¸UÓC‰IˆA‰vàˆFØ�;‰;Ð"Ø—‘˜A“ˆAà�;‰;Ð"Ü'¨¨9°d×6HÑ6HÓI‰HˆAˆuØ—‘˜A“ˆAÜ)¨!¨T¯^©^¸UÓC‰IˆA‰vàˆFØ�;‰;Ð"Ø—‘˜A“ˆAà�Q‰i˜& ™)Ñ#¤c¨$×*<Ñ*<Ó&=Ñ=ˆØ�K‰K˜˜1Ó×%Ñ% a¨¨bÓ1ˆØ�F‰F�1‹I×Ñ˜a  d§n¡n°bÓ9×CÑCÀAÀqÓIˆà�Q‰i˜& ™)Ñ#¤c¨$×*<Ñ*<Ó&=Ñ=ˆØ�K‰K˜˜1Ó×%Ñ% a¨¨bÓ1ˆØ�F‰F�1‹I×Ñ˜a  d§n¡n°bÓ9ˆà�Q‰i˜& ™)Ñ#¤c¨$×*<Ñ*<Ó&=Ñ=ˆØ�K‰K˜˜1Ó×%Ñ% a¨¨bÓ1ˆØ�F‰F�1‹I×Ñ˜a  d§n¡n°bÓ9×CÑCÀAÀqÓIˆà�D—J‘J‘ !Ñ#ˆØ×Ñ 	Ò)Ü#ØØØ×"Ñ"ØØØ—‘Ø—‘óˆDð �|‰| ˆ|Ó#ˆØ�1‰Hˆà× Ò Ø�A‘ˆAà�K‰K˜˜1Ó×%Ñ% a¨¨T¯\©\Ó:ˆØ�I‰I�a‹Lˆà�&ˆyÐrk   ©rb   rc   rd   r   Ú	LayerNormre   r	   rh   ri   r   ÚModuler~   r   r‹   r�   r�   s   @rQ   rÁ   rÁ   Ê   s  ø„ ð Ø!ØØ(.Ø)/Ø(.Ø)/Ø"&Ø )Ø%)Ø*,¯,©,ØØñ#h1àðh1ð ðh1ð ˜S #˜X‘ð	h1ð
 ðh1ð ðh1ð ðh1ð ˜C ˜H‘oðh1ð ˜S #˜X‘ðh1ð ˜C ˜H‘oðh1ð ˜S #˜X‘ðh1ð  ðh1ð ðh1ð #ðh1ð ˜RŸY™Y™õh1ðTB D¨¡I÷ Brk   rÁ   c                   óæ   ‡ — e Zd Zddddddddddej                  ddfdeded	eeef   d
edededeeef   deeef   deeef   deeef   dededede	ej                     fˆ fd„Zd	ee   fd„Zˆ xZS )ÚMultiScaleAttentionrÂ   Tr1   r5   rC   Nrž   rt   r‘   r(   r-   r2   rÃ   rÄ   r7   r8   r’   rD   r0   rH   c           	      óì  •— ||dœ}t         ‰| �  «        || _        || _        ||z  | _        | j                  dz  | _        || _        t        |D �cg c]  }t        |dz  «      ‘Œ c}«      }t        |D �cg c]  }t        |dz  «      ‘Œ c}«      }t        j                  ||dz  fd|i|¤Ž| _        t        j                  ||fi |¤Ž| _        t        |«      dk(  rt        |	«      dk(  rd }t        |«      dk(  rt        |
«      dk(  rd }|| _        |dk(  | _        d\  | _        | _        | _        d\  | _        | _        | _        |d	v rW|d
k(  rt        j,                  nt        j.                  }|r |||	|«      | _        |rØ |||
|«      | _         |||
|«      | _        n¹|dk(  s|dk(  r¡|dk(  r||z  n|}|r0t        j0                  |||f|	||ddœ|¤Ž| _         ||fi |¤Ž| _        |rot        j0                  |||f|
||ddœ|¤Ž| _         ||fi |¤Ž| _        t        j0                  |||f|
||ddœ|¤Ž| _         ||fi |¤Ž| _        nt3        d|› �«      ‚|| _        | j4                  dk(  rô|d   |d   k(  sJ ‚|d   }t7        |	«      dkD  r||	d   z  n|}t7        |
«      dkD  r||
d   z  n|}dt9        ||«      z  dz
  }t        j:                  t=        j>                  || j                  fi |¤Ž«      | _         t        j:                  t=        j>                  || j                  fi |¤Ž«      | _!        tE        | j@                  d¬«       tE        | jB                  d¬«       || _#        y c c}w c c}w )Nry   rÆ   r"   r#   rÇ   r   rÈ   rÉ   rÊ   rA   r1   FrÌ   rÎ   rC   r   rÏ   rÐ   )$r}   r~   r(   rt   rÒ   rÓ   r’   rX   re   r   rÔ   Úqkvr€   rp   r2   r×   rÛ   rÜ   rÝ   rØ   rÙ   rÚ   rÞ   rß   r   rà   rD   rV   rA   rá   r�   râ   r¦   r§   r   r0   rã   s                               €rQ   r~   zMultiScaleAttention.__init__{  s�  ø€ ð& ¨Ñ/ˆÜ‰ÑÔØ"ˆŒØˆŒØ 9Ñ,ˆŒØ—]‘] dÑ*ˆŒ
Ø*ˆÔÜ°Ö9¨1œ3˜q A™v�;Ò9Ó:ˆ	Ü°9Ö=¨RœC  a¡�LÒ=Ó>ˆ
ä—9‘9˜S '¨A¡+ÑC°HÐCÀÑCˆŒÜ—I‘I˜g wÑ5°"Ñ5ˆŒ	ô �‹>˜QÒ¤4¨£>°QÒ#6ØˆHÜ�	‹?˜aÒ¤D¨£O°qÒ$8ØˆIØˆŒ	Ø Ñ/ˆŒØ0@Ñ-ˆŒ�T”[ $¤+Ø0@Ñ-ˆŒ�T”[ $¤+Ø�>Ñ!Ø&*¨e¢m”b—l’l¼¿¹ˆGÙÙ% h°¸)ÓD�”ÙÙ% i°¸JÓG�”Ù% i°¸JÓG�•Ø�VŠ^˜t Ò6Ø/3°vª~�w )Ò+À7ˆHÙÜ Ÿi™iØØØð	ð $Ø%Ø#Øñ	ð ñ	�”ñ )¨Ñ8°RÑ8�”ÙÜ Ÿi™iØØØð	ð %Ø&Ø#Øñ	ð ñ	�”ñ )¨Ñ8°RÑ8�”Ü Ÿi™iØØØð	ð %Ø&Ø#Øñ	ð ñ	�”ñ )¨Ñ8°RÑ8�•ä%Ð(:¸4¸&Ð&AÓBÐBð )ˆÔØ×Ñ 	Ò)Ø˜Q‘< 9¨Q¡<Ò/Ð/Ð/Ø˜Q‘<ˆDÜ,/°«M¸AÒ,=�T˜X a™[Ò(À4ˆFÜ.1°)«n¸qÒ.@�d˜i¨™lÒ*ÀdˆGØœS ¨Ó1Ñ1°AÑ5ˆJäŸ\™\¬%¯+©+°jÀ$Ç-Á-Ñ*VÐSUÑ*VÓWˆDŒNÜŸ\™\¬%¯+©+°jÀ$Ç-Á-Ñ*VÐSUÑ*VÓWˆDŒNÜ˜TŸ^™^°Õ6Ü˜TŸ^™^°Õ6à 0ˆÕùòY :ùÚ=s   ÁM,Á6M1c           	      ó  — |j                   \  }}}| j                  |«      j                  ||d| j                  d«      j	                  ddddd«      }|j                  d¬«      \  }}}	| j                  �Ft        ||| j                  «      \  }}
| j                  |«      }t        || j                  |
«      \  }}n|}| j                  �| j                  |«      }| j                  �Ft        ||| j                  «      \  }}| j                  |«      }t        || j                  |«      \  }}n|}| j                  �| j                  |«      }| j                  �Et        |	|| j                  «      \  }	}| j                  |	«      }	t        |	| j                  |«      \  }	}| j                  �| j                  |	«      }	|| j                  z  |j!                  dd«      z  }| j"                  d	k(  r/t%        ||| j                  ||| j&                  | j(                  «      }|j+                  d¬«      }||	z  }| j,                  r||z   }|j!                  dd«      j                  |d| j.                  «      }| j1                  |«      }||fS )
Nr#   r”   r"   r   r   r:   r�   r…   rC   )rˆ   rý   r•   r(   r–   ÚunbindrØ   r›   r’   r¡   rÛ   rÙ   rÜ   rÚ   rÝ   rÓ   r‡   rD   r¿   r¦   r§   rí   r0   rt   r€   )rP   rŠ   r‘   r»   rî   rï   rý   r£   rÕ   rÖ   rñ   r¤   rò   r¥   ró   r¢   s                   rQ   r‹   zMultiScaleAttention.forwardã  s=  € Ø—'‘'‰ˆˆ1ˆaà�h‰h�q‹k×!Ñ! ! Q¨¨4¯>©>¸2Ó>×FÑFÀqÈ!ÈQÐPQÐSTÓUˆØ—*‘* �*Ó#‰ˆˆ1ˆaà�;‰;Ð"Ü'¨¨9°d×6HÑ6HÓI‰HˆAˆuØ—‘˜A“ˆAÜ)¨!¨T¯^©^¸UÓC‰IˆA‰vàˆFØ�;‰;Ð"Ø—‘˜A“ˆAà�;‰;Ð"Ü'¨¨9°d×6HÑ6HÓI‰HˆAˆuØ—‘˜A“ˆAÜ)¨!¨T¯^©^¸UÓC‰IˆA‰vàˆFØ�;‰;Ð"Ø—‘˜A“ˆAà�;‰;Ð"Ü'¨¨9°d×6HÑ6HÓI‰HˆAˆuØ—‘˜A“ˆAÜ$ Q¨¯©¸Ó>‰DˆAˆqØ�;‰;Ð"Ø—‘˜A“ˆAà�D—J‘J‘ !§+¡+¨b°"Ó"5Ñ5ˆØ×Ñ 	Ò)Ü#ØØØ×"Ñ"ØØØ—‘Ø—‘óˆDð �|‰| ˆ|Ó#ˆØ�1‰Hˆà× Ò Ø�A‘ˆAà�K‰K˜˜1Ó×%Ñ% a¨¨T¯\©\Ó:ˆØ�I‰I�a‹Lˆà�&ˆyÐrk   r÷   r�   s   @rQ   rû   rû   z  s  ø„ ð Ø!ØØ(.Ø)/Ø(.Ø)/Ø"&Ø )Ø%)Ø*,¯,©,ØØñ#f1àðf1ð ðf1ð ˜S #˜X‘ð	f1ð
 ðf1ð ðf1ð ðf1ð ˜C ˜H‘oðf1ð ˜S #˜X‘ðf1ð ˜C ˜H‘oðf1ð ˜S #˜X‘ðf1ð  ðf1ð ðf1ð #ðf1ð ˜RŸY™Y™õf1ðP3 D¨¡I÷ 3rk   rû   c            %       ó  ‡ — e Zd Zdddej                  ddddddddddddfd	ed
ededeeef   dededede	ej                     deeef   deeef   deeef   deeef   dedededededef$ˆ fd„Zdee   fd„Zdee   fd„Zˆ xZS )ÚMultiScaleBlockr)   Tç        r5   r1   FrC   Nrž   rt   r(   r‘   r*   r-   Ú	drop_pathrH   rÃ   rÄ   r7   r8   r2   r’   r,   r+   rD   r0   c                 ó\  •— ||dœ}t         ‰| �  «        ||k7  }|| _        || _        || _         ||fi |¤Ž| _        |r|rt        j                  ||fi |¤Žnd | _        |rat        |«      dkD  rS|D �cg c]  }|dkD  r|dz   n|‘Œ }}|}|D �cg c]  }t        |dz  «      ‘Œ }}t        j                  |||«      | _        nd | _        |r|n|}|rt        nt        } |||f||||	|
|||||||dœ|¤Ž| _        |dkD  rt!        |«      nt        j"                  «       | _         ||fi |¤Ž| _        |}|r|st        j                  ||fi |¤Žnd | _        t+        d|t        ||z  «      |dœ|¤Ž| _        |dkD  rt!        |«      | _        y t        j"                  «       | _        y c c}w c c}w )Nry   r   r"   )r(   r‘   r-   rÃ   rÄ   r7   r8   rH   r’   r2   rD   r0   r  )Úin_featuresÚhidden_featuresÚout_featuresrj   )r}   r~   rž   rt   r’   Únorm1r   rÔ   Úshortcut_proj_attnrp   re   rÞ   Úshortcut_pool_attnrÁ   rû   r¢   r   ÚIdentityÚ
drop_path1Únorm2Úshortcut_proj_mlpr   ÚmlpÚ
drop_path2) rP   rž   rt   r(   r‘   r*   r-   r  rH   rÃ   rÄ   r7   r8   r2   r’   r,   r+   rD   r0   rz   r{   r�   Úproj_neededÚsÚkernel_skipÚstride_skipÚskipÚpadding_skipÚatt_dimÚ
attn_layerÚmlp_dim_outr‚   s                                   €rQ   r~   zMultiScaleBlock.__init__  sÜ  ø€ ð. ¨Ñ/ˆÜ‰ÑÔØ˜W‘nˆØˆŒØˆŒØ*ˆÔá Ñ* rÑ*ˆŒ
áCNÑS^¤"§)¡)¨C°Ñ"?¸BÒ"?ÐdhˆÔÙœ˜X›¨Ò*Ø:BÖC°Q A¨¢E˜1˜qš5¨qÑ0ÐCˆKÐCØ"ˆKØ7BÖC¨tœC ¨¡	�NÐCˆLÐCÜ&(§l¡l°;ÀÈ\Ó&ZˆDÕ#à&*ˆDÔ#á(‘'¨cˆÙ5?Õ1ÔEXˆ
ÙØØð
ð  ØØØØØØØ!Ø'ØØ%Ø-ñ
ð ñ
ˆŒ	ð" 2;¸S²œ( 9Ô-ÄbÇkÁkÃmˆŒá Ñ.¨2Ñ.ˆŒ
ØˆÙBMÑVa¤§¡¨3°Ñ!>¸2Ò!>ÐgkˆÔÜð 
ØÜ ¨)Ñ 3Ó4Ø$ñ
ð ñ	
ˆŒð 2;¸S²œ( 9Ó-ˆ�ÄbÇkÁkÃmˆ�ùòK DùâCs   Á3F$ÂF)c                 ó¨  — | j                   €|S | j                  r|d d …d d…d d …f   |d d …dd …d d …f   }}nd }|j                  \  }}}|\  }}|j                  ||||«      j	                  dddd«      j                  «       }| j                  |«      }|j                  ||d«      j                  dd«      }|�t        j                  ||fd¬«      }|S )Nr   r   r#   r"   r”   r�   )	r
  r’   rˆ   r•   r–   r—   r‡   r�   rŸ   )	rP   rŠ   r‘   rš   r»   ÚLÚCr˜   r™   s	            rQ   Ú_shortcut_poolzMultiScaleBlock._shortcut_poolc  sØ   € Ø×"Ñ"Ð*ØˆHØ×ÒØš1˜b˜q˜b¢!˜8™ aª¨1©2ªq¨¡k�Q‰GàˆGØ—'‘'‰ˆˆ1ˆaØ‰ˆˆ1Ø�I‰I�a˜˜A˜qÓ!×)Ñ)¨!¨Q°°1Ó5×@Ñ@ÓBˆØ×#Ñ# AÓ&ˆØ�I‰I�a˜˜BÓ×)Ñ)¨!¨QÓ/ˆØÐÜ—	‘	˜7 A˜,¨AÔ.ˆAØˆrk   c                 ó†  — | j                  |«      }| j                  €|n| j                  |«      }| j                  ||«      }| j                  ||«      \  }}|| j	                  |«      z   }| j                  |«      }| j                  €|n| j                  |«      }|| j                  | j                  |«      «      z   }||fS ©N)	r  r	  r  r¢   r  r  r  r  r  )rP   rŠ   r‘   Úx_normÚ
x_shortcutÚfeat_size_news         rQ   r‹   zMultiScaleBlock.forwards  s¹   € Ø—‘˜A“ˆà×1Ñ1Ð9‘Q¸t×?VÑ?VÐW]Ó?^ˆ
Ø×(Ñ(¨°YÓ?ˆ
ØŸ9™9 V¨YÓ7Ñˆˆ=Ø˜Ÿ™¨Ó+Ñ+ˆà—‘˜A“ˆØ×0Ñ0Ð8‘Q¸d×>TÑ>TÐU[Ó>\ˆ
Ø˜Ÿ™¨¯©°&Ó)9Ó:Ñ:ˆØ�-ÐÐrk   )rb   rc   rd   r   rø   re   r	   rg   rh   r   rù   ri   r~   r   r  r‹   r�   r�   s   @rQ   r  r    sg  ø„ ð  #Ø!Ø"Ø*,¯,©,Ø(.Ø)/Ø(.Ø)/ØØ"&Ø %Ø$Ø )Ø%)ØØñ+GTàðGTð ðGTð ð	GTð
 ˜S #˜X‘ðGTð ðGTð ðGTð ðGTð ˜RŸY™Y™ðGTð ˜C ˜H‘oðGTð ˜S #˜X‘ðGTð ˜C ˜H‘oðGTð ˜S #˜X‘ðGTð ðGTð  ðGTð  ð!GTð" ð#GTð$ ð%GTð& #õ'GTðR¨4°©9ó ð   D¨¡I÷  rk   r  c            *       ó  ‡ — e Zd Zddddddddddddej                  dddfd	ed
edededeeef   dededeeef   deeef   deeef   deeef   de	dededede	dede
ej                     deeee   f   f&ˆ fd„Zdee   fd„Zˆ xZS )ÚMultiScaleVitStager)   Tr5   r1   FrC   r  Nrž   rt   Údepthr(   r‘   r*   r-   rÃ   rÄ   r7   r8   r2   r’   r,   r+   rD   r0   rH   r  c                 ó@  •— ||dœ}t         ‰| �  «        d| _        t        j                  «       | _        |r|f|z  }n|f|dz
  z  |fz   }t        |«      D ]¹  }t        di d|“d||   “d|“d|“d|“d	|“d
|“d|	“d|dk(  r|
nd“d|“d|“d|“d|“d|“d|“d|“d|“dt        |t        t        f«      r||   n|“|¤Ž}||   }| j
                  j                  |«       |dk(  sŒ�t        t        ||
«      D ��cg c]
  \  }}||z  ‘Œ c}}«      }Œ» || _        y c c}}w )Nry   Fr   rž   rt   r(   r‘   r*   r-   rÃ   rÄ   r7   r   r5   r8   r2   r’   r+   rD   r0   r,   rH   r  rj   )r}   r~   Úgrad_checkpointingr   Ú
ModuleListÚblocksrZ   r  rW   rY   rX   r\   Úzipr‘   )rP   rž   rt   r%  r(   r‘   r*   r-   rÃ   rÄ   r7   r8   r2   r’   r,   r+   rD   r0   rH   r  rz   r{   r�   Úout_dimsrO   Úattention_blockré   rv   r‚   s                               €rQ   r~   zMultiScaleVitStage.__init__ƒ  s¡  ø€ ð0 ¨Ñ/ˆÜ‰ÑÔØ"'ˆÔä—m‘m“oˆŒÙØ�z EÑ)‰Hà�v ¨¡Ñ+¨w¨jÑ8ˆHä�u“ò 	aˆAÜ-ò Ùðà  šðñ $ðñ $ð	ñ
 $ðñ "ðñ "ðñ $ðð &'¨!¢V™°øðñ $ðñ ðñ ,ðñ &ðñ *ðñ "2ðñ  (ð!ñ" &ð#ô$ +5°YÄÄuÀÔ*N˜) Aš,ÐT]øØñ'ˆOð* ˜1‘+ˆCØ�K‰K×Ñ˜Ô/Ø�A‹vÜ!ÄcÈ)ÐU]ÓF^×"_±l°d¸F 4¨6£>Ó"_Ó`‘	ð3	að6 #ˆ�ùó #`s   Ã9Dc                 óº   — | j                   D ]I  }| j                  r/t        j                  j	                  «       st        |||«      \  }}Œ> |||«      \  }}ŒK ||fS r  )r)  r'  r�   ÚjitÚis_scriptingr   )rP   rŠ   r‘   Úblks       rQ   r‹   zMultiScaleVitStage.forwardÂ  s[   € Ø—;‘;ò 	1ˆCØ×&Ò&¬u¯y©y×/EÑ/EÔ/GÜ)¨#¨q°)Ó<‘�‘9á" 1 iÓ0‘�‘9ð		1ð
 �)ˆ|Ðrk   )rb   rc   rd   r   rø   re   r	   rg   rh   ri   r   rù   r   r   r~   r‹   r�   r�   s   @rQ   r$  r$  �  sI  ø„ ð  #Ø!Ø(.Ø)/Ø(.Ø)/ØØ"&Ø %Ø$Ø )Ø%)Ø*,¯,©,Ø36ØØñ-=#àð=#ð ð=#ð ð	=#ð
 ð=#ð ˜S #˜X‘ð=#ð ð=#ð ð=#ð ˜C ˜H‘oð=#ð ˜S #˜X‘ð=#ð ˜C ˜H‘oð=#ð ˜S #˜X‘ð=#ð ð=#ð  ð=#ð ð=#ð  ð!=#ð" ð#=#ð$ #ð%=#ð& ˜RŸY™Y™ð'=#ð( ˜U D¨¡KÐ/Ñ0õ)=#ð~ D¨¡I÷ rk   r$  c                   óz  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 ddedeeef   dedee   dede	de	fˆ fd	„Z
d
„ Zej                  j                  d„ «       Zej                  j                  d d„«       Zej                  j                  d!d„«       Zej                  j                  dej&                  fd„«       Zd"dedee   fd„Z	 	 	 	 	 d#dej,                  deeeee   f      dededededeeej,                     eej,                  eej,                     f   f   fd„Z	 	 	 d$deeee   f   dedefd„Zd„ Zd defd„Zd„ Zˆ xZS )%r   a¸  
    Improved Multiscale Vision Transformers for Classification and Detection
    Yanghao Li*, Chao-Yuan Wu*, Haoqi Fan, Karttikeya Mangalam, Bo Xiong, Jitendra Malik,
        Christoph Feichtenhofer*
    https://arxiv.org/abs/2112.01526

    Multiscale Vision Transformers
    Haoqi Fan*, Bo Xiong*, Karttikeya Mangalam*, Yanghao Li*, Zhicheng Yan, Jitendra Malik,
        Christoph Feichtenhofer*
    https://arxiv.org/abs/2104.11227
    ÚcfgÚimg_sizeÚin_chansÚglobal_poolÚnum_classesÚdrop_path_rateÚ	drop_ratec
                 óZ	  •— t         ‰| �  «        ||	dœ}
t        |«      }t        t	        |j
                  «      |j                  ¬«      }|| _        || _        || _	        |€|j                  rdnd}|| _        t        |j                  «      | _        |j                  | _        |j                  d   }t!        d#|||j"                  |j$                  |j&                  dœ|
¤Ž| _        |d   |j$                  d   z  |d   |j$                  d   z  f}t+        |«      }|j                  r=t-        j.                  t1        j2                  dd|fi |
¤Ž«      | _        d| _        |dz   }nd| _        d | _        |}|j8                  r1t-        j.                  t1        j2                  d||fi |
¤Ž«      | _        nd | _        t=        |j                  «      }|}t?        |j$                  «      }tA        ||j                  d¬	«      }t-        jB                  «       | _"        g | _#        tI        |«      D �]†  }|j                  r|j                  |   }n|j                  tK        |dz   |dz
  «         }tM        d#i d
|“d|“d|j                  |   “d|jN                  |   “d|“d|jP                  “d|jR                  “d|jT                  “d|jV                  “d|j                  “d|jX                  “d|jX                  “d|jZ                  |   “d|j\                  |   “d|j                  “d|j^                  “d|j`                  “d|“d||   “|
¤Ž}|t?        |jZ                  |   «      z  }| xjF                  tc        d|› �||¬«      gz  c_#        |}|jd                  }| jD                  jg                  |«       �Œ‰ |x| _4        | _5         ||fi |
¤Ž| _6        t-        jn                  tq        dt-        jr                  | j                  «      fd |dkD  r!t-        jt                  | jh                  |fi |
¤Žnt-        jv                  «       fg«      «      | _<        | j:                  �t{        | j:                  d!¬"«       | j4                  �t{        | j4                  d!¬"«       | j}                  | j~                  «       y )$Nry   )ÚepsÚtokenrË   r   )rs   rt   ru   rv   rw   r   T)Ú	stagewiserž   rt   r%  r(   r‘   r*   r-   r2   r+   r,   rÃ   rÄ   r7   r8   r’   rD   r0   rH   r  zblock.)ÚmoduleÚnum_chsÚ	reductionÚdropÚfcrÏ   rÐ   rj   )@r}   r~   r   r   r   rH   rI   r6  r4  r8  r.   r5  rX   r%   r,   r'   rr   r>   r?   r@   Úpatch_embedrp   r   rá   r�   râ   Ú	cls_tokenÚnum_prefix_tokensr/   Ú	pos_embedrV   rA   r   r(  ÚstagesÚfeature_inforZ   r[   r$  r(   r*   r-   r2   r+   r4   r7   r8   rD   r0   Údictr‘   r\   Únum_featuresÚhead_hidden_sizeÚnormÚ
Sequentialr   ÚDropoutrÔ   r  Úheadr   ÚapplyÚ_init_weights)rP   r2  r3  r4  r5  r6  r7  r8  rz   r{   r�   rH   r'   Ú
patch_dimsÚnum_patchesÚpos_embed_dimr]   r‘   Úcurr_strideÚdprrO   rt   Ústager‚   s                          €rQ   r~   zMultiScaleVit.__init__Ø  si  ø€ ô 	‰ÑÔØ¨Ñ/ˆÜ˜XÓ&ˆÜœ^¨C¯N©NÓ;ÀÇÁÔNˆ
Ø&ˆÔØ ˆŒØ"ˆŒØÐØ%(×%6Ò%6™'¸EˆKØ&ˆÔÜ˜CŸJ™JÓ'ˆŒØŸ?™?ˆÔà—M‘M !Ñ$ˆ	Ü%ð 
ØØØ×#Ñ#Ø×#Ñ#Ø×%Ñ%ñ
ð ñ
ˆÔð ˜q‘k S×%5Ñ%5°aÑ%8Ñ8¸(À1¹+È×IYÑIYÐZ[ÑI\Ñ:\Ð]ˆ
Ü˜:Ó&ˆà×ÒÜŸ\™\¬%¯+©+°a¸¸IÑ*LÈÑ*LÓMˆDŒNØ%&ˆDÔ"Ø'¨!™O‰Mà%&ˆDÔ"Ø!ˆDŒNØ'ˆMà�?Š?ÜŸ\™\¬%¯+©+°a¸È	Ñ*XÐUWÑ*XÓYˆD�Nà!ˆDŒNä˜Ÿ™Ó'ˆ
Øˆ	Ü˜#×*Ñ*Ó+ˆÜ'¨¸¿
¹
ÈdÔSˆÜ—m‘m“oˆŒØˆÔÜ�zÓ"ó 	&ˆAØ�ŠØŸ-™-¨Ñ*‘àŸ-™-¬¨A°©E°:À±>Ó(BÑC�Ü&ò Ùðáðð —j‘j ’mðð Ÿ-™-¨Ò*ð	ñ
 $ðð Ÿ-š-ðð Ÿšðð —X’Xðð Ÿ>š>ðð  ŸOšOðð Ÿšðð Ÿ.š.ðð Ÿ™ ašðð Ÿ-™-¨Ò*ðð "×/Ò/ðð  !×-Ò-ð!ð" "%×!5Ò!5ð#ñ$ &ð%ð& ˜aš&Øñ)ˆEð, œ3˜sŸ|™|¨A™Ó/Ñ/ˆKØ×Ò¤$°°q°c¨lÀGÐWbÔ"cÐ!dÑdÕØˆIØŸ™ˆIØ�K‰K×Ñ˜uÖ%ð?	&ðB 5>Ð=ˆÔ˜DÔ1Ù˜yÑ/¨BÑ/ˆŒ	Ü—M‘M¤+Ø”R—Z‘Z §¡Ó/Ð0ØÀkÐTUÂo”2—9‘9˜T×.Ñ.°ÑB¸rÒBÔ[]×[fÑ[fÓ[hÐið/
ó #ó ˆŒ	ð
 �>‰>Ð%Ü˜TŸ^™^°Õ6Ø�>‰>Ð%Ü˜TŸ^™^°Õ6Ø�
‰
�4×%Ñ%Õ&rk   c                 ó  — t        |t        j                  «      rjt        |j                  d¬«       t        |t        j                  «      r8|j
                  �+t        j                  j                  |j
                  d«       y y y y )NrÏ   rÐ   r  )rW   r   rÔ   r   ÚweightrÇ   ÚinitÚ	constant_)rP   Úms     rQ   rP  zMultiScaleVit._init_weights?  sZ   € Ü�aœŸ™Ô#Ü˜QŸX™X¨4Õ0Ü˜!œRŸY™YÔ'¨A¯F©FÐ,>Ü—‘×!Ñ! !§&¡&¨#Õ.ð -?Ð'ð $rk   c                 ó€   ‡— | j                  «       D �‡�ch c]  \  Š}t        ˆfd„dD «       «      r‰’Œ c}}S c c}}w )Nc              3   ó&   •K  — | ]  }|‰v –— Œ
 y ­wr  rj   )rN   ÚnrÕ   s     €rQ   rR   z0MultiScaleVit.no_weight_decay.<locals>.<genexpr>H  s   øè ø€ Ò\ !�q˜A”vÑ\ùs   ƒ)rE  r¦   r§   rC  )Únamed_parametersÚany)rP   rÕ   rï   s    ` rQ   Úno_weight_decayzMultiScaleVit.no_weight_decayE  sH   ø€ à"×3Ñ3Ó5÷ ^ð ^‘d�a˜ÜÓ\Ð'[Ô\Ô\ò ó ^ð 	^ùó ^s   – :c                 ó$   — t        dddg¬«      }|S )Nz^patch_embed)z^stages\.(\d+)N)z^norm)iŸ† )Ústemr)  )rH  )rP   ÚcoarseÚmatchers      rQ   Úgroup_matcherzMultiScaleVit.group_matcherJ  s   € äØ Ø-Ð/CÐDô
ˆð ˆrk   c                 ó4   — | j                   D ]	  }||_        Œ y r  )rF  r'  )rP   Úenabler  s      rQ   Úset_grad_checkpointingz$MultiScaleVit.set_grad_checkpointingR  s   € à—‘ò 	*ˆAØ#)ˆAÕ ñ	*rk   rƒ   c                 ó.   — | j                   j                  S r  )rN  rA  )rP   s    rQ   Úget_classifierzMultiScaleVit.get_classifierW  s   € à�y‰y�|‰|Ðrk   c                 óT  — || _         |�|| _        t        | j                  j                  d«      r*| j                  j                  j
                  j                  nd }t        | j                  j                  d«      r*| j                  j                  j
                  j                  nd }t        j                  t        dt        j                  | j                  «      fd|dkD  r#t        j                  | j                  |||¬«      nt        j                  «       fg«      «      | _        y )NrX  r@  rA  r   ry   )r6  r5  ÚhasattrrN  rA  rX  rz   r{   r   rL  r   rM  r8  rÔ   rI  r  )rP   r6  r5  rz   r{   s        rQ   Úreset_classifierzMultiScaleVit.reset_classifier[  sÖ   € Ø&ˆÔØÐ"Ø*ˆDÔÜ/6°t·y±y·|±|ÀXÔ/N�—‘—‘×$Ñ$×+Ò+ÐTXˆÜ-4°T·Y±Y·\±\À8Ô-L�—	‘	—‘×#Ñ#×)Ò)ÐRVˆÜ—M‘M¤+Ø”R—Z‘Z §¡Ó/Ð0ØÐ\gÐjkÒ\k”2—9‘9˜T×.Ñ.°ÀFÐRWÕXÔqs×q|Ñq|Óq~Ðð/
ó #ó ˆ�	rk   rŠ   ÚindicesrK  Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 ó  — |dv sJ d«       ‚|dk(  }g }t        t        | j                  «      |«      \  }	}
| j                  |«      \  }}|j                  d   }| j
                  �6| j
                  j                  |dd«      }t        j                  ||fd¬«      }| j                  �|| j                  z   }t        | j                  «      dz
  }t        | j                  «      D ]‡  \  }} |||«      \  }}||	v sŒ|r||k(  r| j                  |«      }n|}|rC| j
                  �|dd…dd…f   }|j                  ||d   |d   d«      j                  dd	dd
«      }|j                  |«       Œ‰ |r|S |k(  r| j                  |«      }||fS )aè   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )ÚNCHWÚNLCz!Output shape must be NCHW or NLC.rt  r   Nr”   r   r�   r#   r"   )r   rV   rF  rB  rˆ   rC  Úexpandr�   rŸ   rE  Ú	enumeraterK  r•   r–   r\   )rP   rŠ   ro  rK  rp  rq  rr  r•   ÚintermediatesÚtake_indicesÚ	max_indexr‘   r»   Ú
cls_tokensÚlast_idxÚfeat_idxrV  Úx_inters                     rQ   Úforward_intermediatesz#MultiScaleVit.forward_intermediatesf  s›  € ð* ˜_Ñ,ÐQÐ.QÓQÐ,Ø Ñ&ˆØˆÜ"6´s¸4¿;¹;Ó7GÈÓ"QÑˆ�ið ×'Ñ'¨Ó*‰ˆˆ9Ø�G‰G�A‰JˆØ�>‰>Ð%ØŸ™×.Ñ.¨q°"°bÓ9ˆJÜ—	‘	˜: q˜/¨qÔ1ˆAØ�>‰>Ð%Ø�D—N‘NÑ"ˆAä�t—{‘{Ó# aÑ'ˆÜ(¨¯©Ó5ò 	.‰OˆH�eÙ   IÓ.‰LˆAˆyØ˜<Ò'Ù˜H¨Ò0Ø"Ÿi™i¨›l‘Gà�GÙØ—~‘~Ð1à")ª!¨Q©R¨%¡.˜Ø%Ÿo™o¨a°¸1±¸yÈ¹|ÈRÓP×XÑXÐYZÐ\]Ð_`ÐbcÓd�GØ×$Ñ$ WÕ-ð	.ñ Ø Ð à�xÒØ—	‘	˜!“ˆAà�-ÐÐrk   Ú
prune_normÚ
prune_headc                 ó¨   — t        t        | j                  «      |«      \  }}|rt        j                  «       | _        |r| j                  dd«       |S )z@ Prune layers not required for specified intermediates.
        r   Ú )r   rV   rF  r   r  rK  rn  )rP   ro  r€  r�  ry  rz  s         rQ   Úprune_intermediate_layersz'MultiScaleVit.prune_intermediate_layers¡  sI   € ô #7´s¸4¿;¹;Ó7GÈÓ"QÑˆ�iñ ÜŸ™›ˆDŒIÙØ×!Ñ! ! RÔ(ØÐrk   c                 ód  — | j                  |«      \  }}|j                  \  }}}| j                  �6| j                  j                  |dd«      }t	        j
                  ||fd¬«      }| j                  �|| j                  z   }| j                  D ]  } |||«      \  }}Œ | j                  |«      }|S )Nr”   r   r�   )	rB  rˆ   rC  rv  r�   rŸ   rE  rF  rK  )rP   rŠ   r‘   r»   rî   r  r{  rV  s           rQ   Úforward_featureszMultiScaleVit.forward_features²  sª   € Ø×'Ñ'¨Ó*‰ˆˆ9Ø—'‘'‰ˆˆ1ˆaà�>‰>Ð%ØŸ™×.Ñ.¨q°"°bÓ9ˆJÜ—	‘	˜: q˜/¨qÔ1ˆAà�>‰>Ð%Ø�D—N‘NÑ"ˆAà—[‘[ò 	/ˆEÙ   IÓ.‰LˆA‰yð	/ð �I‰I�a‹LˆØˆrk   Ú
pre_logitsc                 ó¾   — | j                   r=| j                   dk(  r%|d d …| j                  d …f   j                  d«      }n	|d d …df   }|r|S | j                  |«      S )NrË   r   r   )r5  rD  ÚmeanrN  )rP   rŠ   r‡  s      rQ   Úforward_headzMultiScaleVit.forward_headÃ  s^   € Ø×ÒØ×Ñ 5Ò(Ø’a˜×/Ñ/Ñ0Ð0Ñ1×6Ñ6°qÓ9‘à’a˜�d‘G�ÙˆqÐ0 D§I¡I¨a£LÐ0rk   c                 óJ   — | j                  |«      }| j                  |«      }|S r  )r†  rŠ  r‰   s     rQ   r‹   zMultiScaleVit.forwardË  s'   € Ø×!Ñ! !Ó$ˆØ×Ñ˜aÓ ˆØˆrk   ))éà   rŒ  r#   Néè  r  r  NN©F©Tr  )NFFrt  F)r   FT) rb   rc   rd   rŒ   r   r	   re   r
   ri   rg   r~   rP  r�   r.  Úignorera  rf  ri  r   rù   rk  rn  rŽ   r   r   rh   r  r„  r†  rŠ  r‹   r�   r�   s   @rQ   r   r   Ë  s  ø„ ñ
ð )3ØØ)-Ø#Ø$&Ø!ØØñe'à!ðe'ð ˜C ˜H‘oðe'ð ð	e'ð
 " #™ðe'ð ðe'ð "ðe'ð õe'òN/ð ‡Y�Y×Ññ^ó ð^ð ‡Y�Y×Ñòó ðð ‡Y�Y×Ñò*ó ð*ð ‡Y�Y×Ñð §	¡	ò ó ðñ	¨Cð 	¸hÀs¹mó 	ð 8<ØØ$Ø$Ø',ñ9 à�|‰|ð9 ð ˜e C¨¨c© NÑ3Ñ4ð9 ð ð	9 ð
 ð9 ð ð9 ð !%ð9 ð 
ˆt�E—L‘LÑ! 5¨¯©°t¸E¿L¹LÑ7IÐ)IÑ#JÐJÑ	Kó9 ðz ./Ø$Ø#ñ	à˜3  S¡	˜>Ñ*ðð ðð ó	ò"ñ"1¨$ó 1örk   c           
      ó²  ‡— d| v rÚ| j                  «       D ]Å  }d|v sŒ| |   }|j                  «       |   j                  }|j                  d   |d   k7  sŒ@t        j                  j
                  j                  |j                  d|j                  d   d«      j                  ddd«      |d   d¬«      }|j                  d|d   «      j                  dd«      | |<   ŒÇ | S dd l	}d	| v r| d	   } t        |d
d «      }t        |dd«      }|€J d«       ‚i Šd}	t        |«      D ]<  \  }
}‰j                  t        |	|	|z   «      D �ci c]
  }||
||	z
  f“Œ c}«       |	|z  }	Œ> i }| j                  «       D ]_  \  }}|j                  dˆfd„|«      }|r|j                  dd|«      }n|j                  dd|«      }d|v r|j!                  dd«      }|||<   Œa |S c c}w )Nzstages.0.blocks.0.norm1.weightÚrel_posr   r   r”   r"   Úlinear)ré   r2   Úmodel_stater%   r,   Tz3model requires depth attribute to remap checkpointszblocks\.(\d+)c           	      óŒ   •— d‰t        | j                  d«      «         d   › d‰t        | j                  d«      «         d   › �S )Nzstages.r   r   z.blocks.)re   Úgroup)rŠ   Ú	depth_maps    €rQ   ú<lambda>z&checkpoint_filter_fn.<locals>.<lambda>ò  sJ   ø€ ˜ 	¬#¨a¯g©g°a«j«/Ñ :¸1Ñ =Ð>¸hÀyÔQTÐUV×U\ÑU\Ð]^ÓU_ÓQ`ÑGaÐbcÑGdÐFeÐf€ rk   z stages\.(\d+).blocks\.(\d+).projz&stages.\1.blocks.\2.shortcut_proj_attnz%stages.\1.blocks.\2.shortcut_proj_mlprN  zhead.projectionúhead.fc)ÚkeysÚ
state_dictrˆ   r�   r   Ú
functionalÚinterpolater•   r–   ÚreÚgetattrrw  ÚupdaterZ   ÚitemsÚsubÚreplace)r›  ÚmodelrÕ   r’  Údest_rel_pos_shapeÚrel_pos_resizedrž  r%   r,   Ú	block_idxÚ	stage_idxr`   rO   Úout_dictrÖ   r—  s                  @rQ   Úcheckpoint_filter_fnrª  Ñ  s'  ø€ Ø'¨:Ñ5à—‘Ó"ò 
	eˆAØ˜AŠ~Ø$ Q™-�Ø%*×%5Ñ%5Ó%7¸Ñ%:×%@Ñ%@Ð"Ø—=‘= Ñ#Ð'9¸!Ñ'<Ó<Ü&+§h¡h×&9Ñ&9×&EÑ&EØŸ™¨¨7¯=©=¸Ñ+;¸RÓ@×HÑHÈÈAÈqÓQØ/°Ñ2Ø%ð 'Fó '�Oð
 %4×$;Ñ$;¸BÐ@RÐSTÑ@UÓ$V×$^Ñ$^Ð_`ÐbcÓ$d�J˜q’Mð
	eð ÐãØ˜
Ñ"Ø Ñ.ˆ
ä�U˜H dÓ+€FÜ˜% °Ó5€KØÐÐTÐTÓTÐØ€IØ€IÜ! &Ó)ò ‰ˆ	�1Ø×ÑÄÀyÐR[Ð^_ÑR_ÓA`Öa¸A˜!˜i¨¨Y©Ð7Ñ7ÒaÔbØ�Q‰‰	ðð €HØ× Ñ Ó"ò ‰ˆˆ1Ø�F‰FØÛfØóˆñ
 Ø—‘Ð:Ð>fÐijÓk‰Aà—‘Ð:Ð>eÐhiÓjˆAØ�Q‰;Ø—	‘	Ð+¨YÓ7ˆAØˆ�Šðð €Oùò% bs   ÅG
)r   r"   é   r"   )r%   )r   r"   é   r"   r!   )r"   é   é$   r:   é�   r"   F)r%   r'   r(   r,   )r%   r.   )r%   r'   r(   r.   r,   )r:   rÂ   é<   rÂ   éÀ   r#   )Úmvitv2_tinyÚmvitv2_smallÚmvitv2_baseÚmvitv2_largeÚmvitv2_small_clsÚmvitv2_base_clsÚmvitv2_large_clsÚmvitv2_huge_clsc           	      ó”   — |j                  dd«      }t        t        | |f|s	t        |    nt        |   t        t        |d¬«      dœ|¤ŽS )NÚout_indicesr:   Úgetter)r»  Úfeature_cls)Ú	model_cfgÚpretrained_filter_fnÚfeature_cfg)Úpopr   r   Ú
model_cfgsrª  rH  )ÚvariantÚcfg_variantÚ
pretrainedÚkwargsr»  s        rQ   Ú_create_mvitv2rÇ  *  sW   € Ø—*‘*˜]¨AÓ.€KÜÜØØðñ .9”*˜WÒ%¼jÈÑ>UÜ1Ü [¸hÔGñð ñð rk   rƒ  c                 ó4   — | ddd ddt         t        ddddd	œ|¥S )
Nr�  )r#   rŒ  rŒ  gÍÌÌÌÌÌì?Úbicubiczpatch_embed.projr™  Tz
apache-2.0)Úurlr6  Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationr‰  rÑ   Ú
first_convÚ
classifierÚfixed_input_sizeÚlicenser   )rÊ  rÆ  s     rQ   Ú_cfgrÓ  7  s6   € àØ¨=ÀtØ¨Ü%Ô.BØ(¸	Ø Øñ	ð ð	ð 	rk   zDhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_T_in1k.pythztimm/)rÊ  Ú	hf_hub_idzDhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_S_in1k.pythzDhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_B_in1k.pythzDhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_L_in1k.pyth)rÊ  zEhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_B_in21k.pythiàJ  )rÊ  rÔ  r6  zEhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_L_in21k.pythzEhttps://dl.fbaipublicfiles.com/mvit/mvitv2_models/MViTv2_H_in21k.pyth)zmvitv2_tiny.fb_in1kzmvitv2_small.fb_in1kzmvitv2_base.fb_in1kzmvitv2_large.fb_in1kr¶  zmvitv2_base_cls.fb_inw21kzmvitv2_large_cls.fb_inw21kzmvitv2_huge_cls.fb_inw21kc                 ó   — t        dd| i|¤ŽS )NrÅ  )r²  ©rÇ  ©rÅ  rÆ  s     rQ   r²  r²  _  ó   € äÑI°JÐIÀ&ÑIÐIrk   c                 ó   — t        dd| i|¤ŽS )NrÅ  )r³  rÖ  r×  s     rQ   r³  r³  d  ó   € äÑJ°ZÐJÀ6ÑJÐJrk   c                 ó   — t        dd| i|¤ŽS )NrÅ  )r´  rÖ  r×  s     rQ   r´  r´  i  rØ  rk   c                 ó   — t        dd| i|¤ŽS )NrÅ  )rµ  rÖ  r×  s     rQ   rµ  rµ  n  rÚ  rk   c                 ó   — t        dd| i|¤ŽS )NrÅ  )r¶  rÖ  r×  s     rQ   r¶  r¶  s  ó   € äÑN¸ÐNÀvÑNÐNrk   c                 ó   — t        dd| i|¤ŽS )NrÅ  )r·  rÖ  r×  s     rQ   r·  r·  x  ó   € äÑM¸
ÐMÀfÑMÐMrk   c                 ó   — t        dd| i|¤ŽS )NrÅ  )r¸  rÖ  r×  s     rQ   r¸  r¸  }  rÞ  rk   c                 ó   — t        dd| i|¤ŽS )NrÅ  )r¹  rÖ  r×  s     rQ   r¹  r¹  ‚  rà  rk   r�  r  )NF)rƒ  rŽ  )ErŒ   rm   Úcollectionsr   Údataclassesr   Ú	functoolsr   r   Útypingr   r   r	   r
   r   r   r�   r   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_features_fxr   Ú_manipulater   Ú	_registryr   r   Ú__all__r   rp   rù   rr   re   rh   rŽ   r›   r¡   r¿   rÁ   rû   r  r$  r   rª  rH  rÂ  rÇ  rÓ  Údefault_cfgsr²  r³  r´  rµ  r¶  r·  r¸  r¹  rj   rk   rQ   ú<module>rð     s  ðñó Ý #Ý !ß %ß :× :ã Ý ç Aß m× mÝ *Ý +Ý 3Ý #ß <àÐ.Ð
/€ð ÷-3ð -3ó ð-3ò`-ô:�—‘ô :ðB ð #ñà˜‘9ðð ðð ˆ5�<‰<˜ %§,¡,Ñ/Ð/Ñ0ò	ó ðð ð +/ñ
àð
ð ˜%Ÿ,™,Ñ'ð
ð ˆ5�<‰<˜˜c™Ð"Ñ#ò	
ó ð
ð ð/Ø�l‰lð/à�<‰<ð/ð ð/ð �S‘	ð	/ð
 �S‘	ð/ð —<‘<ð/ð —<‘<ò/ó ð/ôdm 2§9¡9ô mô`\˜"Ÿ)™)ô \ô~e �b—i‘iô e ôPG˜Ÿ™ô GôTC�B—I‘Iô CòL,ñ^ Ù Øôñ "Øôñ !Øôñ "ØØØØô	ñ &ØØôñ %ØØôñ &ØØØØØôñ %ØØØØØôôA'€
óT
ó
ñ %ÙØRØôñ !Ð%kØôáÐ$jØôá Ð%kØôñ  œÙ!%ØSØØô"ñ #'ØSØØô#ñ "&ØSØØô"ñ)&ó €ð6 ñJ¨}ò Jó ðJð ñK°ò Kó ðKð ñJ¨}ò Jó ðJð ñK°ò Kó ðKð ñO°Mò Oó ðOð ñN°=ò Nó ðNð ñO°Mò Oó ðOð ñN°=ò Nó ñNrk   