Ë
    èÓ^jl  ã                   ór  — d dl mZ d dlmZmZmZ d dlZd dlmc m	Z
 d dlmZmZ ddlmZ ddlmZ dd	lmZmZmZ dd
lmZ ddlmZmZ ddlmZmZ g d¢Zdee   dee   dee   deee   ee   f   fd„Z ejB                  jE                  d«       dej                  dej                  dee   defd„Z#ejB                  jE                  d«       deeeef   deeeef   deeeef   fd„Z$ejB                  jE                  d«       dedeeeef   deeeef   deeeef   def
d„Z%ejB                  jE                  d«       	 	 	 	 	 dGded ed!ed"edee   d#edee   d$e&d%e&d&ee   d'ee   d(e'defd)„Z(ejB                  jE                  d*«        G d+„ d,ejR                  «      Z* G d-„ d.ejR                  «      Z+ G d/„ d0ejR                  «      Z,dee   d1ed2ee   d#ee   dee   d3e&d4ee   d5e'd6ede,fd7„Z-ed8d9d:œZ. G d;„ d<e«      Z/ G d=„ d>e«      Z0 G d?„ d@e«      Z1 e«        edAe/jd                  f¬B«      dddCœd4ee/   d5e'd6ede,fdD„«       «       Z3 e«        edAe0jd                  f¬B«      dddCœd4ee0   d5e'd6ede,fdE„«       «       Z4 e«        edAe1jd                  f¬B«      dddCœd4ee1   d5e'd6ede,fdF„«       «       Z5y)Hé    )Úpartial)ÚAnyÚCallableÚOptionalN)ÚnnÚTensoré   )ÚVideoClassification)Ú_log_api_usage_onceé   )Úregister_modelÚWeightsÚWeightsEnum)Ú_KINETICS400_CATEGORIES)Ú_ovewrite_named_paramÚhandle_legacy_interface)ÚPatchMergingÚSwinTransformerBlock)ÚSwinTransformer3dÚSwin3D_T_WeightsÚSwin3D_S_WeightsÚSwin3D_B_WeightsÚswin3d_tÚswin3d_sÚswin3d_bÚ
shift_sizeÚsize_dhwÚwindow_sizeÚreturnc                 ó\   — t        d«      D ]  }||   ||   k  sŒ||   ||<   d| |<   Œ || fS )Nr	   r   ©Úrange)r   r   r   Úis       út/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/torchvision/models/video/swin_transformer.pyÚ_get_window_and_shift_sizer%       sJ   € ô �1‹Xò ˆØ�A‰;˜+ a™.Ó(à% a™[ˆK˜‰NØˆJ�qŠMð	ð ˜
Ð"Ð"ó    r%   Úrelative_position_bias_tableÚrelative_position_indexc                 óæ   — |d   |d   z  |d   z  }| |d |…d |…f   j                  «          }|j                  ||d«      }|j                  ddd«      j                  «       j	                  d«      }|S )Nr   é   r   éÿÿÿÿ)ÚflattenÚviewÚpermuteÚ
contiguousÚ	unsqueeze)r'   r(   r   Ú
window_volÚrelative_position_biass        r$   Ú_get_relative_position_biasr3   /   s�   € ð ˜Q‘ +¨a¡.Ñ0°;¸q±>ÑA€Jà9Ø   ¨[¨j¨[Ð 8Ñ9×AÑAÓCñÐð 4×8Ñ8¸ÀZÐQSÓTÐØ3×;Ñ;¸A¸qÀ!ÓD×OÑOÓQ×[Ñ[Ð\]Ó^ÐØ!Ð!r&   r3   Ú
patch_sizec                 ó‚   — t        d«      D �cg c]  }||   | |   ||   z  z
  ||   z  ‘Œ }}|d   |d   |d   fS c c}w )Nr	   r   r*   r   r!   )r   r4   r#   Úpad_sizes       r$   Ú_compute_pad_size_3dr7   ?   sZ   € ÜW\Ð]^ÓW_Ö`ÐRS�˜A‘ ¨!¡¨z¸!©}Ñ!<Ñ<À
È1ÁÓMÐ`€HÐ`Ø�A‰;˜ ™ X¨a¡[Ð0Ð0ùò as   Ž<r7   Úxc           
      óø  —  | j                   |Ž }|d   |d   z  |d   |d   z  z  |d   |d   z  z  }t        d«      D �cg c]  }d||    f||    ||    f||    d ff‘Œ }}d}|d   D ];  }	|d   D ]1  }
|d   D ]'  }|||	d   |	d   …|
d   |
d   …|d   |d   …f<   |dz  }Œ) Œ3 Œ= |j                  |d   |d   z  |d   |d   |d   z  |d   |d   |d   z  |d   «      }|j                  dddddd«      j	                  ||d   |d   z  |d   z  «      }|j                  d«      |j                  d«      z
  }|j                  |dk7  t        d«      «      j                  |dk(  t        d«      «      }|S c c}w )	Nr   r*   r   r	   é   é   g      YÀç        )Ú	new_zerosr"   r-   r.   Úreshaper0   Úmasked_fillÚfloat)r8   r   r   r   Ú	attn_maskÚnum_windowsr#   ÚslicesÚcountÚdÚhÚws               r$   Ú_compute_attention_mask_3drH   G   s  € ð �—‘˜XÐ&€IØ˜A‘; +¨a¡.Ñ0°X¸a±[ÀKÐPQÁNÑ5RÑSÐW_Ð`aÑWbÐfqÐrsÑftÑWtÑu€Kô �q“öð ð	 �˜Q‘�Ð Ø˜!‰nˆ_˜z¨!™}˜nÐ-Ø˜‰mˆ^˜TÐ"ò	
ð€Fð ð €EØ�A‰Yò ˆØ˜‘ò 	ˆAØ˜A‘Yò �ØCH�	˜!˜A™$  1¡˜+ q¨¡t¨a°©d {°A°a±D¸1¸Q¹4°KÐ?Ñ@Ø˜‘
‘ññ	ðð —‘Ø�‰�{ 1‘~Ñ%Ø�A‰Ø�‰�{ 1‘~Ñ%Ø�A‰Ø�‰�{ 1‘~Ñ%Ø�A‰ó€Ið ×!Ñ! ! Q¨¨1¨a°Ó3×;Ñ;Ø�[ ‘^ k°!¡nÑ4°{À1±~ÑEó€Ið ×#Ñ# AÓ&¨×)<Ñ)<¸QÓ)?Ñ?€IØ×%Ñ% i°1¡n´e¸F³mÓD×PÑPÐQZÐ^_ÑQ_ÔafÐgjÓakÓl€IØÐùò;s   Á "E7rH   TÚinputÚ
qkv_weightÚproj_weightr2   Ú	num_headsÚattention_dropoutÚdropoutÚqkv_biasÚ	proj_biasÚtrainingc                 ó@  — | j                   \  }}}}}t        |||f|d   |d   |d   f«      }t        j                  | ddd|d   d|d   d|d   f«      }|j                   \  }}}}}|||f}t	        |«      dkD  r't        j                  ||d    |d    |d    fd¬«      }|d   |d   z  |d   |d   z  z  |d   |d   z  z  }|j                  ||d   |d   z  |d   |d   |d   z  |d   |d   |d   z  |d   |«      }|j                  ddddddd	d
«      j                  ||z  |d   |d   z  |d   z  |«      }t        j                  |||	«      }|j                  |j                  d«      |j                  d«      d|||z  «      j                  ddddd«      }|d   |d   |d   }}}|||z  dz  z  }|j                  |j                  dd«      «      }||z   }t	        |«      dkD  rÌt        ||d   |d   |d   f|d   |d   |d   f|d   |d   |d   f«      }|j                  |j                  d«      |z  |||j                  d«      |j                  d«      «      }||j                  d«      j                  d«      z   }|j                  d||j                  d«      |j                  d«      «      }t        j                   |d¬«      }t        j"                  |||¬«      }|j                  |«      j                  dd«      j                  |j                  d«      |j                  d«      |«      }t        j                  |||
«      }t        j"                  |||¬«      }|j                  ||d   |d   z  |d   |d   z  |d   |d   z  |d   |d   |d   |«      }|j                  ddddddd	d
«      j                  |||||«      }t	        |«      dkD  r$t        j                  ||d   |d   |d   fd¬«      }|dd…d|…d|…d|…dd…f   j%                  «       }|S )aÊ  
    Window based multi-head self attention (W-MSA) module with relative position bias.
    It supports both of shifted and non-shifted window.
    Args:
        input (Tensor[B, T, H, W, C]): The input tensor, 5-dimensions.
        qkv_weight (Tensor[in_dim, out_dim]): The weight tensor of query, key, value.
        proj_weight (Tensor[out_dim, out_dim]): The weight tensor of projection.
        relative_position_bias (Tensor): The learned relative position bias added to attention.
        window_size (List[int]): 3-dimensions window size, T, H, W .
        num_heads (int): Number of attention heads.
        shift_size (List[int]): Shift size for shifted window attention (T, H, W).
        attention_dropout (float): Dropout ratio of attention weight. Default: 0.0.
        dropout (float): Dropout ratio of output. Default: 0.0.
        qkv_bias (Tensor[out_dim], optional): The bias tensor of query, key, value. Default: None.
        proj_bias (Tensor[out_dim], optional): The bias tensor of projection. Default: None.
        training (bool, optional): Training flag used by the dropout parameters. Default: True.
    Returns:
        Tensor[B, T, H, W, C]: The output tensor after shifted window attention.
    r   r*   r   )r*   r   r	   )ÚshiftsÚdimsr	   r;   r:   é   é   g      à¿éþÿÿÿr+   )Údim)ÚprQ   N)Úshaper7   ÚFÚpadÚsumÚtorchÚrollr-   r.   r>   ÚlinearÚsizeÚmatmulÚ	transposerH   r0   ÚsoftmaxrN   r/   )rI   rJ   rK   r2   r   rL   r   rM   rN   rO   rP   rQ   ÚbÚtrF   rG   Úcr6   r8   Ú_ÚtpÚhpÚwpÚpadded_sizerB   ÚqkvÚqÚkÚvÚattnrA   s                                  r$   Úshifted_window_attention_3drr   s   s�  € ðB —K‘K�M€A€qˆ!ˆQ�ä# Q¨¨1 I°¸A±ÀÈAÁÐP[Ð\]ÑP^Ð/_Ó`€HÜ	�‰ˆe�a˜˜A˜x¨™{¨A¨x¸©{¸A¸xÈ¹{ÐKÓL€AØ—w‘wÑ€A€rˆ2ˆr�1Ø�r˜2�,€Kô ˆ:ƒ˜ÒÜ�J‰J�q :¨a¡= .°:¸a±=°.À:ÈaÁ=À.Ð!QÐXaÔbˆð 
�Q‰˜; q™>Ñ	)¨k¸!©nÀÈAÁÑ.NÑOÐS^Ð_`ÑSaÐepÐqrÑesÑSsÑtð ð 	
�‰Ø	Ø�A‰˜+ a™.Ñ(Ø�A‰Ø�A‰˜+ a™.Ñ(Ø�A‰Ø�A‰˜+ a™.Ñ(Ø�A‰Ø	ó		€Að 	
�	‰	�!�Q˜˜1˜a  A qÓ)×1Ñ1Ø	ˆK‰˜ Q™¨+°a©.Ñ8¸;Àq¹>ÑIÈ1ó	€Aô
 �(‰(�1�j (Ó
+€CØ
�+‰+�a—f‘f˜Q“i §¡¨£¨A¨y¸!¸y¹.Ó
I×
QÑ
QÐRSÐUVÐXYÐ[\Ð^_Ó
`€CØ�!‰f�c˜!‘f˜c !™fˆ!€q€AØ	ˆQ�)‰^ Ñ$Ñ$€AØ�8‰8�A—K‘K  BÓ'Ó(€DàÐ(Ñ(€Dä
ˆ:ƒ˜Òä.ØØ˜‰^˜[¨™^¨[¸©^Ð<Ø˜‰^˜[¨™^¨[¸©^Ð<Ø˜‰]˜J q™M¨:°a©=Ð9ó	
ˆ	ð �y‰y˜Ÿ™ › kÑ1°;À	È1Ï6É6ÐRSË9ÐVW×V\ÑV\Ð]^ÓV_Ó`ˆØ�i×)Ñ)¨!Ó,×6Ñ6°qÓ9Ñ9ˆØ�y‰y˜˜Y¨¯©¨q«	°1·6±6¸!³9Ó=ˆä�9‰9�T˜rÔ"€DÜ�9‰9�TÐ.¸ÔB€Dà�‰�A‹× Ñ   AÓ&×.Ñ.¨q¯v©v°a«y¸!¿&¹&À»)ÀQÓG€AÜ	�‰��K Ó+€AÜ	�	‰	�!�w¨Ô2€Að 	
�‰Ø	Ø�A‰˜+ a™.Ñ(Ø�A‰˜+ a™.Ñ(Ø�A‰˜+ a™.Ñ(Ø�A‰Ø�A‰Ø�A‰Ø	ó		€Að 	
�	‰	�!�Q˜˜1˜a  A qÓ)×1Ñ1°!°R¸¸RÀÓC€Aô ˆ:ƒ˜ÒÜ�J‰J�q *¨Q¡-°¸A±À
È1ÁÐ!NÐU^Ô_ˆð 	
Š!ˆRˆaˆR��!��R�a�RšÐ
Ñ×&Ñ&Ó(€AØ€Hr&   rr   c                   ó¨   ‡ — e Zd ZdZ	 	 	 	 ddedee   dee   dedededed	ed
dfˆ fd„Zdd„Z	dd„Z
dee   d
ej                  fd„Zded
efd„Zˆ xZS )ÚShiftedWindowAttention3dz2
    See :func:`shifted_window_attention_3d`.
    rX   r   r   rL   rO   rP   rM   rN   r   Nc	                 óp  •— t         ‰	| �  «        t        |«      dk7  st        |«      dk7  rt        d«      ‚|| _        || _        || _        || _        || _        t        j                  ||dz  |¬«      | _        t        j                  |||¬«      | _        | j                  «        | j                  «        y )Nr	   z.window_size and shift_size must be of length 2)Úbias)ÚsuperÚ__init__ÚlenÚ
ValueErrorr   r   rL   rM   rN   r   ÚLinearrm   ÚprojÚ#define_relative_position_bias_tableÚdefine_relative_position_index)
ÚselfrX   r   r   rL   rO   rP   rM   rN   Ú	__class__s
            €r$   rx   z!ShiftedWindowAttention3d.__init__ë   s�   ø€ ô 	‰ÑÔÜˆ{Ó˜qÒ ¤C¨
£O°qÒ$8ÜÐMÓNÐNà&ˆÔØ$ˆŒØ"ˆŒØ!2ˆÔØˆŒä—9‘9˜S #¨¡'°Ô9ˆŒÜ—I‘I˜c 3¨YÔ7ˆŒ	à×0Ñ0Ô2Ø×+Ñ+Õ-r&   c                 óH  — t        j                  t        j                  d| j                  d   z  dz
  d| j                  d   z  dz
  z  d| j                  d   z  dz
  z  | j
                  «      «      | _        t         j                  j                  | j                  d¬«       y )Nr   r   r*   ç{®Gáz”?©Ústd)	r   Ú	Parameterr^   Úzerosr   rL   r'   ÚinitÚtrunc_normal_)r   s    r$   r}   z<ShiftedWindowAttention3d.define_relative_position_bias_table  s•   € ä,.¯L©LÜ�K‰KØ�T×%Ñ% aÑ(Ñ(¨1Ñ,°°T×5EÑ5EÀaÑ5HÑ1HÈ1Ñ1LÑMÐQRÐUY×UeÑUeÐfgÑUhÑQhÐklÑQlÑmØ—‘óó-
ˆÔ)ô 	�‰×Ñ˜d×?Ñ?ÀTÐÕJr&   c                 ón  — t        d«      D �cg c]$  }t        j                  | j                  |   «      ‘Œ& }}t        j                  t        j
                  |d   |d   |d   d¬«      «      }t        j                  |d«      }|d d …d d …d f   |d d …d d d …f   z
  }|j                  ddd«      j                  «       }|d d …d d …dfxx   | j                  d   dz
  z  cc<   |d d …d d …dfxx   | j                  d   dz
  z  cc<   |d d …d d …dfxx   | j                  d   dz
  z  cc<   |d d …d d …dfxx   d| j                  d   z  dz
  d| j                  d   z  dz
  z  z  cc<   |d d …d d …dfxx   d| j                  d   z  dz
  z  cc<   |j                  d«      }| j                  d|«       y c c}w )	Nr	   r   r*   r   Úij)Úindexingr+   r(   )r"   r^   Úaranger   ÚstackÚmeshgridr,   r.   r/   r]   Úregister_buffer)r   r#   Ú
coords_dhwÚcoordsÚcoords_flattenÚrelative_coordsr(   s          r$   r~   z7ShiftedWindowAttention3d.define_relative_position_index  s¶  € äAFÀqÃÖJ¸A”e—l‘l 4×#3Ñ#3°AÑ#6Õ7ÐJˆ
ÐJÜ—‘Ü�N‰N˜: a™=¨*°Q©-¸ÀA¹ÐQUÔVó
ˆô Ÿ™ v¨qÓ1ˆØ(ªªA¨t¨Ñ4°~ÂaÈÊqÀjÑ7QÑQˆØ)×1Ñ1°!°Q¸Ó:×EÑEÓGˆØšš1˜a˜Ó  D×$4Ñ$4°QÑ$7¸!Ñ$;Ñ;Ó Øšš1˜a˜Ó  D×$4Ñ$4°QÑ$7¸!Ñ$;Ñ;Ó Øšš1˜a˜Ó  D×$4Ñ$4°QÑ$7¸!Ñ$;Ñ;Ó àšš1˜a˜Ó  Q¨×)9Ñ)9¸!Ñ)<Ñ%<¸qÑ%@ÀQÈ×IYÑIYÐZ[ÑI\ÑE\Ð_`ÑE`Ñ$aÑaÓ Øšš1˜a˜Ó  A¨×(8Ñ(8¸Ñ(;Ñ$;¸aÑ$?Ñ?Ó à"1×"5Ñ"5°bÓ"9ÐØ×ÑÐ6Ð8OÕPùò Ks   Ž)F2c                 óD   — t        | j                  | j                  |«      S )N)r3   r'   r(   )r   r   s     r$   Úget_relative_position_biasz3ShiftedWindowAttention3d.get_relative_position_bias#  s   € Ü*¨4×+LÑ+LÈd×NjÑNjÐlwÓxÐxr&   r8   c                 óø  — |j                   \  }}}}}|||g}| j                  j                  «       | j                  j                  «       }}t	        |||«      \  }}| j                  |«      }	t        || j                  j                  | j                  j                  |	|| j                  || j                  | j                  | j                  j                  | j                  j                  | j                  ¬«      S )N)r   rM   rN   rO   rP   rQ   )rZ   r   Úcopyr   r%   r•   rr   rm   Úweightr|   rL   rM   rN   rv   rQ   )
r   r8   rh   rf   rF   rG   r   r   r   r2   s
             r$   Úforwardz ShiftedWindowAttention3d.forward&  sÏ   € ØŸ™‰ˆˆ1ˆa��AØ�q˜!�9ˆØ"&×"2Ñ"2×"7Ñ"7Ó"9¸4¿?¹?×;OÑ;OÓ;Q�Zˆä"<¸ZÈÐS^Ó"_Ñˆ�Zà!%×!@Ñ!@ÀÓ!MÐä*ØØ�H‰H�O‰OØ�I‰I×ÑØ"ØØ�N‰NØ!Ø"×4Ñ4Ø—L‘LØ—X‘X—]‘]Ø—i‘i—n‘nØ—]‘]ô
ð 	
r&   )TTr<   r<   )r   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚintÚlistÚboolr@   rx   r}   r~   r^   r   r•   r™   Ú__classcell__©r€   s   @r$   rt   rt   æ   s¸   ø„ ñð ØØ#&Øñ.àð.ð ˜#‘Yð.ð ˜‘Ið	.ð
 ð.ð ð.ð ð.ð !ð.ð ð.ð 
õ.ó6KóQð&y°d¸3±ið yÀEÇLÁLó yð
˜ð 
 F÷ 
r&   rt   c                   óz   ‡ — e Zd ZdZ	 	 	 ddee   dededeedej                  f      ddf
ˆ fd	„Z
d
edefd„Zˆ xZS )ÚPatchEmbed3da;  Video to Patch Embedding.

    Args:
        patch_size (List[int]): Patch token size.
        in_channels (int): Number of input channels. Default: 3
        embed_dim (int): Number of linear projection output channels. Default: 96.
        norm_layer (nn.Module, optional): Normalization layer. Default: None
    Nr4   Úin_channelsÚ	embed_dimÚ
norm_layer.r   c                 ó  •— t         ‰| �  «        t        | «       |d   |d   |d   f| _        t	        j
                  ||| j                  | j                  ¬«      | _        |� ||«      | _        y t	        j                  «       | _        y )Nr   r*   r   )Úkernel_sizeÚstride)	rw   rx   r   Útuple_patch_sizer   ÚConv3dr|   ÚnormÚIdentity)r   r4   r¥   r¦   r§   r€   s        €r$   rx   zPatchEmbed3d.__init__K  s|   ø€ ô 	‰ÑÔÜ˜DÔ!Ø!+¨A¡°
¸1±¸zÈ!¹}Ð MˆÔä—I‘IØØØ×-Ñ-Ø×(Ñ(ô	
ˆŒ	ð Ð!Ù" 9Ó-ˆD�IäŸ™›ˆD�Ir&   r8   c           
      ó4  — |j                  «       \  }}}}}t        |||f| j                  «      }t        j                  |d|d   d|d   d|d   f«      }| j                  |«      }|j                  ddddd«      }| j                  �| j                  |«      }|S )zForward function.r   r   r*   r	   r:   )ra   r7   r«   r[   r\   r|   r.   r­   )r   r8   rh   rf   rF   rG   r6   s          r$   r™   zPatchEmbed3d.forwarda  s™   € ð Ÿ™›‰ˆˆ1ˆa��AÜ'¨¨A¨q¨	°4×3HÑ3HÓIˆÜ�E‰E�!�a˜ !™ a¨°!©°a¸À!¹ÐEÓFˆØ�I‰I�a‹LˆØ�I‰I�a˜˜A˜q !Ó$ˆØ�9‰9Ð Ø—	‘	˜!“ˆAØˆr&   )r	   é`   N)rš   r›   rœ   r�   rŸ   rž   r   r   r   ÚModulerx   r   r™   r¡   r¢   s   @r$   r¤   r¤   A  sq   ø„ ñð ØØ9=ñ&à˜‘Ið&ð ð&ð ð	&ð
 ˜X c¨2¯9©9 nÑ5Ñ6ð&ð 
õ&ð,
˜ð 
 F÷ 
r&   r¤   c                    ó&  ‡ — e Zd ZdZdddddddedf	dee   ded	ee   d
ee   dee   dedededededee	de
j                  f      dee	de
j                  f      de	de
j                  f   dee	de
j                  f      ddfˆ fd„Zdedefd„Zˆ xZS )r   aY  
    Implements 3D Swin Transformer from the `"Video Swin Transformer" <https://arxiv.org/abs/2106.13230>`_ paper.
    Args:
        patch_size (List[int]): Patch size.
        embed_dim (int): Patch embedding dimension.
        depths (List(int)): Depth of each Swin Transformer layer.
        num_heads (List(int)): Number of attention heads in different layers.
        window_size (List[int]): Window size.
        mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4.0.
        dropout (float): Dropout rate. Default: 0.0.
        attention_dropout (float): Attention dropout rate. Default: 0.0.
        stochastic_depth_prob (float): Stochastic depth rate. Default: 0.1.
        num_classes (int): Number of classes for classification head. Default: 400.
        norm_layer (nn.Module, optional): Normalization layer. Default: None.
        block (nn.Module, optional): SwinTransformer Block. Default: None.
        downsample_layer (nn.Module): Downsample layer (patch merging). Default: PatchMerging.
        patch_embed (nn.Module, optional): Patch Embedding layer. Default: None.
    g      @r<   çš™™™™™¹?i�  Nr4   r¦   ÚdepthsrL   r   Ú	mlp_ratiorN   rM   Ústochastic_depth_probÚnum_classesr§   .ÚblockÚdownsample_layerÚpatch_embedr   c                 ó$  •— t         ‰| �  «        t        | «       |
| _        |€t	        t
        t        ¬«      }|€t	        t        j                  d¬«      }|€t        } ||||¬«      | _
        t        j                  |¬«      | _        g }t        |«      }d}t        t        |«      «      D ]Æ  }g }|d|z  z  }t        ||   «      D ]^  }|	t!        |«      z  |dz
  z  }|j#                   ||||   ||D �cg c]  }|dz  dk(  rdn|dz  ‘Œ c}|||||t        ¬	«
      «       |dz  }Œ` |j#                  t        j$                  |Ž «       |t        |«      dz
  k  sŒ¯|j#                   |||«      «       ŒÈ t        j$                  |Ž | _        |dt        |«      dz
  z  z  | _         || j(                  «      | _        t        j,                  d«      | _        t        j0                  | j(                  |
«      | _        | j5                  «       D ]~  }t7        |t        j0                  «      sŒt        j8                  j;                  |j<                  d
¬«       |j>                  €ŒVt        j8                  jA                  |j>                  «       Œ€ y c c}w )N)Ú
attn_layergñhãˆµøä>)Úeps)r4   r¦   r§   )rY   r   r   r*   )r   r   rµ   rN   rM   r¶   r§   r¼   r‚   rƒ   )!rw   rx   r   r·   r   r   rt   r   Ú	LayerNormr¤   rº   ÚDropoutÚpos_dropr]   r"   ry   r@   ÚappendÚ
SequentialÚfeaturesÚnum_featuresr­   ÚAdaptiveAvgPool3dÚavgpoolr{   ÚheadÚmodulesÚ
isinstancer‡   rˆ   r˜   rv   Úzeros_)r   r4   r¦   r´   rL   r   rµ   rN   rM   r¶   r·   r§   r¸   r¹   rº   ÚlayersÚtotal_stage_blocksÚstage_block_idÚi_stageÚstagerX   Úi_layerÚsd_probrG   Úmr€   s                            €r$   rx   zSwinTransformer3d.__init__‚  sS  ø€ ô" 	‰ÑÔÜ˜DÔ!Ø&ˆÔàˆ=ÜÔ0Ô=UÔVˆEàÐÜ ¤§¡°4Ô8ˆJàÐÜ&ˆKñ '°*È	Ð^hÔiˆÔÜŸ
™
 WÔ-ˆŒà"$ˆÜ  ›[ÐØˆäœS ›[Ó)ò 	AˆGØ%'ˆEØ˜a ™jÑ(ˆCÜ  ¨¡Ó1ò $�à/´%¸Ó2GÑGÐK]Ð`aÑKaÑb�Ø—‘ÙØØ! 'Ñ*Ø$/ØOZÖ#[È!¨°1©¸Ò)9¡A¸qÀA¹vÑ$EÒ#[Ø"+Ø 'Ø*;Ø.5Ø#-Ü#;ôôð  !Ñ#‘ð#$ð$ �M‰Mœ"Ÿ-™-¨Ð/Ô0àœ#˜f›+¨™/Ó*Ø—‘Ñ.¨s°JÓ?Õ@ð1	Aô2 Ÿ™ vÐ.ˆŒà%¨¬c°&«k¸A©oÑ(>Ñ>ˆÔÙ˜t×0Ñ0Ó1ˆŒ	Ü×+Ñ+¨AÓ.ˆŒÜ—I‘I˜d×/Ñ/°Ó=ˆŒ	à—‘“ò 	+ˆAÜ˜!œRŸY™YÕ'Ü—‘×%Ñ% a§h¡h°DÐ%Ô9Ø—6‘6Ñ%Ü—G‘G—N‘N 1§6¡6Õ*ñ		+ùò+ $\s   Ã5Jr8   c                 ó(  — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|j	                  ddddd«      }| j                  |«      }t        j                  |d«      }| j                  |«      }|S )Nr   r:   r*   r   r	   )	rº   rÀ   rÃ   r­   r.   rÆ   r^   r,   rÇ   )r   r8   s     r$   r™   zSwinTransformer3d.forwardÎ  s€   € à×Ñ˜QÓˆØ�M‰M˜!ÓˆØ�M‰M˜!ÓˆØ�I‰I�a‹LˆØ�I‰I�a˜˜A˜q !Ó$ˆØ�L‰L˜‹OˆÜ�M‰M˜!˜QÓˆØ�I‰I�a‹LˆØˆr&   )rš   r›   rœ   r�   r   rŸ   rž   r@   r   r   r   r±   rx   r   r™   r¡   r¢   s   @r$   r   r   n  s5  ø„ ñð4 ØØ#&Ø'*ØØ9=Ø48Ø5AØ:>ñJ+à˜‘IðJ+ð ðJ+ð �S‘	ð	J+ð
 ˜‘9ðJ+ð ˜#‘YðJ+ð ðJ+ð ðJ+ð !ðJ+ð  %ðJ+ð ðJ+ð ˜X c¨2¯9©9 nÑ5Ñ6ðJ+ð ˜  b§i¡i Ñ0Ñ1ðJ+ð # 3¨¯	©	 >Ñ2ðJ+ð ˜h s¨B¯I©I ~Ñ6Ñ7ðJ+ð  
õ!J+ðX
˜ð 
 F÷ 
r&   r   r¦   r´   r¶   ÚweightsÚprogressÚkwargsc           
      ó¼   — |�#t        |dt        |j                  d   «      «       t        d| |||||dœ|¤Ž}	|�"|	j	                  |j                  |d¬«      «       |	S )Nr·   Ú
categories)r4   r¦   r´   rL   r   r¶   T)rÕ   Ú
check_hash© )r   ry   Úmetar   Úload_state_dictÚget_state_dict)
r4   r¦   r´   rL   r   r¶   rÔ   rÕ   rÖ   Úmodels
             r$   Ú_swin_transformer3drß   Û  s{   € ð ÐÜ˜f m´S¸¿¹ÀlÑ9SÓ5TÔUäð ØØØØØØ3ñð ñ€Eð ÐØ×Ñ˜g×4Ñ4¸hÐSWÐ4ÓXÔYà€Lr&   )r*   r*   r*   )rØ   Úmin_sizeÚmin_temporal_sizec                   ó\   — e Zd Z ed eedddd¬«      i e¥ddd	d
dddœidddœ¥¬«      ZeZy)r   z9https://download.pytorch.org/models/swin3d_t-7615ae03.pth©éà   rä   ©é   ©g
×£p=
ß?gÉv¾Ÿ/Ý?g–C‹lçûÙ?©gZd;ßOÍ?gyé&1¬Ì?gÍÌÌÌÌÌÌ?©Ú	crop_sizeÚresize_sizeÚmeanr„   úFhttps://github.com/SwinTransformer/Video-Swin-Transformer#kinetics-400úœThe weights were ported from the paper. The accuracies are estimated on video-level with parameters `frame_rate=15`, `clips_per_video=12`, and `clip_len=32`iv¨­úKinetics-400gö(\�ÂmS@gð§ÆK7aW@©zacc@1zacc@5g7‰A`åðE@g˜nƒÀb^@©ÚrecipeÚ_docsÚ
num_paramsÚ_metricsÚ_opsÚ
_file_size©ÚurlÚ
transformsrÛ   N©	rš   r›   rœ   r   r   r
   Ú_COMMON_METAÚKINETICS400_V1ÚDEFAULTrÚ   r&   r$   r   r      óh   „ ÙØGÙØØ ØØ)Ø(ô
ð
Øð
à^ð[ð #àØ#Ø#ñ!ðð Ø!ò
ô€Nð6 �Gr&   r   c                   ó\   — e Zd Z ed eedddd¬«      i e¥ddd	d
dddœidddœ¥¬«      ZeZy)r   z9https://download.pytorch.org/models/swin3d_s-da41c237.pthrã   rå   rç   rè   ré   rí   rî   if$ørï   gÓMbXáS@g'1¬ŠW@rð   gçû©ñÒµT@gð§ÆK7Ik@rñ   rø   Nrû   rÚ   r&   r$   r   r     rÿ   r&   r   c                   ó¨   — e Zd Z ed eedddd¬«      i e¥ddd	d
dddœidddœ¥¬«      Z ed eedddd¬«      i e¥ddd	d
dddœidddœ¥¬«      ZeZ	y)r   z<https://download.pytorch.org/models/swin3d_b_1k-24f7c7c6.pthrã   rå   rç   rè   ré   rí   rî   iX…?rï   gÙÎ÷SÛS@gbX9´˜W@rð   gÓMbX•a@g /Ý$Âv@rñ   rø   z=https://download.pytorch.org/models/swin3d_b_22k-7c6ae6fa.pthgþÔxé&iT@gÛù~j¼äW@N)
rš   r›   rœ   r   r   r
   rü   rý   ÚKINETICS400_IMAGENET22K_V1rþ   rÚ   r&   r$   r   r   >  sÊ   „ ÙØJÙØØ ØØ)Ø(ô
ð
Øð
à^ð[ð #àØ#Ø#ñ!ðð Ø!ò
ô€Nñ6 ")ØKÙØØ ØØ)Ø(ô
ð
Øð
à^ð[ð #àØ#Ø#ñ!ðð Ø!ò
ô"Ðð6 �Gr&   r   Ú
pretrained)rÔ   )rÔ   rÕ   c                 ód   — t         j                  | «      } t        dg d¢dg d¢g d¢g d¢d| |dœ|¤ŽS )	a·  
    Constructs a swin_tiny architecture from
    `Video Swin Transformer <https://arxiv.org/abs/2106.13230>`_.

    Args:
        weights (:class:`~torchvision.models.video.Swin3D_T_Weights`, optional): The
            pretrained weights to use. See
            :class:`~torchvision.models.video.Swin3D_T_Weights` below for
            more details, and possible values. By default, no pre-trained
            weights are used.
        progress (bool, optional): If True, displays a progress bar of the
            download to stderr. Default is True.
        **kwargs: parameters passed to the ``torchvision.models.video.swin_transformer.SwinTransformer``
            base class. Please refer to the `source code
            <https://github.com/pytorch/vision/blob/main/torchvision/models/video/swin_transformer.py>`_
            for more details about this class.

    .. autoclass:: torchvision.models.video.Swin3D_T_Weights
        :members:
    ©r   r:   r:   r°   )r   r   rU   r   ©r	   rU   é   é   ©é   rV   rV   r³   ©r4   r¦   r´   rL   r   r¶   rÔ   rÕ   rÚ   )r   Úverifyrß   ©rÔ   rÕ   rÖ   s      r$   r   r   x  sH   € ô. ×%Ñ% gÓ.€Gäð 
ÚØÚÚ ÚØ!ØØñ
ð ñ
ð 
r&   c                 ód   — t         j                  | «      } t        dg d¢dg d¢g d¢g d¢d| |dœ|¤ŽS )	a¸  
    Constructs a swin_small architecture from
    `Video Swin Transformer <https://arxiv.org/abs/2106.13230>`_.

    Args:
        weights (:class:`~torchvision.models.video.Swin3D_S_Weights`, optional): The
            pretrained weights to use. See
            :class:`~torchvision.models.video.Swin3D_S_Weights` below for
            more details, and possible values. By default, no pre-trained
            weights are used.
        progress (bool, optional): If True, displays a progress bar of the
            download to stderr. Default is True.
        **kwargs: parameters passed to the ``torchvision.models.video.swin_transformer.SwinTransformer``
            base class. Please refer to the `source code
            <https://github.com/pytorch/vision/blob/main/torchvision/models/video/swin_transformer.py>`_
            for more details about this class.

    .. autoclass:: torchvision.models.video.Swin3D_S_Weights
        :members:
    r  r°   ©r   r   é   r   r  r	  r³   r  rÚ   )r   r  rß   r  s      r$   r   r   ž  sH   € ô. ×%Ñ% gÓ.€Gäð 
ÚØÚÚ ÚØ!ØØñ
ð ñ
ð 
r&   c                 ód   — t         j                  | «      } t        dg d¢dg d¢g d¢g d¢d| |dœ|¤ŽS )	a·  
    Constructs a swin_base architecture from
    `Video Swin Transformer <https://arxiv.org/abs/2106.13230>`_.

    Args:
        weights (:class:`~torchvision.models.video.Swin3D_B_Weights`, optional): The
            pretrained weights to use. See
            :class:`~torchvision.models.video.Swin3D_B_Weights` below for
            more details, and possible values. By default, no pre-trained
            weights are used.
        progress (bool, optional): If True, displays a progress bar of the
            download to stderr. Default is True.
        **kwargs: parameters passed to the ``torchvision.models.video.swin_transformer.SwinTransformer``
            base class. Please refer to the `source code
            <https://github.com/pytorch/vision/blob/main/torchvision/models/video/swin_transformer.py>`_
            for more details about this class.

    .. autoclass:: torchvision.models.video.Swin3D_B_Weights
        :members:
    r  é€   r  )r:   r
  é   é    r	  r³   r  rÚ   )r   r  rß   r  s      r$   r   r   Ä  sH   € ô. ×%Ñ% gÓ.€Gäð 
ÚØÚÚ ÚØ!ØØñ
ð ñ
ð 
r&   )r<   r<   NNT)6Ú	functoolsr   Útypingr   r   r   r^   Útorch.nn.functionalr   Ú
functionalr[   r   Útransforms._presetsr
   Úutilsr   Ú_apir   r   r   Ú_metar   Ú_utilsr   r   Úswin_transformerr   r   Ú__all__rŸ   rž   Útupler%   ÚfxÚwrapr3   r7   rH   r@   r    rr   r±   rt   r¤   r   rß   rü   r   r   r   rý   r   r   r   rÚ   r&   r$   ú<module>r#     sQ  ðõ ß *Ñ *ã ß Ð ß å 6å (ç 7Ñ 7å +ß Cß Aò€ð	#Ø�S‘	ð	#Ø%)¨#¡Yð	#Ø=AÀ#¹Yð	#à
ˆ4�‰9�d˜3‘iÐÑ ó	#ð ‡�‡�Ð*Ô +ð
"Ø"'§,¡,ð
"ØINÏÉð
"ØdhÐilÑdmð
"àó
"ð ‡�‡�Ð+Ô ,ð1 5¨¨c°3¨Ñ#7ð 1ÀUÈ3ÐPSÐUXÈ=ÑEYð 1Ð^cÐdgÐilÐnqÐdqÑ^ró 1ð
 ‡�‡�Ð$Ô %ð&Øð&à�C˜˜c�MÑ"ð&ð �s˜C �}Ñ%ð&ð �c˜3 �mÑ$ð	&ð
 ó&ðR ‡�‡�Ð*Ô +ð  #ØØ!%Ø"&ØñmØðmàðmð ðmð #ð	mð
 �c‘ðmð ðmð �S‘	ðmð ðmð ðmð �vÑðmð ˜Ñðmð ðmð ómð` ‡�‡�Ð+Ô ,ôV
˜rŸy™yô V
ôv*�2—9‘9ô *ôZj˜Ÿ	™	ô jðZØ�S‘	ðàðð �‰Iðð �C‰yð	ð
 �c‘ðð !ðð �kÑ"ðð ðð ðð óð> *ØØñ€ô�{ô ô>�{ô ô>7�{ô 7ñt ÓÙ ,Ð0@×0OÑ0OÐ!PÔQØ6:ÈTò !˜Ð"2Ñ3ð !Àdð !Ð]`ð !Ðevò !ó Ró ð!ñH ÓÙ ,Ð0@×0OÑ0OÐ!PÔQØ6:ÈTò !˜Ð"2Ñ3ð !Àdð !Ð]`ð !Ðevò !ó Ró ð!ñH ÓÙ ,Ð0@×0OÑ0OÐ!PÔQØ6:ÈTò !˜Ð"2Ñ3ð !Àdð !Ð]`ð !Ðevò !ó Ró ñ!r&   