^j/~dZddlZddlmZddlmZmZmZmZm Z m Z m Z m Z m Z ddlZddlmZddlmcmZddlmZmZddlmZmZmZmZmZmZmZmZm Z m!Z!m"Z"ddl#m$Z$dd l%m&Z&dd l'm(Z(dd l)m*Z*dd l+m,Z,m-Z-m.Z.d gZ/e e0e e0e0ffZ1dejdde e0e0fdejdfdZ3e(dejdde e0e0fde e0e0fdejdfdZ4GddejjZ6GddejjZ7GddejjZ8GddejjZ9Gdd ejjZ:dee;ejdfdejjdee;ejdffd ZdPd$Z?e,e?d%d&'e?d%d(d)d*d+,e?d%d-'e?d%d.d)d*d+,e?d%d/'e?d%d0'e?d%d1'e?d%d2'e?d%d3'e?d%d4'e?d%d5d6d7d89e?d%d:d6d7d89d; Z@e-dOd"e=de:fd<ZAe-dOd"e=de:fd=ZBe-dOd"e=de:fd>ZCe-dOd"e=de:fd?ZDe-dOd"e=de:fd@ZEe-dOd"e=de:fdAZFe-dOd"e=de:fdBZGe-dOd"e=de:fdCZHe-dOd"e=de:fdDZIe-dOd"e=de:fdEZJe-dOd"e=de:fdFZKe-dOd"e=de:fdGZLe.eMdHdIdJdKdLdMdNy)QaK Swin Transformer V2 A PyTorch impl of : `Swin Transformer V2: Scaling Up Capacity and Resolution` - https://arxiv.org/abs/2111.09883 Code/weights from https://github.com/microsoft/Swin-Transformer, original copyright/license info below Modifications and additions for timm hacked together by / Copyright 2022, Ross Wightman N)partial) AnyCallableDictListOptionalSetTupleTypeUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD) PatchEmbedMlpDropPathcalculate_drop_path_rates to_2tuple trunc_normal_ClassifierHeadresample_patch_embedndgrid get_act_layer LayerType)build_model_with_cfg)feature_take_indices)register_notrace_function) checkpoint)generate_default_cfgsregister_modelregister_model_deprecationsSwinTransformerV2x window_sizereturnc|j\}}}}|j|||dz|d||dz|d|}|jddddddjjd|d|d|}|S)zPartition into non-overlapping windows. Args: x: Input tensor of shape (B, H, W, C). window_size: Window size (height, width). Returns: Windows tensor of shape (num_windows*B, window_size[0], window_size[1], C). rrshapeviewpermute contiguous)r$r%BHWCwindowss j/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/swin_transformer_v2.pywindow_partitionr8%sJAq!Q q!{1~%{1~qKN7JKXYN\]^Aii1aAq)446;;B AP[\]P^`abG Nr6img_sizec|\}}|jd}|jd||dz||dz|d|d|}|jddddddjjd|||}|S)a1Merge windows back to feature map. Args: windows: Windows tensor of shape (num_windows * B, window_size[0], window_size[1], C). window_size: Window size (height, width). img_size: Image size (height, width). Returns: Feature map tensor of shape (B, H, W, C). r,rrr(r)r*r+r-)r6r%r:r3r4r5r$s r7window_reverser<8s DAq bA Rk!n,a;q>.A;q>S^_`SacdeA !Q1a#..055b!QBA Hr9c:eZdZdZ ddedeeefdedededed ed eeefd dffd Zdd Z ddZ dd ee je jffdZ deeefd dfdZddZdde jdee jd e jfdZxZS)WindowAttentionzWindow based multi-head self attention (W-MSA) module with relative position bias. Supports both shifted and non-shifted window attention with continuous relative position bias and cosine attention. Ndimr% num_headsqkv_biasqkv_bias_separate attn_drop proj_droppretrained_window_sizer&c | | d} t|||_||_t ||_||_||_tjtj|ddffi| |_ tjtjdddi| tjdtjd|fddi| |_tj||d zfddi| |_|rtjtj|fi| |_|j'd tj|fi| d tjtj|fi| |_nd |_d |_d |_tj,||_tj||fi| |_tj,||_tj4d |_|j\} } |j'dtjdd| zdz d| zdz dfi| d |j'dtj| | z| | z| tj8d |j;y )a4Initialize window attention module. Args: dim: Number of input channels. window_size: The height and width of the window. num_heads: Number of attention heads. qkv_bias: If True, add a learnable bias to query, key, value. qkv_bias_separate: If True, use separate bias for q, k, v projections. attn_drop: Dropout ratio of attention weight. proj_drop: Dropout ratio of output. pretrained_window_size: The height and width of the window in pre-training. devicedtyperr)biasT)inplaceFr(k_bias persistentNr,r?relative_coords_tablerelative_position_index)r)rJ)super__init__r?r%rrEr@rBnn Parametertorchempty logit_scale SequentialLinearReLUcpb_mlpqkvq_biasregister_bufferv_biasrMDropoutrCprojrDSoftmaxsoftmaxlongreset_parameters)selfr?r%r@rArBrCrDrErHrIddwin_hwin_w __class__s r7rTzWindowAttention.__init__Vs.2 / &&/0F&G#"!2<< Y14E(L(LM}} II .4 .2 . GGD ! IIc9 75 7B 7  99S#'<<< ,,u{{3'="'=>DK  5;;s+Ab+Ae T,,u{{3'="'=>DKDKDKDKI.IIc3-"- I.zzb) '' u  # KK1u9q=!e)a- Ab A   % KK uu}V5:: V   r9cZtjj|jt j d|j Rtjj|j tjj|j|jy)z"Initialize parameters and buffers. N) rUinit constant_rYmathlogr_zeros_ra _init_buffersrhs r7rgz WindowAttention.reset_parameterssb $**DHHRL9 ;; " GGNN4;; ' GGNN4;; ' r9c`|j|jj|j|jjj |jjj \}}|jj||jj|y)z.Compute and fill non-persistent buffer values.NrG) rMzero_"_make_pair_wise_relative_positionsrcweightrHrIrQcopy_rR)rhrQrRs r7rtzWindowAttention._init_bufferss ;; " KK   9=9`9`99##**$))2B2B2H2H:a: 66 ""(()>? $$**+BCr9ctj|jddz |jd|tj}tj|jddz |jd|tj}tjt ||}|j dddjjd}|jddkDrO|dddddddfxx|jddz zcc<|dddddddfxx|jddz zcc<nN|dddddddfxx|jddz zcc<|dddddddfxx|jddz zcc<|dz}tj|tjtj|dzztjdz }|j|}tj|jd|tj}tj|jd|tj}tjt ||}tj |d} | dddddf| dddddfz } | j dddj} | dddddfxx|jddz z cc<| dddddfxx|jddz z cc<| dddddfxxd|jdzdz zcc<| j#d } || fS) zCompute pair-wise relative position index and coordinates table. Returns: Tuple of (relative_coords_table, relative_position_index) rrrGr)N?)rIr,)rWaranger%float32stackrr0r1 unsqueezerEsignlog2absrqtorfflattensum) rhrHrIrelative_coords_hrelative_coords_wrQcoords_hcoords_wcoordscoords_flattenrelative_coordsrRs r7rxz2WindowAttention._make_pair_wise_relative_positionss"LLq!A% &(8(8(;FRWR_R_a!LLq!A% &(8(8(;FRWR_R_a % F3DFW,X Y 5 = =aA F Q Q S ] ]^_ `  & &q )A - !!Q1* -$2M2Ma2PST2T U - !!Q1* -$2M2Ma2PST2T U - !!Q1* -$2B2B12E2I J - !!Q1* -$2B2B12E2I J -" % +@ AEJJ II+ ,s 2E4!46:iil!C 5 8 8u 8 E<< 0 0 3F%**U<< 0 0 3F%**UVHh78vq1(At4~aqj7QQ)11!Q:EEG1a D$4$4Q$7!$;; 1a D$4$4Q$7!$;; 1a A(8(8(;$;a$?? "1"5"5b"9$&===r9c8t|}||jk7r|jJ|jj}|jj}||_|j ||\}}|j d|d|j d|dyy)zUpdate window size and regenerate relative position tables. Args: window_size: New window size (height, width). NrGrQFrNrR)rr%rQrHrIrxr`)rhr%rHrIrQrRs r7set_window_sizezWindowAttention.set_window_sizes  , $** *--9 99//66F..44E*D 77vU7S ; !#:  !8:O\a b  !:Z>Z>_>_`b>c!d!i!i   Q $"2"21"5 5t7G7G7JTM]M]^_M`7`bd"f!7!?!?1a!H!S!S!U!#emm4J&K!K,66q99  jjmG99R$..!Q?$..QRBSB]B]^_B``D99RA6D<<%D<<%D~~d# AX A & . .r1a 8 IIaL NN1 r9)TFr)rrNNr&N)NNN)__name__ __module__ __qualname____doc__intr boolfloatrTrgrtrWTensorrxrrrr __classcell__rls@r7r>r>Os"&+!!6<F F sCxF  F  F $ F F F %*#s(OF  F PD(> u||U\\) * (>Tg5c?gtg"11Xell-C1u||1r9r>ceZdZdZddddddddddej dd d fd ed ed ed edededede dede de de de de ejdeffd Z d&deej deej"deej$deej fdZ d'ded eedeeeefeeefffd!Z d'd"eeefd eeefdeedd fd#Zdej dej fd$Zdej dej fd%ZxZS)(SwinTransformerV2BlockzSwin Transformer V2 Block. A standard transformer block with window attention and shifted window attention for modeling long-range dependencies efficiently. rF@TrgeluNr?input_resolutionr@r% shift_sizealways_partition dynamic_mask mlp_ratiorArDrC drop_path act_layer norm_layerrEc 8||d}t|||_t||_||_t||_||_||_|j||\|_ |_ |jd|jdz|_ ||_ t| } t|ft|j|| | | t|d||_||fi||_| dkDr t%| nt'j(|_t-d |t/||z| | d||_||fi||_| dkDr t%| nt'j(|_|j7d|jrdn|j8d i|d y) a Args: dim: Number of input channels. input_resolution: Input resolution. num_heads: Number of attention heads. window_size: Window size. shift_size: Shift size for SW-MSA. always_partition: Always partition into full windows and shift mlp_ratio: Ratio of mlp hidden dim to embedding dim. qkv_bias: If True, add a learnable bias to query, key, value. proj_drop: Dropout rate. attn_drop: Attention dropout rate. drop_path: Stochastic depth rate. act_layer: Activation layer. norm_layer: Normalization layer. pretrained_window_size: Window size in pretraining. rGrr)r%r@rArCrDrEr) in_featureshidden_featuresrdrop attn_maskNFrN)rSrTr?rrr@target_shift_sizerr_calc_window_shiftr%r window_arearrr>rnorm1rrUIdentity drop_path1rrmlpnorm2 drop_path2r` get_attn_mask)rhr?rr@r%rrrrrArDrCrrrrErHrIrirls r7rTzSwinTransformerV2Block.__init__*sJ /  )*: ;"!*:!6 0(,0,C,CKQ[,\)$/++A.1A1A!1DD"!), #   !$"2"23#,-C#D      *r* 1:R(9-R[[] i0     *r* 1:R(9-R[[]  %%D+=4+=+=+C+C  r9r$rHrIr&c t|jr|)tjdg|jd||}nJtjd|j d|j ddf|j |j}d}d|jd f|jd |jd f|jd dffD]l}d|jd f|jd |jd f|jd dffD]$}||dd|d|d|d|dddf<|dz }&nt||j}|jd|j}|jd|jdz } | j| dk7tdj| dk(td} | Sd} | S) zGenerate attention mask for shifted window attention. Args: x: Input tensor for dynamic shape calculation. Returns: Attention mask or None if no shift. NrrGr)rr,gYr)anyrrWzerosrr.rHrIr%r8r/rr masked_fillr) rhr$rHrIimg_maskcnthw mask_windowsrs r7rz$SwinTransformerV2Block.get_attn_maskys t y ;;'ED,A,A'E1'Ef\ab ;;1771:qwwqz1'Eahh^_^e^efC))!,,-&&q))DOOA,>+>?ooa(($/  T--a001**1--0B/BC//!,,d3A '>>$BRBRSTBUU!!!$q4+;+;A+>'>>$BRBRSTBUUHH''++I1a57QR   2r1%Y0@0@A NN2t'7'7;  4 /**95IIyyy; $((T-=-=a-@$BRBRSTBUWXY "<1A1AB8L a!RaRl+668   9T__6JAAr9c>|j\}}}}||j|j|j|z}|j |d|}||j |j |j|z}|j ||||}|S)Nr,)r.rrrrrrr)rhr$r2r3r4r5s r7rzSwinTransformerV2Block.forward sWW 1a  4::a= 9: : IIaQ   488A; 78 8 IIaAq !r9)NNNr)rrrrrU LayerNormr_int_or_tuple_2_trrrr ModulerTrrWrrHrIrr rrrrrrs@r7rr#s./,-%*!&!!!!!#)*,,,89%M M 0M  M + M * M #M M M M M M M !M RYYM %6!M b)--1+/ ' %'U\\*'EKK( ' %,,  'X>B5 15 ((9:5 uS#Xc3h/ 0 5J04  S#X sCx 'tn    8,u||, ,\%,,r9rceZdZdZdej ddfdedeedeejffd Z de jde jfd Z xZS) PatchMergingzPatch Merging Layer. Merges 2x2 neighboring patches and projects to higher dimension, effectively downsampling the feature maps. Nr?out_dimrc||d}t|||_|xsd|z|_t j d|z|jfddi||_||jfi||_y)z Args: dim (int): Number of input channels. out_dim (int): Number of output channels (or 2 * dim if None) norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm rGr)r*rKFN)rSrTr?rrUr[ reductionnorm)rhr?rrrHrIrirls r7rTzPatchMerging.__init__sj / )!c' 1s7DLLKuKKt||2r2 r9r$r&ch|j\}}}}ddd|dzd|dzf}tjj||}|j\}}}}|j ||dzd|dzd|j ddddddj d}|j|}|j|}|S)Nrr)rr(r*r+) r.rUrrrr0rrr)rhr$r2r3r4r5 pad_valuesrs r7rzPatchMerging.forward2sWW 1aAq1uaQ/ MM  a ,WW 1a IIaaAFAq 1 9 9!Q1a K S STU V NN1  IIaLr9)rrrrrUrrrr rrTrWrrrrs@r7rrsb&**,,, 33c]3RYY 3*  %,, r9rc'LeZdZdZdddddddddej ddddfd ed ed ed ed ededededede dede de de de e e ejfde ejdededdf&fd Z d"deeefdedeeddfdZdej&dej&fd Zd#d!ZxZS)$SwinTransformerV2StagezA Swin Transformer V2 Stage. A single stage consisting of multiple Swin Transformer blocks with optional downsampling at the beginning. FrTrrrNr?rrdepthr@r%rr downsamplerrArDrCrrrrE output_nchwr&c^||d}t|||_||_| rt d|Dn||_||_||_d|_t|}t |Dcgc]}|dz c}}| rtd |||d||_ n ||k(sJtj|_ tjt|Dcgc]F}t!d ||j |||dzdk(rdn|||| | | | t#|t$r||n||||d|Hc}|_ycc}wcc}w) a  Args: dim: Number of input channels. out_dim: Number of output channels. input_resolution: Input resolution. depth: Number of blocks. num_heads: Number of attention heads. window_size: Local window size. always_partition: Always partition into full windows and shift dynamic_mask: Create attention mask in forward based on current input size downsample: Use downsample layer at start of the block. mlp_ratio: Ratio of mlp hidden dim to embedding dim. qkv_bias: If True, add a learnable bias to query, key, value. proj_drop: Projection dropout rate attn_drop: Attention dropout rate. drop_path: Stochastic depth rate. act_layer: Activation layer type. norm_layer: Normalization layer. pretrained_window_size: Local window size in pretraining. output_nchw: Output tensors on NCHW format instead of NHWC. rGc3&K|] }|dz ywr)Nr.0is r7 z2SwinTransformerV2Stage.__init__..vs&H!qAv&HFr))r?rrr)r?rr@r%rrrrrArDrCrrrrENr)rSrTr?rroutput_resolutionrr grad_checkpointingrrrrUr ModuleListranger isinstancelistblocks)rhr?rrrr@r%rrrrrArDrCrrrrEr rHrIrirrrrls r7rTzSwinTransformerV2Stage.__init__FsQX /  0LV&H7G&H!H\l &"' , K8qAF89  *asGPZa^`aDO'> !> kkmDOmm&5\'%#&% # !%!7!7#'!"Q!1*!1)#!##*4Y*E)A,9#%'=  ! %#$ 9%#s ' D% A D*rc.||_t|jtjr||_n3t|jt sJtd|D|_|jD] }|j|j ||"y)zUpdate resolution, window size and relative positions. Args: feat_size: New input (feature) resolution. window_size: New window size. always_partition: Always partition / shift the window. c3&K|] }|dz ywr rr s r7rz8SwinTransformerV2Stage.set_input_size..s*Ea16*Errr%rN) rrrrUrrrrrr)rhrr%rblocks r7rz%SwinTransformerV2Stage.set_input_sizes!* door{{ 3%.D "doo|< <<%**E9*E%ED "[[ E  00'!1 !  r9r$c|j|}|jD]A}|jr+tjj s t ||}:||}C|S)zForward pass through the stage. Args: x: Input tensor of shape (B, H, W, C). Returns: Output tensor of shape (B, H', W', C'). )rrrrWjit is_scriptingr)rhr$blks r7rzSwinTransformerV2Stage.forwardsY OOA ;; C&&uyy/E/E/GsA&F   r9c|jD]}tjj|jj dtjj|jj dtjj|jj dtjj|jj dy)z/Initialize residual post-normalization weights.rN)rrUrorprrKryr)rhr s r7_init_respostnormz(SwinTransformerV2Stage._init_respostnorms;; 3C GG  ciinna 0 GG  cii.. 2 GG  ciinna 0 GG  cii.. 2  3r9rr)rrrrrUrrrrrr strr rrTr rrrWrrr"rrs@r7rr?s&+!&$!!!!!5;*,,,89 %+R$R$R$0 R$  R$  R$+R$#R$R$R$R$R$R$R$R$ S$ryy/12!R$"RYY#R$$%6%R$&'R$, -R$p04 S#X'tn   4%,,$3r9rc+leZdZdZdddddddd d d d d d dddddej dddfdedededededede edfde edfdede de de d e d!e d"e d#e d$e d%e ee fd&eejd'e edff(fd( ZdEd)e d*dfd+ZdEd,ejd)e d*dfd-Z dFdee eefdee eefdee eefd.eedee f d/Zej,j.d*eefd0Zej,j.dGd1e d*eeeffd2Zej,j.dEd3e d*dfd4Zej,j.d*ejfd5ZdHdedeed*dfd6Z dId7ej@d8ee ee!efd9e d:e d;ede d?e fd@Z#d7ej@d*ej@fdAZ$dGd7ej@dBe d*ej@fdCZ%d7ej@d*ej@fdDZ&xZ'S)Kr#a Swin Transformer V2. A hierarchical vision transformer using shifted windows for efficient self-attention computation with continuous position bias. A PyTorch impl of : `Swin Transformer V2: Scaling Up Capacity and Resolution` - https://arxiv.org/abs/2111.09883 r*r(avg`r)r)r)r(r* rFTrrg?r)rrrrNr: patch_sizein_chans num_classes global_pool embed_dimdepths.r@r%rstrict_img_sizerrA drop_rateproj_drop_rateattn_drop_ratedrop_path_raterrpretrained_window_sizesc dt |||d}||_||_|dvsJ||_d|_t ||_||_t|d|jdz zzx|_ |_ g|_ t|ttfs1t!|jDcgc]}t|d|zz}}t#d"||||d|| dd||_|j$j&}t)||d }g}|d}d}t!|jD]}||}|t+d"id |d |d |d|z|d|zfd ||d|dkDd||d| d| d| d| d| d|d|d||d|d|d|||gz }|}|dkDr|dz}|xjt-|d|zd|gz c_ t/j0||_||jfi||_t7|j|f|||j d||_|j;d y!cc}w)#a] Args: img_size: Input image size. patch_size: Patch size. in_chans: Number of input image channels. num_classes: Number of classes for classification head. embed_dim: Patch embedding dimension. depths: Depth of each Swin Transformer stage (layer). num_heads: Number of attention heads in different layers. window_size: Window size. mlp_ratio: Ratio of mlp hidden dim to embedding dim. qkv_bias: If True, add a learnable bias to query, key, value. drop_rate: Head dropout rate. proj_drop_rate: Projection dropout rate. attn_drop_rate: Attention dropout rate. drop_path_rate: Stochastic depth rate. norm_layer: Normalization layer. act_layer: Activation layer type. patch_norm: If True, add normalization after patch embedding. pretrained_window_sizes: Pretrained window sizes of each layer. output_fmt: Output tensor format if not None, otherwise output 'NHWC' by default. rG)r'NHWCr)rr)r:r.r/r2rr4 output_fmtT) stagewiser?rrrrr@r%rrrrArDrCrrrrEr*layers.)num_chsrmodule) pool_typer5 input_fmtF needs_resetNr)rSrTr0r/r1r=len num_layersr2r num_featureshead_hidden_size feature_inforrrrr patch_embed grid_sizerrdictrUrZlayersrrhead init_weights)!rhr:r.r/r0r1r2r3r@r%rr4rrAr5r6r7r8rrr9rHrIkwargsrirrLdprrNin_dimscalerrls! r7rTzSwinTransformerV2.__init__s`  /&  k)))& f+"47 A$//\]J]D^8^4__D1)eT]3:?:PQQYa/0QIQ&  !l!+    $$.. '$O1t' dAlG -#,A,%"719N!OQi  q5  $A, ("2"10$"))a&$ &!"(?q'A% F(F1u    $w!e)V]^_]`Ta"b!c c 3 d6mmV, t007B7 "     "oo      e,uRs,H-rEr&c|jt|j||jD]}|j y)zInitialize model weights. Args: needs_reset: If True, call reset_parameters() on modules (default for after to_empty()). If False, skip reset_parameters() (for __init__ where modules already self-initialized). rDN)applyr _init_weightsrNr")rhrEblys r7rPzSwinTransformerV2.init_weightsSs< 74--;GH;; $C  ! ! # $r9mct|tjrOt|jd|j +tj j|j dyy|rt|dr|jyyy)zInitialize weights for Linear layers. Args: m: Module to initialize. needs_reset: Whether to call reset_parameters() on modules. g{Gz?)stdNrrg) rrUr[rryrKrorphasattrrg)rhrYrEs r7rWzSwinTransformerV2._init_weights^sg a # !(( ,vv!!!!&&!," WQ(:;   <[r9 window_ratioc^||3|jj|||jj}||tDcgc]}||z c}}t |j D]9\}} dt |dz dz} | jd| z|d| zf||;ycc}w)aTUpdates the image resolution, window size, and so the pair-wise relative positions. Args: img_size (Optional[Tuple[int, int]]): New input resolution, if None current resolution is used patch_size (Optional[Tuple[int, int]): New patch size, if None use current patch size window_size (Optional[int]): New window size, if None based on new_img_size // window_div window_ratio (int): divisor for calculating window size from patch grid size always_partition: always partition / shift windows even if feat size is < window N)r:r.r)rrr)rKrrLr enumeraterNr) rhr:r.r%r]rrLrindexstage stage_scales r7rz SwinTransformerV2.set_input_sizels"  :#9    + +X* + U((22I  <#;I Fql!2 FGK%dkk2 LE5s519a00K  $Q<;6 ! 8ST'!1 !  !Gs B*ct}|jD]2\}}tdDcgc]}||vc}s"|j|4|Scc}w)zGet parameter names that should not use weight decay. Returns: Set of parameter names to exclude from weight decay. )r]rY)set named_modulesradd)rhnodnrYkws r7no_weight_decayz!SwinTransformerV2.no_weight_decaysWe&&( DAq&@AB!GAB   Bs A coarsec2td|rdSgdS)zCreate parameter group matcher for optimizer parameter groups. Args: coarse: If True, use coarse grouping. Returns: Dictionary mapping group names to regex patterns. z^absolute_pos_embed|patch_embedz^layers\.(\d+)))z^layers\.(\d+).downsample)r)z^layers\.(\d+)\.\w+\.(\d+)N)z^norm)i)stemr)rM)rhrks r7 group_matcherzSwinTransformerV2.group_matchers)3(.$  5  r9enablec4|jD] }||_ y)z}Enable or disable gradient checkpointing. Args: enable: If True, enable gradient checkpointing. N)rNr)rhrols r7set_grad_checkpointingz(SwinTransformerV2.set_grad_checkpointings *A#)A  *r9c.|jjS)z_Get the classifier head. Returns: The classification head module. )rOfcrus r7get_classifierz SwinTransformerV2.get_classifiersyy||r9cJ||_|jj||y)zReset the classification head. Args: num_classes: Number of classes for new head. global_pool: Global pooling type. N)r0rOreset)rhr0r1s r7reset_classifierz"SwinTransformerV2.reset_classifiers'  [1r9r$indicesr stop_earlyr=intermediates_onlyc>|dvsJdg}tt|j|\}} |j|}t|j} tj j s|s |j} n|jd| dz} t| D]c\} } | |}| |vs|r| | dz k(r|j|}n|}|jddddj}|j|e|r|S|j|}||fS)a Forward features that returns intermediates. Args: x: Input image tensor indices: Take last n blocks if int, all if None, select matching indices if sequence norm: Apply norm layer to compatible intermediates stop_early: Stop iterating over blocks when last desired intermediate hit output_fmt: Shape of intermediate feature outputs intermediates_only: Only return intermediate features Returns: )NCHWzOutput shape must be NCHW.Nrrr(r)) rrFrNrKrWrrr_rr0r1append)rhr$ryrrzr=r{ intermediates take_indices max_index num_stagesstagesrrax_inters r7forward_intermediatesz'SwinTransformerV2.forward_intermediatess*Y&D(DD& "6s4;;7G"Q i   Q % 99 ! ! #:[[F[[)a-0F!&) .HAuaAL Aa/"iilGG!//!Q15@@B$$W- .  IIaL-r9 prune_norm prune_headctt|j|\}}|jd|dz|_|rtj|_|r|j dd|S)z@ Prune layers not required for specified intermediates. Nrrr;)rrFrNrUrrrx)rhryrrrrs r7prune_intermediate_layersz+SwinTransformerV2.prune_intermediate_layerss]#7s4;;7G"Q ikk.9q=1  DI   ! !!R (r9cl|j|}|j|}|j|}|S)zForward pass through feature extraction layers. Args: x: Input tensor of shape (B, C, H, W). Returns: Feature tensor of shape (B, H', W', C). )rKrNrrhr$s r7forward_featuresz"SwinTransformerV2.forward_features s3   Q  KKN IIaLr9 pre_logitscN|r|j|dS|j|S)aForward pass through classification head. Args: x: Feature tensor of shape (B, H, W, C). pre_logits: If True, return features before final linear layer. Returns: Logits tensor of shape (B, num_classes) or pre-logits. T)r)rO)rhr$rs r7 forward_headzSwinTransformerV2.forward_heads&1;tyyty,L ! Lr9cJ|j|}|j|}|S)zForward pass through the model. Args: x: Input tensor of shape (B, C, H, W). Returns: Logits tensor of shape (B, num_classes). )rrrs r7rzSwinTransformerV2.forward%s)  ! !! $   a r9)T)NNNr|NFr)NFFr}F)rFT)(rrrrrUrrrr#r rrr rr rrTrPrWrrrWrignorer rjrrrnrrrurxrrrrrrrrrs@r7r#r#s+.#$&2)7-.%*$(!!!$&$&$'.4*,,,7C/x-'x-x- x-  x-  x-x-#s(Ox-S#Xx-+x-#x-"x-x-x-x- "!x-""#x-$"%x-&S(]+'x-(RYY)x-*&+38_+x-t $ $ $ !ryy !t !t ! 374859*+/3 uS#X/!sCx1"%S/2  #3-  'tn @ YY S   YY D T#s(^  $ YY*T*T** YY 2C2hsm2W[28<$$',0 ||0 eCcN340  0  0  0 !%0  tELL!5tELL7I)I#JJ K0 h./$# 3S >*  %,, 5<<  Mell M M M  %,, r9 state_dictmodelc|jd|}|jd|}d|v}i}ddl}|jD]\}}tdDcgc]}||vc}r"d|vre|jj j j\}}} } |jd| k7s|jd | k7rt|| | fd d d }|s&|jd d|}|jdd}|||<|Scc}w)aMFilter and process checkpoint state dict for loading. Handles resizing of patch embeddings and relative position tables when model size differs from checkpoint. Args: state_dict: Checkpoint state dictionary. model: Target model to load weights into. Returns: Filtered state dictionary. rrzhead.fc.weightrN)rRrQrzpatch_embed.proj.weightrr,bicubicT) interpolation antialiasverbosezlayers.(\d+).downsamplecDdt|jddzdS)Nr?rz .downsample)rgroup)r$s r7z&checkpoint_filter_fn..Vs$ws177ST:YZGZF[[f=gr9zhead.zhead.fc.) getreitemsrrKrcryr.rsubreplace) rrnative_checkpointout_dictrrrrhrr3r4s r7checkpoint_filter_fnr3s4J j9J(J6H   "1  ab1Qb c  $ )**//66<.hs\da\rr3)rrrr out_indicesT)flatten_sequentialr)pretrained_filter_fn feature_cfg)rr_rpoprr#rrM)rrrQdefault_out_indicesrrs r7_create_swin_transformer_v2r]sj \i 8\8Z.[\\**],?@K 7J 1DkJ    E Lr9c 4|ddddddttddd d |S) Nr&)r(r)r|r|g?rTzpatch_embed.projzhead.fcmit) urlr0 input_size pool_sizecrop_pctrfixed_input_sizemeanr[ first_conv classifierlicenser )rrQs r7_cfgrss5=v%.B(    #  r9ztimm/z{https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window12to16_192to256_22kto1k_ft.pth) hf_hub_idrz{https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window12to24_192to384_22kto1k_ft.pth)r(r)r,r,r})rrrrrz|https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_large_patch4_window12to16_192to256_22kto1k_ft.pthz|https://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_large_patch4_window12to24_192to384_22kto1k_ft.pthzfhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_tiny_patch4_window8_256.pthzghttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_tiny_patch4_window16_256.pthzghttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_small_patch4_window8_256.pthzhhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_small_patch4_window16_256.pthzfhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window8_256.pthzghttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window16_256.pthzkhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_base_patch4_window12_192_22k.pthiQU)r(r)r*r*)rrr0rrzlhttps://github.com/SwinTransformer/storage/releases/download/v2.0.0/swinv2_large_patch4_window12_192_22k.pth) 2swinv2_base_window12to16_192to256.ms_in22k_ft_in1k2swinv2_base_window12to24_192to384.ms_in22k_ft_in1k3swinv2_large_window12to16_192to256.ms_in22k_ft_in1k3swinv2_large_window12to24_192to384.ms_in22k_ft_in1kzswinv2_tiny_window8_256.ms_in1kz swinv2_tiny_window16_256.ms_in1kz swinv2_small_window8_256.ms_in1kz!swinv2_small_window16_256.ms_in1kzswinv2_base_window8_256.ms_in1kz swinv2_base_window16_256.ms_in1k!swinv2_base_window12_192.ms_in22k"swinv2_large_window12_192.ms_in22kc Ptdddd}t dd|it|fi|S)z"Swin-T V2 @ 256x256, window 16x16.rr(r)r+r%r2r3r@r)swinv2_tiny_window16_256rMrrrQ model_argss r7rrsD"B:>XQW>X ZZr9c Ptdddd}t dd|it|fi|S)z Swin-S V2 @ 256x256, window 8x8.r|r(rr+rr)swinv2_small_window8_256rrs r7rrsD!r-SabJ &" Y/9 Y=A*=WPV=W YYr9c Ptdddd}t dd|it|fi|S)z"Swin-B V2 @ 256x256, window 16x16.rrr*r|r rr)swinv2_base_window16_256rrs r7rrD"MUcdJ &" Y/9 Y=A*=WPV=W YYr9c Ptdddd}t dd|it|fi|S)z Swin-B V2 @ 256x256, window 8x8.r|rrrrr)swinv2_base_window8_256rrs r7rrsC!s=TbcJ &! X.8 X<@B:>XQW>X ZZr9c Rtddddd}t dd|it|fi|S) zQSwin-L V2 @ 192x192, trained at window 12x12, fine-tuned to 256x256 window 16x16.rrrrrrr)"swinv2_large_window12to16_192to256rrs r7rr K#m /1J ', c9C cGKJGaZ`Ga ccr9c Rtddddd}t dd|it|fi|S) zQSwin-L V2 @ 192x192, trained at window 12x12, fine-tuned to 384x384 window 24x24.r-rrrrrr)"swinv2_large_window12to24_192to384rrs r7rrrr9rrrrrr)swinv2_base_window12_192_22k)swinv2_base_window12to16_192to256_22kft1k)swinv2_base_window12to24_192to384_22kft1kswinv2_large_window12_192_22k*swinv2_large_window12to16_192to256_22kft1k*swinv2_large_window12to24_192to384_22kft1kr)r;)Nrrq functoolsrtypingrrrrrr r r r rWtorch.nnrUtorch.nn.functionalrr timm.datarr timm.layersrrrrrrrrrrr_builderr _featuresr _features_fxr _manipulater _registryr r!r"__all__rrrr8r<rr>rrrr#r#rrrr default_cfgsrrrrrrrrrrrrrrr9r7rs OOO A;;;;*+3#YY  #uS#X./ <<38_ \\&  38_ S/  \\   ,QbiiQhpRYYpf&299&RM3RYYM3`a aH 'T#u||*;%<'RYY'SWX[]b]i]iXiSj'T$Uf,%:> J;;? J Hs; <@ K<<@ K Hs< (, t()- u))- u)*. v*(, t()- u) *. ymv* +/ zmv+e7&7 tYYDUYYXXCTXXZ$ZEVZZYYDUYYYYDUYYXXCTXXYYDUYYb$bM^bbb$bM^bbZ$ZEVZZc4cN_ccc4cN_ccH$G1e1e%I2g2g 'r9