^j] dZddlZddlmZmZmZmZmZmZm Z m Z ddl Z ddl m Z ddlm cmZddlmZmZddlmZmZmZmZmZddlmZddlmZdd lmZdd l m!Z!m"Z"d gZ#Gd d e jHZ%Gdde jHZ&Gdde jHZ'Gdde jHZ(Gdde jHZ)Gdde jHZ*Gdde jHZ+Gdde jHZ,Gdd e jHZ-dee.e j^fde jHdee.e j^ffd Z0d.d!e.d"edee.effd#Z1e!e1d$%e1d$%e1d$%e1d$%d&Z2d/d'e.d(e3d"ede-fd)Z4e"d/d(e3d"ede-fd*Z5e"d/d(e3d"ede-fd+Z6e"d/d(e3d"ede-fd,Z7e"d/d(e3d"ede-fd-Z8y)0ajSwiftFormer SwiftFormer: Efficient Additive Attention for Transformer-based Real-time Mobile Vision Applications Code: https://github.com/Amshaker/SwiftFormer Paper: https://arxiv.org/pdf/2303.15446 @InProceedings{Shaker_2023_ICCV, author = {Shaker, Abdelrahman and Maaz, Muhammad and Rasheed, Hanoona and Khan, Salman and Yang, Ming-Hsuan and Khan, Fahad Shahbaz}, title = {SwiftFormer: Efficient Additive Attention for Transformer-based Real-time Mobile Vision Applications}, booktitle = {Proceedings of the IEEE/CVF International Conference on Computer Vision (ICCV)}, year = {2023}, } N)AnyDictListOptionalSetTupleTypeUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPathLinear LayerType to_2tuple trunc_normal_)build_model_with_cfg)feature_take_indices)checkpoint_seq)generate_default_cfgsregister_model SwiftFormercfeZdZddededeffd ZdejdejfdZ xZ S) LayerScale2ddim init_valuesinplacec||d}t|||_tj|t j |ddfi|zd|_y)NdevicedtyperT) requires_grad)super__init__rnn Parametertorchonesgamma)selfrrrr!r"dd __class__s b/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/swiftformer.pyr%zLayerScale2d.__init__sL /  \\ %**S!Q5"5 5TK xreturncn|jr|j|jS||jzSN)rmul_r*r+r0s r.forwardzLayerScale2d.forward&s(%)\\qvvdjj!Eq4::~Er/)h㈵>FNN) __name__ __module__ __qualname__intfloatboolr%r(Tensorr6 __classcell__r-s@r.rrs?KCKeKTKFF%,,Fr/rceZdZdZdddddej ddfdeded ed ed ed eejf fd Z de jde jfdZ xZ S) Embeddingz Patch Embedding that is implemented by a layer of conv. Input: tensor in shape [B, C, H, W] Output: tensor in shape [B, C, H/stride, W/stride] irNin_chans embed_dim patch_sizestridepadding norm_layerc ||d} t |t|}t|}t|}tj|||||fi| |_|r||fi| |_ytj |_y)Nr )r$r%rr&Conv2dprojIdentitynorm) r+rErFrGrHrIrJr!r"r,r-s r.r%zEmbedding.__init__0su / z* 6"G$IIh :vwURTU 3=Jy/B/ 2;;= r/r0r1cJ|j|}|j|}|Sr3)rMrOr5s r.r6zEmbedding.forwardCs! IIaL IIaLr/)r8r9r:__doc__r& BatchNorm2dr;r Moduler%r(r>r6r?r@s@r.rBrB*s  *,..QQQ Q  Q  QRYYQ&%,,r/rBceZdZdZdddej ej dddfdeded ed ed e ejd e ejd e ffd Z de jde jfdZxZS) ConvEncoderz Implementation of ConvEncoder with 3*3 and 1*1 convolutions. Input: tensor with shape [B, C, H, W] Output: tensor with shape [B, C, H, W] @rCTNr hidden_dim kernel_size drop_path act_layerrJuse_layer_scalec || d} t |tj|||f|dz|d| |_||fi| |_tj||dfi| |_||_tj||dfi| |_|dkDr t|ntj|_ |rt|dfi| |_ ytj|_ y)Nr rIgroupsrrWr$r%r&rLdwconvrOpwconv1actpwconv2rrNrZr layer_scale) r+rrXrYrZr[rJr\r!r"r,r-s r.r%zConvEncoder.__init__Os / iiS+b{a?OX[b_ab s)b) yyj!:r: ;yyS!:r: 09B),BKKM9H<Q5"5bkkmr/r0r1c|}|j|}|j|}|j|}|j|}|j |}|j |}||j |z}|Sr3rbrOrcrdrerfrZ)r+r0inputs r.r6zConvEncoder.forwardesp KKN IIaL LLO HHQK LLO   Q  DNN1% %r/r8r9r:rQr&GELUrRr;r<r rSr=r%r(r>r6r?r@s@r.rUrUIs! !)+*,..$(\\\ \  \ BII \RYY\"\,  %,, r/rUc eZdZdZddej ej dddfdedeedeede ejde ejd e f fd Z d e jd e jfd ZxZS)Mlpz Implementation of MLP layer with 1*1 convolutions. Input: tensor with shape [B, C, H, W] Output: tensor with shape [B, C, H, W] NrW in_featureshidden_features out_featuresr[rJdropc  ||d} t ||xs|}|xs|}||fi| |_tj||dfi| |_||_tj||dfi| |_tj||_ y)Nr r) r$r%norm1r&rLfc1rdfc2Dropoutrq) r+rnrorpr[rJrqr!r"r,r-s r.r%z Mlp.__init__ws / #2{ )8[ 2r2 99[/1CC;99_lADDJJt$ r/r0r1c|j|}|j|}|j|}|j|}|j |}|j|}|Sr3)rsrtrdrqrur5s r.r6z Mlp.forwardsU JJqM HHQK HHQK IIaL HHQK IIaLr/)r8r9r:rQr&rkrRr;rr rSr<r%r(r>r6r?r@s@r.rmrmqs.2*.)+*,..%%&c]%#3- % BII % RYY %%*%,,r/rmcteZdZdZ d dededeffd ZdejdejfdZxZ S) EfficientAdditiveAttentionz Efficient Additive Attention module for SwiftFormer. Input: tensor in shape [B, C, H, W] Output: tensor in shape [B, C, H, W] in_dims token_dim num_headsc||d}t||dz|_tj|||zfi||_tj|||zfi||_tjtj||zdfi||_ tj||z||zfi||_ tj||z|fi||_ y)Nr gr) r$r% scale_factorr&rto_queryto_keyr'r(randnw_grMfinal)r+rzr{r|r!r"r,r-s r.r%z#EfficientAdditiveAttention.__init__s / %- '9y+@GBG iiY)>E"E << I ,A1 K KLIIi)3Y5JQbQ YYy94iF2F r/r0r1c2|j\}}}}|jdjddd}tj|j |d}tj|j |d}tj||jz|jzd}tj||zdd}|j||z|z} |j| jdddj|d||} | S)Nr^rrrT)rkeepdim)shapeflattenpermuteF normalizerrrr~r(sumrMrreshape) r+r0B_HWquerykeyattnouts r.r6z"EfficientAdditiveAttention.forwardsWW 1a IIaL Aq ) DMM!,"5kk$++a.b1{{5488+d.?.??QGyy1d;iis #e+jjo%%aA.66q"aC r/)rNN) r8r9r:rQr;r%r(r>r6r?r@s@r.ryrysY  GGG G&  %,, r/ryceZdZdZdddej ej ddfdededed e d e ejd e ejf fd Z d e jde jfdZxZS)LocalRepresentationz Local Representation module for SwiftFormer that is implemented by 3*3 depth-wise and point-wise convolutions. Input: tensor in shape [B, C, H, W] Output: tensor in shape [B, C, H, W] rCrWTNrrYrZr\r[rJc ||d} t |tj|||f|dz|d| |_||fi| |_tj||fddi| |_||_tj||fddi| |_|dkDr t|ntj|_ |rt|dfi| |_ ytj|_ y)Nr r^r_rYrrWra) r+rrYrZr\r[rJr!r"r,r-s r.r%zLocalRepresentation.__init__s / iiS+b{a?OX[b_ab s)b) yyc?q?B? ;yyc?q?B? 09B),BKKM9H<Q5"5bkkmr/r0r1c|}|j|}|j|}|j|}|j|}|j |}|j |}||j |z}|Sr3rh)r+r0skips r.r6zLocalRepresentation.forwardsp KKN IIaL LLO HHQK LLO   Q  4>>!$ $r/)r8r9r:rQr&rkrRr;r<r=r rSr%r(r>r6r?r@s@r.rrs !!$()+*,..\\\ \ " \ BII \RYY\*  %,, r/rceZdZdZdddej ej ddddf deded ed ed e ejd e ejd e deffd Z de jde jfdZxZS)Blockz SwiftFormer Encoder Block for SwiftFormer. It consists of : (1) Local representation module, (2) EfficientAdditiveAttention, and (3) MLP block. Input: tensor in shape [B, C, H, W] Output: tensor in shape [B, C, H, W] @rWTr7Nr mlp_ratio drop_raterZr[rJr\layer_scale_init_valuec | | d} t |td||||d| |_t d||d| |_t d|t||z|||d| |_|dkDr t|ntj|_ |r t||fi| ntj|_|rt||fi| |_ytj|_y)Nr )rr\r[rJ)rzr{)rnror[rJrqrW)r$r%rlocal_representationryrrmr;linearrr&rNrZr layer_scale_1 layer_scale_2) r+rrrrZr[rJr\rr!r"r,r-s r.r%zBlock.__init__s / $7% +! %  % !/PscPRP  i0!     1:B),BKKM*#/ELL$&KKM *#/ELL$&KKM r/r0r1c|j|}||j|j|j|z}||j|j |j |z}|Sr3)rrZrrrrr5s r.r6z Block.forwardsc  % %a ( t11$))A,?@ @ t11$++a.AB Br/rjr@s@r.rrs "!!)+*,..$(,0#2#2#2 #2  #2 BII #2RYY#2"#2%*#2J%,,r/rc eZdZdZdej ej dddddddf deded eed e d e ejd e ejd e de de de de e ejffd Zdej dej fdZxZS)Stagez Implementation of each SwiftFormer stages. Here, SwiftFormerEncoder used as the last block in all stages, while ConvEncoder used in the rest of the blocks. Input: tensor in shape [B, C, H, W] Output: tensor in shape [B, C, H, W] rrWTr7Nrindexlayersrr[rJrdrop_path_rater\r downsamplec| | d}t|d|_| | ntj|_g}t ||D]}||t|d|zzt|dz z }|||z dkr$|jt|f|||||| | d|U|jtd|t||zd|||| d|tj||_ y)Nr Fr)rrrZr[rJr\rrC)rrXrYrZr[rJr\r)r$r%grad_checkpointingr&rNrrangerappendrrUr; Sequentialblocks)r+rrrrr[rJrrr\rrr!r"r,r block_idx block_dprr-s r.r%zStage.__init__s  / "'(2(>*BKKMve}- I&)c&%.6I*IJcRXk\]o^Ie}y(A- e '''')$3+A    k "9s?3 !'')$3    2mmV, r/r0r1c|j|}|jr6tjj st |j |}|S|j |}|Sr3)rrr(jit is_scriptingrrr5s r.r6z Stage.forwardOsS OOA   " "599+A+A+Ct{{A.A AAr/)r8r9r:rQr&rkrRr;rr<r rSr=rr%r(r>r6r?r@s@r.rrs ")+*,..!$&$(,048/-/-/-I /-  /- BII /-RYY/-/-"/-"/-%*/-!bii1/-b%,,r/rc!eZdZgdgddgdejddddd d d d d d dddfdeedeededeedeejdedededede de dede de dedef fd Z d Z ejj d!efd"Zejj d8d#ed!ee effd$Zejj d9d%efd&Zejj d!eejejffd'Zd:dedee fd(Zejj d9d%efd)Z d;d*ej8d+eeeeefd,ed-ed.e d/ed!eeej8eej8eej8fffd0Z d\} } | |}| |vs|r| | k(r|j|}n|}|j|@|r|S | k(r|j|}||fS)a Forward features that returns intermediates. Args: x: Input image tensor indices: Take last n blocks if int, all if None, select matching indices if sequence norm: Apply norm layer to compatible intermediates stop_early: Stop iterating over blocks when last desired intermediate hit output_fmt: Shape of intermediate feature outputs intermediates_only: Only return intermediate features Returns: )NCHWzOutput shape must be NCHW.rN) rrrrr(rr enumeraterOr)r+r0rrOrrr  intermediates take_indices max_indexlast_idxrfeat_idxrx_inters r.forward_intermediatesz!SwiftFormer.forward_intermediatess*Y&D(DD& "6s4;;7G"Q it{{#a' IIaL 99 ! ! #:[[F[[)a-0F(0 .OHeaA<'H0"iilGG$$W- .  x  ! A-r/ prune_norm prune_headctt|j|\}}|jd|dz|_|rtj|_|r|j dd|S)z@ Prune layers not required for specified intermediates. Nrr)rrrr&rNrOr)r+rrrrrs r.prune_intermediate_layersz%SwiftFormer.prune_intermediate_layers s]#7s4;;7G"Q ikk.9q=1  DI   ! !!R (r/cl|j|}|j|}|j|}|Sr3)rrrOr5s r.forward_featureszSwiftFormer.forward_featuress. IIaL KKN IIaLr/ pre_logitsc6|jdk(r|jd}|j|}|r|S|j||j |}}|j r.|j r"tjjs||fS||zdz S)Nr)r^rCrr^) rmeanrrrrtrainingr(rr)r+r0rx_dists r. forward_headzSwiftFormer.forward_heads   u $6"A NN1  HIIaL$.."36  " "t}}UYY=S=S=Uf9 J!# #r/cJ|j|}|j|}|Sr3)rr r5s r.r6zSwiftFormer.forward-s'  ! !! $   a r/F)Tr3)NFFr F)rFT)$r8r9r:r&rkrr;r=r rSr<strr%rr(rignorerrrrrrrrrrrr>r rrrr r6r?r@s@r.rrXs!-$6&?)+#$ #!$&$(,0$!#'L#IL#S L# L# d L# BII L#!L#L#L#L#L#"L#"L#%*L#L# !L#"#L#\ 1 YY YY D T#s(^   YY*T** YY)bii&: ;))BCBhsmB YY)T)) 8<$$',0 ||0 eCcN340  0  0  0 !%0  tELL!5tELL7I)I#JJ K0 h./$# 3S >*  %,,5<< $ell $ $r/ state_dictmodelr1c|jd|}d|vr|Si}|jD]\}}|jdd}|jdd}|jdd}|jd d }|jd d }tjd d|}tj d|}|rLt |jd|jd}}|dz}|dzdk(r d|d|}n d|dzd|}|||<|S)Nr&z stem.0.weightz patch_embed.zstem.z dist_head.z head_dist.z attn.Proj.z attn.proj.z.layer_scale_1z.layer_scale_1.gammaz.layer_scale_2z.layer_scale_2.gammaz\.layer_scale(?=$|\.)z.layer_scale.gammaz^network\.(\d+)\.(.*)rr^rrz.blocks.z .downsample.)getitemsreplaceresubmatchr;group) r%r&out_dictkvrn_idxrest stage_idxs r.checkpoint_filter_fnr53s&4J*$H  "1 IIng . IIlL 1 IIlL 1 II&(> ? II&(> ? FF+-A1 E HH-q 1 aggaj/1771:4E IqyA~i[7ik],tf= !" Or/urlrc :|ddddddttdddd d d d |S) Nr)rCr8Tgffffff?bicubiczstem.0rz apache-2.0zarXiv:2303.15446zdSwiftFormer: Efficient Additive Attention for Transformer-based Real-time Mobile Vision Applicationsz'https://github.com/Amshaker/SwiftFormer)r6r input_size pool_sizefixed_input_sizecrop_pct interpolationrr first_conv classifierlicense paper_ids paper_name origin_urlr )r6rs r._cfgrEMs@=tae)%.B.C'|?    r/ztimm/) hf_hub_id)zswiftformer_xs.dist_in1kzswiftformer_s.dist_in1kzswiftformer_l1.dist_in1kzswiftformer_l3.dist_in1kvariant pretrainedc Ntt||fttddd|}|S)N)rrr^rCT) out_indicesflatten_sequential)pretrained_filter_fn feature_cfg)rrr5r)rGrHrr&s r._create_swiftformerrNls6 Wj 1\dK    E Lr/c Rtgdgd}tdd|it|fi|S)NrrrrrH)swiftformer_xsrrNrHr model_argss r.rQrQvs-\6HIJ  eJ e$zJd]cJd eer/c Rtgdgd}tdd|it|fi|S)N)rCrC r)rrVr8rPrH) swiftformer_srRrSs r.rXrX|s-\6HIJ  d: djIc\bIc ddr/c Rtgdgd}tdd|it|fi|S)N)rrC )r`irPrH)swiftformer_l1rRrSs r.r^r^s-]7IJJ  eJ e$zJd]cJd eer/c Rtgdgd}tdd|it|fi|S)N)rr r)rVi@rrPrH)swiftformer_l3rRrSs r.rbrbs-]7JKJ  eJ e$zJd]cJd eer/)rr")9rQr+typingrrrrrrr r r(torch.nnr&torch.nn.functional functionalr timm.datar r timm.layersrrrrr_builderr _featuresr _manipulater _registryrr__all__rSrrBrUrmryrrrrr#r>r5rE default_cfgsr=rNrQrXr^rbrr/r.rosY  EEE AMM*+'< / F299 F >%"))%P""))"J%%P$"))$N0BII0f=BII=@X"))XvT#u||*;%<RYYSWX[]b]i]iXiSj4 c # $sCx. % $! $ !%!!%! &  $#R]ftfsf{ff edecekeeftfsf{ff ftfsf{ffr/