^jd rdZddlZddlZddlZddlmZddlmZm Z m Z m Z m Z ddl Z ddlmcmZddl mZddlmZmZddlmZmZmZmZmZmZmZmZmZmZm Z m!Z!dd l"m#Z#dd l$m%Z%dd l&m'Z'dd l(m)Z)m*Z*dd l+m,Z,m-Z-m.Z.dgZ/ej`e1Z2GddejfZ4GddejfZ5GddejfZ6de7fdZ8e'de7fdZ9GddejfZ:GddejfZ;d1dejfdedZ?d Z@d2d!ZAd3d"ZBe-eBeBeBeBd#$eBd#$eBd#$d%ZCe,d2d&e;fd'ZDe,d2d&e;fd(ZEe,d2d&e;fd)ZFe,d2d&e;fd*ZGe,d2d&e;fd+ZHe,d2d&e;fd,ZIe.e1d-d.d/d0y)4a Nested Transformer (NesT) in PyTorch A PyTorch implement of Aggregating Nested Transformers as described in: 'Aggregating Nested Transformers' - https://arxiv.org/abs/2105.12723 The official Jax code is released and available at https://github.com/google-research/nested-transformer. The weights have been converted with convert/convert_nest_flax.py Acknowledgments: * The paper authors for sharing their research, code, and model weights * Ross Wightman's existing code off which I based this Copyright 2021 Alexander Soare N)partial)ListOptionalTupleTypeUnion)nnIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD) PatchEmbedMlpDropPathcalculate_drop_path_ratescreate_classifier trunc_normal__assert create_conv2d create_pool2d to_ntupleuse_fused_attn LayerNorm)build_model_with_cfg)feature_take_indices)register_notrace_function)checkpoint_seq named_apply)register_modelgenerate_default_cfgsregister_model_deprecationsNestc eZdZUdZej j eed< d de de dede de f fd Z d Z xZ S) Attentionz This is much like `.vision_transformer.Attention` but uses *localised* self attention by accepting an input with an extra "image block" dim fused_attndim num_headsqkv_bias attn_drop proj_dropcX||d}t |||_||z} | dz|_t |_t j|d|zfd|i||_t j||_ t j||fi||_ t j||_ y)Ndevicedtypegbias) super__init__r'scalerr%r LinearqkvDropoutr)projr*) selfr&r'r(r)r*r-r.ddhead_dim __class__s [/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/nest.pyr2zAttention.__init__=s / ")#% (*99S!C%=h="=I.IIc3-"- I.c |j\}}}}|j|j|||d|j||jzj dddddd}|j d\}}} |j r>>QRZZ[\^_abdeghjkl**Q-1a ??..q!QVZVcVc$..BRBRiklADJJAq{{2r**D<>$'DqA IIaAq! $ , ,Q1a 8 IIaL NN1 r=)FrBrBNN)__name__ __module__ __qualname____doc__torchjitFinalbool__annotations__intfloatr2rY __classcell__r;s@r<r$r$6sk %% "!!/// /  /  /,r=r$ceZdZdZdddddej ej ddf dededed e d ed ed ed e ejde ejffd Z dZ xZS)TransformerLayerz This is much like `.vision_transformer.Block` but: - Called TransformerLayer here to allow for "block" as defined in the paper ("non-overlapping image blocks") - Uses modified Attention layer that handles the "block" dimension @FrBNr&r' mlp_ratior(r*r) drop_path act_layer norm_layerc | | d} t|| |fi| |_t|f||||d| |_|dkDr t |nt j|_| |fi| |_ t||z} td|| ||d| |_ |dkDrt ||_ yt j|_ y)Nr,)r'r(r)r*rB) in_featureshidden_featuresrmdrop)r1r2norm1r$rXrr Identity drop_path1norm2rdrmlp drop_path2)r8r&r'rkr(r*r)rlrmrnr-r.r9mlp_hidden_dimr;s r<r2zTransformerLayer.__init__rs / *r*        2;R(9-R[[]*r* S9_- *    2;R(9-R[[]r=c|j|}||j|j|z}||j|j |j |z}|SN)rtrvrXryrxrw)r8rPys r<rYzTransformerLayer.forwardsS JJqM  ! - - A 78 8r=)r[r\r]r^r GELUrrdrerbrModuler2rYrfrgs@r<ririls ""!!!)+*,,,#S#S#S #S  #S  #S#S#SBII#SRYY#SJr=ric XeZdZ ddededeej deffd ZdZ xZ S)ConvPool in_channels out_channelsrnpad_typec||d}t|t||fd|dd||_||fi||_t ddd||_y)Nr,r/T) kernel_sizepaddingr0maxr@)rstrider)r1r2rconvnormrpool) r8rrrnrr-r.r9r;s r<r2zConvPool.__init__s\ / !+|nT\cgnkmn |2r2 !%Qq(S r=c0t|jddzdk(dt|jddzdk(d|j|}|j|j ddddj dddd}|j |}|S)z: x is expected to have shape (B, C, H, W) rDr@rz1BlockAggregation requires even input spatial dimsrEr/r)rrFrrrHrr8rPs r<rYzConvPool.forwards  a1$&YZ a1$&YZ IIaL IIaii1a+ , 4 4Q1a @ IIaLr=)NN) r[r\r]rdrr rstrr2rYrfrgs@r<rrsK  T T TRYY T  T r=r block_sizec|j\}}}}t||zdk(dt||zdk(d||z}||z}|j||||||}|jddj|||zd|}|S)zimage to blocks Args: x (Tensor): with shape (B, H, W, C) block_size (int): edge length of a single square block in units of H, W rz,`block_size` must divide input height evenlyz+`block_size` must divide input width evenlyr@r/rE)rFrrGrN)rPrrQHWrT grid_height grid_widths r<blockifyrs ''JAq!Q A Na !OP A Na !NOz/KjJ ![*j*aHA Aq!!![:%=r1EA Hr=c|j\}}}}ttj|}||zx}}|j ||||||}|j ddj ||||}|S)zblocks to image Args: x (Tensor): with shape (B, T, N, C) where T is number of blocks and N is sequence size per block block_size (int): edge length of a single square block in units of desired H, W r@r/)rFrdmathsqrtrGrN) rPrrQrR_rT grid_sizeheightwidths r< deblockifyrstJAq!QDIIaL!I++FU !Y :z1EA Aq!!!VUA6A Hr=ceZdZdZ ddededededededeed ed ed ed ed eeedee e jdee e jde ffd Z dZxZS) NestLevelz7 Single hierarchical level of a Nested Transformer num_blocksr seq_lengthr'depth embed_dimprev_embed_dimrkr(r*r)rlrnrmrc||d}t|||_d|_t j t jd|||fi||_|t||f| |d||_ nt j|_ t| rt| |k(sJdt jt|Dcgc]}td|||| | | | r| |nd| |d | c}|_ycc}w)Nr,Fr)rnrzDMust provide as many drop path rates as there are transformer layers) r&r'rkr(r*r)rlrnrmrs)r1r2rgrad_checkpointingr Parameterr_zeros pos_embedrrrulen Sequentialrangeritransformer_encoder)r8rrrr'rrrrkr(r*r)rlrnrmrr-r.r9ir;s r<r2zNestLevel.__init__s ( / $"'ekk!ZY&]Z\&]^  % kz\dkhjkDI DI y>y>U* r,r r*#%==5\ 3#  ##!##*3)A,%#   3# $$ 3#s#C.c|j|}|jdddd}t||j}||jz}|j r5t jjst|j|}n|j|}t||j}|jddddS)z+ expects x as (B, C, H, W) rr@r/r) rrHrrrrr_r` is_scriptingrrrrs r<rYzNestLevel.forwards IIaL IIaAq ! Q (    " "599+A+A+Ct77;A((+A q$// *yyAq!$$r=) NrjTrBrBNNNrNN)r[r\r]r^rdrrerbrrr rrr2rYrfrgs@r<rrs-1!!!!/34837%0$0$0$ 0$  0$  0$0$%SM0$0$0$0$0$ U ,0$!bii10$ RYY00$ !0$d%r=rc'@eZdZdZ d,dededededeedfdeedfd eedfd ed ed ed ededededee e jdee e jde de de f&fd Z ejj d-dZejj dZejj d.dZejj d/dZejj de jfdZd0d ede fdZ d1dej.deeeeefd ed!ed"e d#edeeej.eej.eej.fffd$Z d2deeeefd%ed&efd'Zd(Zd.d)efd*Zd+ZxZS)3r"z Nested Transformer (NesT) A PyTorch impl of : `Aggregating Nested Transformers` - https://arxiv.org/abs/2105.12723 img_sizein_chans patch_size num_levels embed_dims.r'depths num_classesrkr( drop_rateproj_drop_rateattn_drop_ratedrop_path_raternrmr weight_init global_poolct |||d}dD]M}t|}t|tj j s5t||k(rDJd|dt||}t||}t||}||_ ||_ |dx|_ |_ g|_ |xst}|xstj }| |_||_t|tj j r|d|dk(sJd|d}||zdk(sJd ||_d t)j*|d t(j,zj/dj1|_||zt5j6|j2dzdk(sJd t9||zt5j6|j2dz|_t=d||||dd d||_|j>j@|_ |j@|j2dz|_!g}tE||d}d}d }tGt|j2D]}||}|jItK|j2||j:|jB||||||f| | | | |||||d||xjtM||d|gz c_ |}|dz}tjN||_(||dfi||_)tU|j|jfd|i|\}}||_+tjX| |_-||_.|j_|y)a Args: img_size (int, tuple): input image size in_chans (int): number of input channels patch_size (int): patch size num_levels (int): number of block hierarchies (T_d in the paper) embed_dims (int, tuple): embedding dimensions of each level num_heads (int, tuple): number of attention heads for each level depths (int, tuple): number of transformer layers for each level num_classes (int): number of classes for classification head mlp_ratio (int): ratio of mlp hidden dim to embedding dim for MLP of transformer layers qkv_bias (bool): enable bias for qkv if True drop_rate (float): dropout rate for MLP of transformer layers, MSA final projection layer, and classifier attn_drop_rate (float): attention dropout rate drop_path_rate (float): stochastic depth rate norm_layer: (nn.Module): normalization layer for transformer layers act_layer: (nn.Module): activation layer in MLP of transformer layers pad_type: str: Type of padding to use '' for PyTorch symmetric, 'same' for TF SAME weight_init: (str): weight init scheme global_pool: (str): type of pooling operation to apply to final feature map Notes: - Default values follow NesT-B from the original Jax code. - `embed_dims`, `num_heads`, `depths` should be ints or tuples with length `num_levels`. - For those following the paper, Table A1 may have errors! - https://github.com/google-research/nested-transformer/issues/2 r,rr'rz Require `len(z) == num_levels`rErrz Model only handles square inputsz*`patch_size` must divide `img_size` evenlyr?cpuzUFirst level blocks don't fit evenly. Check `img_size`, `patch_size`, and `num_levels`F)rrrrflattenT) stagewiseN)rkr(r*r)rlrnrmrzlevels.)num_chs reductionmoduler@ pool_typers)0r1r2locals isinstance collectionsabcSequencerrrr num_featureshead_hidden_size feature_inforr r~rrrr_arangelongfliptolistrrrrdrr patch_embed num_patchesrrrappendrdictrlevelsrrrr6 head_drophead init_weights)!r8rrrrrr'rrrkr(rrrrrnrmrrrr-r.r9 param_name param_valuerdp_ratesprev_dim curr_striderr&rr;s! r<r2z Nest.__init__'sf  /? dJ (:.K+{'?'?@;':5czlRb7cc5 d +Yz*:6 )Ij))4 &:&v.&  4>rNBD1,9 ( "$ h  8 8 9A;(1+- Q/Q Q-{H*$)W+WW)$ ZUZZ XX^^_`ahhjJ&$))DOOA4F*GG1L e d eL x:5$))DOOTUDV:WWX& ! m     ++77**dooa.@@,^VtT s4??+, AQ-C MM)"! q $!(("1+%#! ! $   $skT[\][^R_"`!a a H 1 K- .mmV, z"~44 .d.?.?AQAQo]holno T&I.  +&r=c|dvsJd|vr tj|j nd}|jD]}t |j dddt tt||y) N)nlhbrrrB{Gz?rDr@stdab) head_bias) rlogrrrrrr_init_nest_weights)r8moderlevels r<rzNest.init_weightssf|###39T>TXXd..//r [[ ?E %//sbA > ?G.)DdKr=cltt|jDchc]}d|d c}Scc}w)Nzlevel.z .pos_embed)rrr)r8rs r<no_weight_decayzNest.no_weight_decays-05c$++6F0GH1&:&HHHs 1c2td|rdnddfddg}|S)Nz ^patch_embedz^levels\.(\d+)z*^levels\.(\d+)\.transformer_encoder\.(\d+))z"^levels\.(\d+)\.(?:pool|pos_embed))r)z^norm)i)stemblocks)r)r8coarsematchers r< group_matcherzNest.group_matchers0 &,"2_aef=$ r=c4|jD] }||_ yr|)rr)r8enablels r<set_grad_checkpointingzNest.set_grad_checkpointings *A#)A  *r=returnc|jSr|)r)r8s r<get_classifierzNest.get_classifiers yyr=cp||_t|j|j|\|_|_y)N)r)rrrrr)r8rrs r<reset_classifierzNest.reset_classifiers2&&7   t//;'H#$)r=rPindicesr stop_early output_fmtintermediates_onlyc |dvsJdg}tt|j|\}} |j|}t|jdz } t j js|s |j} n|jd| dz} t| D]q\} } | |}| |vs|rL| | k(rG|j|jddddjdddd}|j|a|j|s|r|S | k(r5|j|jddddjdddd}||fS)a Forward features that returns intermediates. Args: x: Input image tensor indices: Take last n blocks if int, all if None, select matching indices if sequence norm: Apply norm layer to compatible intermediates stop_early: Stop iterating over blocks when last desired intermediate hit output_fmt: Shape of intermediate feature outputs intermediates_only: Only return intermediate features Returns: )NCHWzOutput shape must be NCHW.rNrr@r/) rrrrrr_r`r enumeraterrHr)r8rPrrrrr intermediates take_indices max_indexlast_idxstagesfeat_idxstagex_inters r<forward_intermediateszNest.forward_intermediatessU*Y&D(DD& "6s4;;7G"Q i   Q t'!+ 99 ! ! #:[[F[[)a-0F(0 ,OHeaA<'H0"ii !Q1(=>FFq!QPQRG!((1!((+ ,  x  !))Aq!Q/088Aq!DA-r= prune_norm prune_headctt|j|\}}|jd|dz|_|rtj|_|r|j dd|S)z@ Prune layers not required for specified intermediates. Nrrr)rrrr rurr)r8rrrrr s r<prune_intermediate_layerszNest.prune_intermediate_layers s]#7s4;;7G"Q ikk.9q=1  DI   ! !!R (r=c|j|}|j|}|j|jddddjdddd}|S)Nrr@r/r)rrrrHrs r<forward_featureszNest.forward_featuressR   Q  KKN IIaii1a+ , 4 4Q1a @r= pre_logitscp|j|}|j|}|r|S|j|Sr|)rrr)r8rPrs r< forward_headzNest.forward_head!s5   Q  NN1 q0DIIaL0r=cJ|j|}|j|}|Sr|)rrrs r<rYz Nest.forward&s'  ! !! $   a r=)r/r?r/ir?rZr@r@rjTrBrBrBg?NNrravgNNrF)T)r#)NFFrF)rFT) r[r\r]r^rdrrerbrrr rrr2r_r`ignorerrrrrrTensorrrrrrrrYrfrgs@r<r"r" s *9)3&0#!!!$&$&$'4837!$-H'H'H' H'  H' c3h H'S#XH'#s(OH'H'H'H'H'"H'"H'"H' !bii1!H'" RYY0#H'$%H'&'H'()H'T YYLL YYII YY   YY** YY HCHcH8<$$',1 ||1 eCcN341  1  1  1 !%1  tELL!5tELL7I)I#JJ K1 j./$# 3S >*  1$1 r=rnamercVt|tjr|jdrDt |j dddtj j|j|yt |j ddd|j*tj j|jyyt|tjrPt |j ddd|j*tj j|jyyy)zn NesT weight initialization Can replicate Jax implementation. Otherwise follows vision_transformer.py rrrDr@rN) rr r4 startswithrweightinit constant_r0zeros_Conv2d)rr(rs r<rr,s&"))$ ??6 " &--SB! < GG  fkk9 5 &--SB! <{{&v{{+' FBII &fmma8 ;; " GGNN6;; ' # 'r=ctjd|j|j|jd}|jdd\}}tt j ||z}t |tt j |jdddd}tj|||gdd}t|jddddtt j |}|S) z Rescale the grid of position embeddings when loading from state_dict Expected shape of position embeddings is (1, T, N, C), and considers only square images z$Resized position embedding: %s to %sr@rr/rbicubicF)sizer align_corners) _loggerinforFrdrrrrHrJ interpolater)posemb posemb_newseq_length_oldnum_blocks_newseq_length_newsize_news r<resize_pos_embedr=>s  LL7zGWGWX\\!_N%/%5%5a%:"NN499^N:; ? G G1aQR SF ]]68(<9\a bF fnnQ1a0#dii6O2P QF Mr=c|jDcgc]}|jds|}}|D]E}||jt||jk7s*t ||t||||<G|Scc}w)z4 resize positional embeddings of pretrained weights pos_embed_)keysr*rFgetattrr=) state_dictmodelrVpos_embed_keyss r<checkpoint_filter_fnrEOs!+!2QAall<6PaQNQ O a=  '%"3"9"9 9,Z]GE1tddddd|}tdd|i|}|S) Nest-B @ 224x224 rrr rrLrs) nest_baserrNrLrM model_kwargsrCs r<r_r_|s<W"jWOUWL  L L| LE Lr=c >tddddd|}tdd|i|}|S) Nest-S @ 224x224 `ir/ r rrLrs) nest_smallr`ras r<rkrks3e>ZPZe^deL  M* M ME Lr=c >tddddd|}tdd|i|}|S) Nest-T @ 224x224 rerhr@r@rZrrLrs) nest_tinyr`ras r<roros3d>ZPYd]cdL  L L| LE Lr=c b|jddtddddd|}td d|i|}|S) r^rsamerrr rrLrs) nest_base_jx setdefaultrrNras r<rrrrsL j&)W"jWOUWL  OJ O, OE Lr=c b|jddtddddd|}td d|i|}|S) rdrrqrerhr rrLrs) nest_small_jxrsras r<rvrvsC j&)e>ZPZe^deL  PZ P< PE Lr=c b|jddtddddd|}td d|i|}|S) rmrrqrerhrnrrLrs) nest_tiny_jxrsras r<rxrxsC j&)d>ZPYd]cdL  OJ O, OE Lr=rrrvrx) jx_nest_base jx_nest_small jx_nest_tiny)rrBr%r$)Jr^collections.abcrloggingr functoolsrtypingrrrrrr_torch.nn.functionalr functionalrJ timm.datar r timm.layersr rrrrrrrrrrr_builderr _featuresr _features_fxr _manipulaterr _registryrr r!__all__ getLoggerr[r4rr$rirrdrrrr"rrerr=rErNr[ default_cfgsr_rkrorrrvrxrsr=r<rs" 55 A    ++34YY ( '  H %3 3l/ryy/dryy:  C    c    C% C%LI299IX(ryy((U($"  %6 F6"W5#g6"W5& TdTH"$"'r=