^jtdZdgZddlZddlmZddlmZmZmZm Z m Z m Z m Z ddl Z ddlmZddlmcmZddlmZmZddlmZmZmZmZmZmZmZdd lmZdd l m!Z!dd l"m#Z#dd l$m%Z%m&Z&dd l'm(Z(m)Z)Gdde jjTZ+GddejXZ-GddejXZ.GddejXZ/GddejXZ0GddejXZ1Gdde jjXZ2GddejXZ3e#e3GddejXZ4Gd dejXZ5d!Z6d6d"Z7e)e7d#d$%e7d#&e7d#&e7d#d$%e7d#&e7d#&e7d#d$%e7d#&e7d#&e7d#d'd(d)*e7d#d+d,d)d-.d/ Z8d7d0Z9e(d7d1Z:e(d7d2Z;e(d7d3Zy)8z TinyViT Paper: `TinyViT: Fast Pretraining Distillation for Small Vision Transformers` - https://arxiv.org/abs/2207.10666 Adapted from official impl at https://github.com/microsoft/Cream/tree/main/TinyViT TinyVitN)partial)DictListOptionalTupleUnionTypeAnyIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD) LayerNorm2dNormMlpClassifierHeadDropPath trunc_normal_resize_rel_pos_bias_table_levituse_fused_attncalculate_drop_path_rates)build_model_with_cfg)feature_take_indices)register_notrace_module) checkpointcheckpoint_seq)register_modelgenerate_default_cfgsceZdZ d dedededededededeffd Zejd ZxZ S) ConvNormin_chsout_chsksstridepaddilationgroupsbn_weight_initc | | d} t |tj|||||||fddi| |_tj |fi| |_tjjj|j j|tjjj|j jdy)NdevicedtypebiasFr) super__init__nnConv2dconv BatchNorm2dbntorchinit constant_weightr,) selfr r!r"r#r$r%r&r'r*r+dd __class__s _/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/tiny_vit.pyr.zConvNorm.__init__s / IIfgr63&cW\c`bc ../B/  ?   a0c B|j|j}}|j|j|jzdzz }|j|dddddfz}|j |j |jz|j|jzdzz z }tjj|jd|jjz|jd|jdd|jj|jj|jj|jj}|jj j#||j j j#||S)Ng?rr)r#paddingr%r&)r1r3r7 running_varepsr, running_meanr4r/r0sizer&shaper#r?r%datacopy_)r8cr3wbms r;fusez ConvNorm.fuse2s5 4772 II"&&0S8 8 HHqD$,- - GGboo 1 ^^bff $ ,- - HHOO FF1I (( (!&&)QWWQR[99##TYY->->I[I[dhdmdmdtdt  v  A  !r<)rrrrrrNN) __name__ __module__ __qualname__intfloatr.r4no_gradrK __classcell__r:s@r;rrs $%111 1  1  111"1(U]]_  r<rcReZdZ ddededeej ffd ZdZxZ S) PatchEmbedr r! act_layerc||d}t|d|_t||dzdddfi||_||_t|dz|dddfi||_y)Nr)r>r)r-r.r#rconv1actconv2)r8r r!rVr*r+r9r:s r;r.zPatchEmbed.__init__Bsd /  fglAq!BrB ;glGQ1CC r<cl|j|}|j|}|j|}|SN)rZr[r\r8xs r;forwardzPatchEmbed.forwardQs. JJqM HHQK JJqMr<NN rLrMrNrOr r/Moduler.rarRrSs@r;rUrUAs=  D D DBII Dr<rUc ZeZdZ ddedededeejdef fd ZdZ xZ S) MBConvr r! expand_ratiorV drop_pathcn||d}t |t||z} t|| fddi||_||_t| | fddd| d||_||_t| |fddd||_||_ |dkDrt||_ ytj|_ y)Nr)r"rrYr"r#r$r&)r"r')r-r.rOrrZact1r\act2conv3act3rr/Identityrh) r8r r!rgrVrhr*r+r9mid_chsr:s r;r.zMBConv.__init__Ys / f|+,fg:!:r: K gw\1QAg\Y[\ K gwO1SOBO K 09B),BKKMr<c|}|j|}|j|}|j|}|j|}|j |}|j |}||z }|j |}|Sr^)rZrlr\rmrnrhro)r8r`shortcuts r;razMBConv.forwardnsq JJqM IIaL JJqM IIaL JJqM NN1  X  IIaLr<rb) rLrMrNrOrPr r/rdr.rarRrSs@r;rfrfXsSRRR R BII R  R* r<rfcReZdZ ddededeej ffd ZdZxZ S) PatchMergingdimout_dimrVc||d}t|t||dddfi||_||_t||dddfd|i||_||_t||dddfi||_y)Nr)rrrYr>r&)r-r.rrZrlr\rmrn)r8rvrwrVr*r+r9r:s r;r.zPatchMerging.__init__|s| / c7Aq!:r: K gw1aNN2N K gw1a>2> r<c|j|}|j|}|j|}|j|}|j |}|Sr^)rZrlr\rmrnr_s r;razPatchMerging.forwardsH JJqM IIaL JJqM IIaL JJqMr<rbrcrSs@r;ruru{s9  ???BII ? r<ruc neZdZ ddededeej deee efdef fd Z dZ xZ S) ConvLayerrvdepthrVrhconv_expand_ratioc||d}t |||_||_t j t |D cgc]'} t||||t|tr|| n|fi|)c} |_ ycc} wNr)) r-r.rvr|r/ Sequentialrangerf isinstancelistblocks) r8rvr|rVrhr}r*r+r9ir:s r;r.zConvLayer.__init__s /  mm5\ &  ! *9d ; !       &   & s,A5c(|j|}|Sr^)rr_s r;razConvLayer.forwards KKNr<)rk@NN) rLrMrNrOr r/rdr rPrr.rarRrSs@r;r{r{sa 46')   BII  UDK/0  %  4r<r{c eZdZddejej dddfdedeedeedeejdeejde f fd Z d Z xZ S) NormMlpNrk in_featureshidden_features out_features norm_layerrVdropc P||d} t ||xs|}|xs|}||fi| |_tj||fi| |_||_tj||_tj||fi| |_ tj||_ yr) r-r.normr/Linearfc1r[Dropoutdrop1fc2drop2) r8rrrrrVrr*r+r9r:s r;r.zNormMlp.__init__s / #2{ )8[{1b1 99[/@R@;ZZ% 99_lAbAZZ% r<c|j|}|j|}|j|}|j|}|j |}|j |}|Sr^)rrr[rrrr_s r;razNormMlp.forwardsU IIaL HHQK HHQK JJqM HHQK JJqMr<)rLrMrNr/ LayerNormGELUrOrr rdrPr.rarRrSs@r;rrs~.2*.*,,,)+&&&c]&#3- & RYY & BII &&,r<rc 6eZdZUejj eed<ee ejfed< dde de de de de e e ff fd Z ejdfd Zd ej d ejfd Zd ZddZddZddZxZS) Attention fused_attnattention_bias_cachervkey_dim num_heads attn_ratio resolutioncD||d}t |t|trt |dk(sJ||_|dz|_||_t||z|_ |j|z|_ ||_ ||_ t|_tj |fi||_tj$|||jd|zzzfi||_tj$|j|fi||_|d|dz} |d|dz} t*jj-t+j.|| fi||_|j3dt+j.| | f|t*j4di|_|j9y) Nr)r>grrattention_bias_idxsF) persistent)r-r.rtuplelenrscalerrOval_dimrwrrrrr/rrrqkvprojr4 Parameteremptyattention_biasesregister_bufferlongrreset_parameters) r8rvrrrrr*r+r9N num_offsetsr:s r;r.zAttention.__init__su / *e,ZA1EEE"_  :/0 ||i/ $$(*LL++ 99S)t||a'k/I"JQbQIIdllC626 qMJqM ) mjm3 % 2 25;;y+3\Y[3\ ]  ! KKAvUZZ @  %'! r<cRt|||r|jri|_yyyr^)r-trainr)r8moder:s r;rzAttention.trains)  d D--(*D %.4r<r*returnc4tjjs |jr|jdd|j fSt |}||jvr*|jdd|j f|j|<|j|Sr^)r4jit is_tracingtrainingrrstrr)r8r* device_keys r;get_attention_biaseszAttention.get_attention_biasess 99   !T]]((D,D,D)DE EVJ!:!::8<8M8MaQUQiQiNi8j))*5,,Z8 8r<c|j|j}|j\}}}|j|}|j |}|j |||j dj|j|j|jgd\}}} |jdddd}|jdddd}| jdddd} |jrtj||| |}n@||jz}||jddz} | |z} | j!d} | | z}|jddj#|||j$}|j'|}|S) NrY)rvrr>r) attn_mask)rr*rDrrviewrsplitrrpermuterFscaled_dot_product_attentionr transposesoftmaxreshaperwr) r8r` attn_biasBr_rqkvattns r;razAttention.forwardsW--ahh7 ''1a IIaLhhqk((1a4::DLL$,,X\XdXd;ekl:m1a IIaAq ! IIaAq ! IIaAq ! ??..q!Q)LADJJAq{{2r**D)#D<s r<)rX)rNNT)rN)rLrMrNr4rFinalbool__annotations__rrTensorrOrr.rQrr*rrarrrrRrSs@r;rrs %%sELL011 *2% % %  %  % c3h % NU]]_++ 95<<9ELL92 ' r<rceZdZdZdddddej ddfdeded ed ed ed ed edeejffd Z dZ de fdZ xZS) TinyVitBlocka5 TinyViT Block. Args: dim (int): Number of input channels. num_heads (int): Number of attention heads. window_size (int): Window size. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. drop (float, optional): Dropout rate. Default: 0.0 drop_path (float, optional): Stochastic depth rate. Default: 0.0 local_conv_size (int): the kernel size of the convolution between Attention and MLP. Default: 3 act_layer: the activation function. Default: nn.GELU rrkrYNrvr window_size mlp_ratiorrhlocal_conv_sizerVc | | d} t|||_||_|dkDsJd||_||_||zdk(sJd||z} ||f} t || |fd| d| |_|dkDr t|ntj|_ td |t||z||d| |_|dkDr t|ntj|_|d z}t!||f|d||d | |_y) Nr)rz"window_size must be greater than 0z"dim must be divisible by num_headsr)rrrk)rrrVrr>rj)r-r.rvrrrrrrr/rp drop_path1rrOmlp drop_path2r local_conv)r8rvrrrrrhrrVr*r+r9head_dimwindow_resolutionr$r:s r;r.zTinyVitBlock.__init__Rs4 / "QD DD&"Y!#I%II#)#(+6    (     2;R(9-R[[] i0    2;R(9-R[[]""3es[^ebder<c |j\}}}}||z}|}||jk(rI||jk(r:|j|||}|j|}|j ||||}n|j||jzz |jz}|j||jzz |jz} |dkDxs| dkD} | rt j |ddd| d|f}||z|| z} } | |jz} | |jz}|j || |j||j|jddj|| z|z|j|jz|}|j|}|j || ||j|j|jddj|| | |}| r|ddd|d|fj}||j|z}|jdddd}|j|}|j|||jdd}||j|j|z}|j ||||SNrr>rYr)rDrrrrrr$r contiguousrrrrr)r8r`rHWCLrspad_bpad_rr?pHpWnHnWs r;razTinyVitBlock.forwards~WW 1a E   Q$*:*:%: !Q"A ! Aq!Q"A%%D,<,<(<<@P@PPE%%D,<,<(<<@P@PPEai,519GEE!aAua78YE Bt'''Bt'''Bq"d..D4D4DaHRRSTVWX``B T--0@0@@!A ! Aq"b$"2"2D4D4DaHRRSTVWX``abdfhjlmnAa!RaRiL++- tq) ) IIaAq ! OOA  IIaA  ( (A .  , ,vvaAq!!r<rcnd|jd|jd|jd|jS)Ndim=z , num_heads=z, window_size=z , mlp_ratio=)rvrrrrs r; extra_reprzTinyVitBlock.extra_reprs@dhhZ|DNN+;<"../|DNN;KM Mr<)rLrMrN__doc__r/rrOrPr rdr.rarrrRrSs@r;rrCs $ !!!#$)+-f-f-f -f  -f  -f-f!-fBII-f^&"PMCMr<rceZdZdZdddddej ddfdededed ed ed ed ed eee efde e ejdede ejffd Z dZdefdZxZS) TinyVitStagea A basic TinyViT layer for one stage. Args: dim (int): Number of input channels. out_dim: the output dimension of the layer depth (int): Number of blocks. num_heads (int): Number of attention heads. window_size (int): Local window size. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. drop (float, optional): Dropout rate. Default: 0.0 drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None local_conv_size: the kernel size of the depthwise convolution between attention and MLP. Default: 3 act_layer: the activation function. Default: nn.GELU rrkNrYrvrwr|rrrrrh downsamplerrVcd| | d}t|||_||_| | d||| d||_n t j |_||k(sJt jt|Dcgc]+}td|||||t|tr||n|| | d|-c}|_ ycc}w)Nr))rvrwrV)rvrrrrrhrrVr) r-r.r|rwr r/rprrrrrr)r8rvrwr|rrrrrhr rrVr*r+r9rr:s r;r.zTinyVitStage.__init__s  /     !(# DO!kkmDO'> !>mm5\ &#  #'#*4Y*E)A,9 /#   &# $ &#s30B-c|j|}|jdddd}|j|}|jdddd}|Sr)r rrr_s r;razTinyVitStage.forwardsJ OOA  IIaAq ! KKN IIaAq !r<rc:d|jd|jS)Nrz, depth=)rwr|rs r;rzTinyVitStage.extra_reprsdll^8DJJ<88r<)rLrMrNrr/rrOrPr rrr rdr.rarrrRrSs@r;r r s. "3548#$)+.$.$.$ .$  .$  .$.$.$UDK/0.$!bii1.$!.$BII.$`9C9r<r cNeZdZddddddddd d d ddejd d fd edededeedfdeedfdeedfdeedfdededede dedede ejffd Z d6de fdZ d6dejde fd Zej j"d!Zej j"d"Zej j"d7d#Zej j"d6d$Zej j"d%ejfd&Zd8dedeefd'Z d9d(ej2d)eeeeefd*e d+e d,ed-e d%eeej2eej2eej2fffd.Z d:d)eeeefd/e d0e fd1Zd2Zd7d3e fd4Zd5Z xZ!S);rrYavg)`ir>r>r>)rYr rrrrrrk皙?FNin_chans num_classes global_pool embed_dims.depthsr window_sizesr drop_ratedrop_path_rateuse_checkpointmbconv_expand_ratiorrVct|||d}||_||_||_t ||_||_| |_td||d|d||_ t| t|}tj|_|jj }|d}g|_t%|j D]}|dk(rt'd|||||d||| d|}n`||}|t|d|t|d|dz} t)d||dz ||||||||j| | | t*|d |}|}|dz}|jj-||xj"t/||d| gz c_|d x|_|_t5t6d }t9|j0|f||d ||_|j=dy)Nr)r)r r!rV)rvr|rVrhr}r) rvrwr|rrrrrrhr rVr>zstages.)num_chs reductionmodulergh㈵>)rA) pool_typerF needs_resetr)r-r.rrrr num_stagesrgrad_checkpointingrU patch_embedrsumr/rstagesr# feature_inforr{r rurdict num_featureshead_hidden_sizerrrhead init_weights)r8rrrrrrr rr!r"r#r$rrVr*r+r9dprr#prev_dim stage_idxstagerw norm_layer_cfr:s r;r.zTinyVit.__init__sJ&  /&   f+""0% qM   (F Dmmo !!((a=t/ jIA~!  +'!"46)#45&9   %Y/!$S );%zTinyVit._init_weights\se a # !(( ,vv!!!!&&!," WQ(:;   <[r<cdhSNrrrs r;no_weight_decay_keywordsz TinyVit.no_weight_decay_keywordsds "##r<cn|jjDchc] }d|vs| c}Scc}wrC) state_dictkeysr_s r;no_weight_decayzTinyVit.no_weight_decayhs.??,113Oa7IQ7NOOOs 22c,td|rdnddg}|S)Nz ^patch_embedz^stages\.(\d+))z^stages\.(\d+).downsample)r)z^stages\.(\d+)\.\w+\.(\d+)N)stemr)r2)r8coarsematchers r; group_matcherzTinyVit.group_matcherls' (.$455 r<c||_yr^)r-)r8enables r;set_grad_checkpointingzTinyVit.set_grad_checkpointingws "(r<rc.|jjSr^)r5fcrs r;get_classifierzTinyVit.get_classifier{syy||r<cL||_|jj||y)N)r))rr5reset)r8rrs r;reset_classifierzTinyVit.reset_classifiers&  {;r<r`indicesr stop_early output_fmtintermediates_onlyc|dvsJdg}tt|j|\}} |j|}tj j s|s |j} n|jd| dz} t| D]Z\} } |jr+tj j s t| |}n| |}| |vsJ|j|\|r|S||fS)a Forward features that returns intermediates. Args: x: Input image tensor indices: Take last n blocks if int, all if None, select matching indices if sequence norm: Apply norm layer to compatible intermediates stop_early: Stop iterating over blocks when last desired intermediate hit output_fmt: Shape of intermediate feature outputs intermediates_only: Only return intermediate features Returns: )NCHWzOutput shape must be NCHW.Nr) rrr0r.r4r is_scripting enumerater-rr) r8r`rWrrXrYrZ intermediates take_indices max_indexr0feat_idxr:s r;forward_intermediateszTinyVit.forward_intermediatess*Y&D(DD& "6s4;;7G"Q i   Q  99 ! ! #:[[F[[)a-0F(0 (OHe&&uyy/E/E/Gua(!H<'$$Q'  (  -r< prune_norm prune_headctt|j|\}}|jd|dz|_|r|jdd|S)z@ Prune layers not required for specified intermediates. Nrr)rrr0rV)r8rWrdrer`ras r;prune_intermediate_layersz!TinyVit.prune_intermediate_layerssM#7s4;;7G"Q ikk.9q=1   ! !!R (r<c|j|}|jr6tjj st |j |}|S|j |}|Sr^)r.r-r4rr]rr0r_s r;forward_featureszTinyVit.forward_featuressU   Q   " "599+A+A+Ct{{A.A AAr< pre_logitscV|r|j||}|S|j|}|S)N)rk)r5)r8r`rks r; forward_headzTinyVit.forward_heads13=DIIaJI /DH99Q<r<cJ|j|}|j|}|Sr^)rjrmr_s r;razTinyVit.forwards'  ! !! $   a r<rFr^)NFFr\F)rFT)"rLrMrNr/rrOrrrPrr rdr.r6r>r4rignorerDrHrMrPrSrrVrr rrcrhrjrmrarRrSs@r;rrs#$*=&2)7,9!!$'#(),#$)+#Y-Y-Y- Y- c3h Y- #s(O Y-S#XY- S/Y-Y-Y-"Y-!Y-"'Y-!Y-BIIY-vII!ryy!t! YY$$ YYPP YY YY)) YY *   $r<c$d|jvr|d}|j}i}|jD]S\}}|jdrd|vr3t |j ||j dddj }|||<U|S)Nmodelrrr)rGrFitemsendswithrTrD)rFrr target_sdout_dictrrs r;checkpoint_filter_fnrxs*//##(   "IH  "1 ::+ ,   "/Yq\5G5G"5MNPPA  Or<c 0|dttddddddd |S) Nrzpatch_embed.conv1.convzhead.fc)rr)rYrzgffffff?z apache-2.0) urlrmeanr@ first_conv classifier pool_size input_sizecrop_pctlicenser )r{kwargss r;_cfgrs4%#.#    r<ztimm/iQU) hf_hub_idr)r)rYrr)rrg?)rrrr)rYr)rsquash)rrrr crop_mode) ztiny_vit_5m_224.dist_in22kz"tiny_vit_5m_224.dist_in22k_ft_in1kztiny_vit_5m_224.in1kztiny_vit_11m_224.dist_in22kz#tiny_vit_11m_224.dist_in22k_ft_in1kztiny_vit_11m_224.in1kztiny_vit_21m_224.dist_in22kz#tiny_vit_21m_224.dist_in22k_ft_in1kztiny_vit_21m_224.in1kz#tiny_vit_21m_384.dist_in22k_ft_in1kz#tiny_vit_21m_512.dist_in22k_ft_in1kc r|jdd}tt||ftd|td|}|S)N out_indices)rrr>rYT)flatten_sequentialr) feature_cfgpretrained_filter_fn)poprrr2rx)variant pretrainedrrrrs r;_create_tiny_vitr$sJ**]L9K  DkJ1     E Lr<c ntgdgdgdgdd}|j|td|fi|S)N)@i@r)r>rX rrkrrrr r"tiny_vit_5m_224r2updaterrr model_kwargss r;rr1s>&" L -z J\ JJr<c ntgdgdgdgdd}|j|td|fi|S)N)rrir)r>rXrrrrrtiny_vit_11m_224rrs r;rr>s>&" L . Kl KKr<c ntgdgdgdgdd}|j|td|fi|S)Nrrri@rrYrrrg?rtiny_vit_21m_224rrs r;rrKs>& " L . Kl KKr<c ntgdgdgdgdd}|j|td|fi|S)Nrrr)rrrrrrtiny_vit_21m_384rrs r;rrX>& % L . Kl KKr<c ntgdgdgdgdd}|j|td|fi|S)Nrrr)rr rrrtiny_vit_21m_512rrs r;rrerr<)rgro)?r__all__r functoolsrtypingrrrrr r r r4torch.nnr/torch.nn.functional functionalr timm.datar r timm.layersrrrrrrr_builderr _featuresr _features_fxr _manipulaterr _registryrrrrrdrUrfrur{rrrr rrxr default_cfgsrrrrrrrr<r;rsN +@@@ A^^^*+13<!uxx""!H. RYY F2994 @biiBjjZhM299hMV %I9299I9XPbiiPf   %"&# +/+!$($ ,0,"$($ ,0,",0 Hs, ,0 Hsh,[2&2 j  K K L L L L L L L Lr<