^jq6dZddlZddlmZddlmZmZmZmZm Z m Z ddl Z ddl m Z ddlm cmZddlmZmZmZmZddlmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%ddl&m'Z'dd l(m)Z)dd l*m+Z+dd l,m-Z-dd l.m/Z/m0Z0dd l1m2Z2m3Z3dgZ4ejje6Z7de8de8de jrde jrfdZ:e-e:de jrde jrde jrdee8e8fdee8e8fde jrf dZ;Gdde jxZ=Gdde jxZ>de jrde8dee jree8e8fffdZ? d8de jrde8d ee8e8fd!eee8e8fde jrf d"Z@Gd#de jxZAd$ZBd9d%ZCe2eCd&d'd(eedd)d*+eCd,d'd(eedd)d*+eCd-d'd(eedd)d*+eCeed.d/d01d2ZDd:d3ZEe3d:deAfd4ZFe3d:deAfd5ZGe3d:deAfd6ZHe3d:deAfd7ZIy);a+ Vision Transformer (ViT) in PyTorch A PyTorch implement of Vision Transformers as described in: 'Exploring Plain Vision Transformer Backbones for Object Detection' - https://arxiv.org/abs/2203.16527 'Segment Anything Model (SAM)' - https://github.com/facebookresearch/segment-anything/ N)partial)CallableListOptionalTupleTypeUnion)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STDIMAGENET_INCEPTION_MEANIMAGENET_INCEPTION_STD) PatchEmbedMlpDropPathcalculate_drop_path_rates PatchDropout LayerNorm2d LayerScaleClassifierHeadNormMlpClassifierHeadFormatresample_abs_pos_embed_nhwcRotaryEmbeddingCatapply_rot_embed_cat to_2tupleuse_fused_attn)Final)build_model_with_cfg)feature_take_indices)register_notrace_function) checkpointcheckpoint_seq)generate_default_cfgsregister_modelVisionTransformerSAMq_sizek_sizerel_posreturnctdt||zdz }|jd|k7rjtj|j d|jddj ddd|d}|j d|j dd}n|}tj|tjdddft||z d z}tj|tjdddft||z d z}||z |dz t||z d zz}||jS) a\ Get relative positional embeddings according to the relative positions of query and key sizes. Args: q_size (int): size of query q. k_size (int): size of key k. rel_pos (Tensor): relative position embeddings (L, C). Returns: Extracted positional embeddings according to relative positions. rrlinear)sizemode)dtypeN?) intmaxshapeF interpolatereshapepermutetorcharangefloat32long)r'r(r) max_rel_distrel_pos_resizedq_coordsk_coordsrelative_coordss m/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/vision_transformer_sam.py get_rel_posrD4s*q3vv..23L}}Q<'-- OOAw}}Q/ 4 < bhwkzbhwc,wkc->bhwkNr-)rDr5r8r:einsum)rFrGrHr'r(q_hq_wk_hk_wRhRwB_dimr_qrel_hrel_w attn_biass rCget_decomposed_rel_pos_biasrXWs(HCHC S#y )B S#y )BIAq# ))AsC %C LL)3 3E LL)3 3EaAq$&'%1aq0@*AAI   RsC#I 66rEceZdZUeeed<dddddejdddddf deded ed ed e d e d e ejdede e eefde ejffd ZdZxZS) Attention fused_attnTFNrS num_headsqkv_biasqk_norm attn_drop proj_drop norm_layer use_rel_pos input_sizeropec | | d} t|||zdk(sJd||_||z|_|jdz|_t |_tj||dzfd|i| |_ |r||jfi| ntj|_ |r||jfi| ntj|_ tj||_tj||fi| |_tj||_||_|j"r| J| Jdtj$t'j(d| dzd z |jfi| |_tj$t'j(d| d zd z |jfi| |_| |_y) Ndevicer1rz$dim should be divisible by num_headsgbiaszBInput size must be provided if using relative positional encoding.r,r)super__init__r^head_dimscalerr[nnLinearqkvIdentityq_normk_normDropoutraprojrbrd Parameterr:zerosrGrHrf)selfrSr^r_r`rarbrcrdrerfrir1dd __class__s rCrmzAttention.__init__|s / Y!#K%KK#"y( ]]d* (*99S#'??B?9@j5"5bkkm 9@j5"5bkkm I.IIc3-"- I.&   < <& TS T& \\%++a*Q-6G!6KT]]*a^`*abDN\\%++a*Q-6G!6KT]]*a^`*abDN rEc|j\}}}}||z}|j||d}|j|j||d|jdj ddddd}|jd||jz|dj d\}} } |j||j| } }|jr(t||j|j||f||f} n^d} |jP|jj} t|| j!| }t| | j!| } |j"rQt$j&j(j+|| | | |j,r|j.j0nd}nS||j2z}|| j5d dz} | | | z} | j7d } |j/| } | | z}|j||j|dj5ddj||d}|j9|}|j;|}|j|||d}|S) Nr-rjr,rrr]) attn_mask dropout_p)rS)r5r8rrviewr^r9unbindrtrurdrXrGrHrf get_embedrtype_asr[r:rp functionalscaled_dot_product_attentiontrainingrapro transposesoftmaxrwrb)rzxrQHWrRNrrrFkvrWrfattns rCforwardzAttention.forwards6WW 1a E IIaB hhqkq!Q;CCAq!QPQR++aT^^!3Q;BB1E1a{{1~t{{1~1   3At~~t~~XY[\W]`acd_efIIyy$yy**,'4088;'4088; ??##@@1a#.2mm$..**AA DJJAq{{2r**D$i'<>$'DqA FF1dnna , 6 6q! < D DQ2 N IIaL NN1  FF1aB rE)__name__ __module__ __qualname__rbool__annotations__rp LayerNormr3floatrModulerrrmr __classcell__r|s@rCrZrZysd  !!!!*,,, %48(,&&& &  &  &&RYY&&!sCx1&299%&P&rErZceZdZdddddddejej eddddddfdeded ed e d e d ed ede edede ejde ejde ejde deffd Z dZxZS)Block@TFr]NrrSr^ mlp_ratior_r`rbra init_values drop_path act_layerrc mlp_layerrd window_sizec J||d}t|||_| |fi||_t |f|||||| | |dk(r|n||f|d ||_|rt |fd|i|ntj|_ | dkDr t| ntj|_ | |fi||_ | d|t||z| |d||_|rt |fd|i|ntj|_| dkDrt| |_ytj|_y)Nrhr) r^r_r`rarbrcrdrerfrr]) in_featureshidden_featuresrdrop)rlrmrnorm1rZrrrprsls1r drop_path1norm2r3mlpls2 drop_path2)rzrSr^rr_r`rbrarrrrcrrdrrerfrir1r{r|s rCrmzBlock.__init__sC* / &*r*    !#%0A%5zK;U     FQ:cA{AbAVXVaVaVc1:R(9-R[[]*r*  i0    FQ:cA{AbAVXVaVaVc1:R(9-R[[]rEc 2|j\}}}}|}|j|}d}|jdkDrt||j\}}|j |j |j |}|jdkDrt||j||f|}||z}|j|||zd}||j|j|j|j|z}|j|||d}|S)Nrr-) r5rrwindow_partitionrrrwindow_unpartitionr8rrrr)rzrrQrrrRshortcutpad_hws rCrz Block.forwardsWW 1a JJqM,0   a (D,<,<=IAv OODHHTYYq\2 3   a "1d&6&6AGA qL IIaQ # $**Q-)@ AB B IIaAr "rE)rrrrpGELUrrr3rrrrrrmrrrs@rCrrs "!!!!+/!)+*,,,), % '2S2S2S 2S  2S  2S2S2S"%2S2SBII2SRYY2SBII2S2S2ShrErrrc L|j\}}}}|||zz |z}|||zz |z}tj|ddd|d|f}||z||z} }|j|||z|| |z||}|j ddddddj jd|||} | || ffS)aU Partition into non-overlapping windows with padding if needed. Args: x (tensor): input tokens with [B, H, W, C]. window_size (int): window size. Returns: windows: windows after partition with [B * num_windows, window_size, window_size, C]. (Hp, Wp): padded height and width before partition rrrjr,r~r-)r5r6padrr9 contiguous) rrrQrrCpad_hpad_wHpWpwindowss rCrrsJAq!Q 1{? *k 9E 1{? *k 9E a!Q5!U+,A YE B q" #[" 2C[RSTAii1aAq)446;;B [Z[\G RH rErhwrc:||n|\}}|\}}|jd||z|z|zz}|j|||z||z||d} | jddddddjj|||d} | ddd|d|ddfj} | S) a Window unpartition into original sequences and removing padding. Args: windows (tensor): input tokens with [B * num_windows, window_size, window_size, C]. window_size (int): window size. pad_hw (Tuple): padded height and width (Hp, Wp). hw (Tuple): original height and width (H, W) before padding. Returns: x: unpartitioned sequences with [B, H, W, C]. Nrr-rrjr,r~r)r5rr9r) rrrrrrrrrQrs rCrr3s)VrFB DAq aR"W 3{BCA Qk)2+ed?ed@ed8ee ej<eej<e ej<fffdAZ! dMdn |j<|$_c}*|_#tE|D*cgc]}*tId|*||& c}*|_%|rjtjBtjL||fddd!|$tO|fi|$tjL||fd"ddd#|$tO|fi|$|_(||_ n/| rtj2|_(ntO|fi|$|_(|}| rtS||f| || d$|$|_*y tW||f|| d%|$|_*y cc}*wcc}*w)'a Args: img_size: Input image size. patch_size: Patch size. in_chans: Number of image input channels. num_classes: Number of classes for classification head. global_pool: Type of global pooling for final sequence (default: 'token'). embed_dim: Transformer embedding dimension. depth: Depth of transformer. num_heads: Number of attention heads. mlp_ratio: Ratio of mlp hidden dim to embedding dim. qkv_bias: Enable bias for qkv projections if True. init_values: Layer-scale init values (layer-scale enabled if not None). drop_rate: Head dropout rate. pos_drop_rate: Position embedding dropout rate. attn_drop_rate: Attention dropout rate. drop_path_rate: Stochastic depth rate. weight_init: Weight initialization scheme. embed_layer: Patch embedding layer. norm_layer: Normalization layer. act_layer: MLP activation layer. block_fn: Transformer block layer. use_abs_pos: If True, use absolute positional embeddings. use_rel_pos: If True, add relative positional embeddings to the attention map. use_rope: If True, add rotary position embeddings to q/k in attention block. window_size: Window size for window attention blocks. If 0, not use window attention. global_attn_indexes: Indexes for blocks using global attention. Used when window_size > 0. global_pool: Global pooling type. head_hidden_size: If set, use NormMlpHead ref_feat_shape: Tuple of reference feature shapes for ROPE, (global, local) rhgư>)epsF)rrrrrk feat_ratiorrN)r)num_prefix_tokenszCROPE and relative pos embeddings should not be enabled at same timer,) in_pixels feat_shaperrSr^rr_r`rrbrarrcrrrdrrerfzblocks.)modulenum_chs reduction) kernel_sizerkrj)rpaddingrk) hidden_size pool_typer)rrr),rlrmrrprrrrr num_featuresrrgrad_checkpointing patch_embed grid_sizehasattrrrxr:ry pos_embedrvpos_dropr patch_droprsnorm_prelenrr rope_global rope_windowr Sequentialrangeblocksdict feature_infoConv2drneckrheadr),rzrrrrrrr^rr_r`rrrrrrrrrrrcrrrrrdrrrrrrrrir1r{rrref_feat_shape_globalref_feat_shape_windowdprir|s, rCrmzVisionTransformerSAM.__init__QsmJ  /B72<*a///(1.2C(D%(1.2C(D%@DD%(=1Y&$4  D  2Y&$[14  D  $D #D (>mm(5\)&#('   # $ "      (  ) ) a& & $ $ ( ,-4G+GKQ %  *+2E)ET%%4K[K[# &#$ ,QVV[P\^KLD'! yA F^   !"   J-"- !"   J-"-#DI&!+D KKM ( 8R8 "J --%#   DI'&#   DIA&#*^s 2A"N*Nc ddhS)Nr dist_tokenrrzs rCno_weight_decayz$VisionTransformerSAM.no_weight_decays \**rEc tdddgS)Nz^pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)i)stemr)r)rzcoarses rC group_matcherz"VisionTransformerSAM.group_matcher!s*-/CD  rEc||_yN)r)rzenables rCset_grad_checkpointingz+VisionTransformerSAM.set_grad_checkpointing(s "(rEr*c|jSrrrs rCget_classifierz#VisionTransformerSAM.get_classifier,s yyrEcJ||_|jj||yr)rrreset)rzrrs rCreset_classifierz%VisionTransformerSAM.reset_classifier0s&  [1rErindicesnorm stop_earlyrintermediates_onlyc |dk(sJdg}tt|j|\}} |j|}|j&|t |j|j ddz}|j|}|j|}|j|}tjjs|s |j} n|jd| dz} t| D]\} } |jr+tjjs t| |}n| |}| |vsJ|r3|j!|j#|j%dddd|j!|j%dddd|r|S|j#|j%dddd}||fS)a Forward features that returns intermediates. Args: x: Input image tensor indices: Take last n blocks if int, all if None, select matching indices if sequence norm: Apply norm layer to all intermediates stop_early: Stop iterating over blocks when last desired intermediate hit output_fmt: Shape of intermediate feature outputs intermediates_only: Only return intermediate features Returns: NCHWz&Output shape for ViT-SAM must be NCHW.Nrrjrr,)r rrrrrr5rrrr:jit is_scripting enumeraterr"appendrr9) rzrrr r!rr" intermediates take_indices max_indexrr blks rCforward_intermediatesz*VisionTransformerSAM.forward_intermediates4s*V#M%MM# "6s4;;7G"Q i   Q  >> %/! MMA MM!  OOA  MM!  99 ! ! #:[[F[[)a-0F' @FAs&&uyy/E/E/GsA&FL "((199Q1a3H)IJ!((1aA)>? @  IIaii1a+ ,-rE prune_norm prune_headctt|j|\}}|jd|dz|_|rtj|_|r|j dd|S)z@ Prune layers not required for specified intermediates. Nrrr)r rrrprsrr)rzrr.r/r*r+s rCprune_intermediate_layersz.VisionTransformerSAM.prune_intermediate_layersos]#7s4;;7G"Q ikk.9q=1  DI   ! !!R (rEc|j|}|j&|t|j|jddz}|j |}|j |}|j |}|jr5tjjst|j|}n|j|}|j|jdddd}|S)Nrrjrr,)rrrr5rrrrr:r%r&r#rrr9rzrs rCforward_featuresz%VisionTransformerSAM.forward_featuress   Q  >> %/! MMA MM!  OOA  MM!   " "599+A+A+Ct{{A.A AA IIaii1a+ ,rE pre_logitscN|r|j|dS|j|S)NT)r5r)rzrr5s rC forward_headz!VisionTransformerSAM.forward_heads$0:tyyty,L ! LrEcJ|j|}|j|}|Sr)r4r7r3s rCrzVisionTransformerSAM.forwards'  ! !! $   a rEF)Tr)NFFr$F)NFT)'rrr__doc__rrrNHWCrprrrrr3rrrstrrrrrmr:r%ignorerrrrrTensorr rr-r1r4r7rrrs@rCr&r&Js! " !!!+/"!#%%'$&$&$&!+2:&++gl+m46LL3577(-), $ %"!35!$.2PTIJJJ J  J  JJJJJJ"%JJJ!J #!J""#J$"%J&"'J()J*bii+J,!bii1-J. RYY0/J0299o1J2BII3J45J67J89J:;J<"'sCx=J>?J@AJB'smCJD%U5c?E#s(O+K%LMEJX YY++ YY   YY)) YY 2C2hsm28<$$',9 ||9 eCcN349  9  9  9 !%9  tELL!5tELL7I)I#JJ K9 z8<$# eCcN34 " M$MrEcd|v}i}|jD]6\}}|jdr|dd}|jdd}n|r2|||<8|S)z Remap SAM checkpoints -> timm z%image_encoder.patch_embed.proj.weightzimage_encoder.rNzmlp.linzmlp.fc)items startswithreplace) state_dictmodelsam_checkpointout_dictrrs rCcheckpoint_filter_fnrGsm = JNH  "1 <<( )"#A )X.A  OrEc 2|ddddddttddd |S) Nrjrr?bicubicTzpatch_embed.projzhead.fc) urlrre pool_sizecrop_pct interpolationfixed_input_sizemeanstd first_conv classifier)r r )rMkwargss rC_cfgrWs2?'0F(     rEzDhttps://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pthztimm/z apache-2.0rJr2)rM hf_hub_idlicenserRrSrrerOzDhttps://dl.fbaipublicfiles.com/segment_anything/sam_vit_l_0b3195.pthzDhttps://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pthrI)rjrZrK)rRrSrrerO)zsamvit_base_patch16.sa1bzsamvit_large_patch16.sa1bzsamvit_huge_patch16.sa1bsamvit_base_patch16_224c n|jdd}tt||ftt |dd|S)N out_indicesrjgetter)r] feature_cls)pretrained_filter_fn feature_cfg)poprr&rGr)variant pretrainedrVr]s rC_create_vision_transformerresF**]A.K  2[hG     rEc `tddddgdddd}t d d |it|fi|}|S) z# ViT-B/16 for Segment-Anything rrrr,rr\ rTrrrrr^rrrdrrd)samvit_base_patch16rrerdrV model_argsrDs rCrjrjsRB"R_D4J ' T*4 T8rDrXrrZrrrr&rGrW default_cfgsrerjrrrxr[rrErCrs ?? rr"*+33< " " '  H %33S35<<3ELL3@+&7 <<7<<7<<7c3h 7 c3h 7  \\ 7DQ QhLBIIL^ 35uUXZ]U]A^;_0gk \\ (+ 16sCx JRSXY\^aYaSbJc  \\ .K299K\ $%!% R "(