^jJdZddlmZddlmZmZmZmZmZddl Z ddl m Z ddl m cm Zddl mZddlmZmZddlmZmZmZmZmZmZmZmZddlmZmZd d lmZd d l m!Z!d d l"m#Z#d d l$m%Z%m&Z&d dl'm(Z(m)Z)dgZ*Gdde jVZ,Gdde jVZ-Gdde jVZ.Gdde jVZ/Gdde jVZ0Gdde jVZ1dedee2e2ffdZ3e#dedee2e2fd e2d!e2fd"Z4Gd#d$e jVZ5Gd%d&e jVZ6Gd'd(e jVZ7Gd)de jVZ8d>d*Z9d+Z:d?d,Z;d@d-Z<e(ee)d?d5e8fd7Z?e)d?d5e8fd8Z@e)d?d5e8fd9ZAe)d?d5e8fd:ZBe)d?d5e8fd;ZCe)d?d5e8fd<ZDe)d?d5e8fd=ZEy)Aaf DaViT: Dual Attention Vision Transformers As described in https://arxiv.org/abs/2204.03645 Input size invariant transformer architecture that combines channel and spacial attention in each block. The attention mechanisms used are linear in complexity. DaViT model defs and weights adapted from https://github.com/dingmyu/davit, original copyright below )partial)ListOptionalTupleTypeUnionN)TensorIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPathcalculate_drop_path_rates to_2tuple trunc_normal_Mlp LayerNorm2dget_norm_layeruse_fused_attn)NormMlpClassifierHeadClassifierHead)build_model_with_cfg)feature_take_indices)register_notrace_function) checkpointcheckpoint_seq)generate_default_cfgsregister_modelDaVitcBeZdZ ddededeffd ZdefdZxZS) ConvPosEncdimkactc||d}t|tj||f|d|dz|d||_|rtj |_ytj |_y)Ndevicedtyper) kernel_sizestridepaddinggroups)super__init__nnConv2dprojGELUIdentityr$)selfr"r#r$r'r(dd __class__s \/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/davit.pyr/zConvPosEnc.__init__#sn / II   F    !$2779xcP|j|}||j|z}|SN)r2r$)r5r:feats r8forwardzConvPosEnc.forward9s&yy|  r9)FNN __name__ __module__ __qualname__intboolr/r r> __classcell__r7s@r8r!r!"s> 777 7,r9r!c heZdZdZdddeddfdededed eejffd Z d e fd Z xZ S) Stemz Size-agnostic implementation of 2D image to patch embedding, allowing input size to be adjusted during model forward operation r?`Nin_chsout_chsr+ norm_layerc||d}t|t|}||_||_||_|ddk(sJt j||fd|dd||_||fi||_ y)Nr&rrKr?r*r+r,) r.r/rr+rLrMr0r1convnorm) r5rLrMr+rNr'r(r6r7s r8r/z Stem.__init__Ds / 6"   ayA~~II        w-"- r9r:ch|j\}}}}|jd||jdzz |jdz}|jd||jdzz |jdz}tj|d|d|f}|j |}|j |}|S)Nrr)shaper+FpadrRrS)r5r:BCHWpad_rpad_bs r8r>z Stem.forward^sWW 1aQ!dkk!n"44 AFQ!dkk!n"44 AF EE!a5) * IIaL IIaLr9) rArBrC__doc__rrDrr0Moduler/r r>rFrGs@r8rIrI?s[ *5... . RYY .4r9rIc `eZdZdeddfdedededeejffd Zde fd Z xZ S) Downsampler?NrLrMr*rNc||d}t|||_||_||fi||_|dzdk(|_t j||f|d|j rdn|dzd||_y)Nr&r)rrQ) r.r/rLrMrSeven_kr0r1rR) r5rLrMr*rNr'r(r6r7s r8r/zDownsample.__init__is /   v,, !Ao* II   $A+*:     r9r:c|j\}}}}|j|}|jrI|jj\}}|||zz |z}|||zz |z} t j |d|d| f}|j|}|S)Nr)rUrSrcrRr*rVrW) r5r:rXrYrZr[k_hk_wr\r]s r8r>zDownsample.forwardsWW 1a IIaL ;;yy,,HC1s7]c)E1s7]c)Ea!UQ./A IIaLr9) rArBrCrrDrr0r_r/r r>rFrGs@r8rarahsP !*5     RYY  2  r9rac BeZdZ ddedededeffd ZdZxZS)ChannelAttentionV2r" num_headsqkv_bias dynamic_scalec||d}t|||_||z|_||_t j ||dzfd|i||_t j ||fi||_y)Nr&r?bias) r.r/r-head_dimrkr0Linearqkvr2) r5r"rirjrkr'r(r6r7s r8r/zChannelAttentionV2.__init__sl /  y( *99S#'??B?IIc3-"- r9c:|j\}}}|j|j||d|j||jzj ddddd}|j d\}}}|j r ||dzz}n||jdzz}|jdd|z} | jd } | |jddzjdd}|jddj|||}|j|}|S) Nr?r)rrrKr") rUrpreshaper-permuteunbindrkrn transposesoftmaxr2 r5r:rXNrYrpqr#vattns r8r>zChannelAttentionV2.forwards ''1ahhqk!!!Q4;;T[[8HIQQRSUVXY[\^_`**Q-1a   AI ADMMT))A{{2r"Q&|||# AKKB' ' 2 22r : KK1  % %aA . IIaLr9)TTNN)rArBrCrDrEr/r>rFrGs@r8rhrhsD !"&... .  .$r9rhcBeZdZ ddededeffd ZdefdZxZS)ChannelAttentionr"rirjc||d}t|||_||z}|dz|_t j ||dzfd|i||_t j ||fi||_y)Nr&rrr?rm)r.r/riscaler0rorpr2) r5r"rirjr'r(r6rnr7s r8r/zChannelAttention.__init__sn / ")#% 99S#'??B?IIc3-"- r9r:c |j\}}}|j|j||d|j||jzj ddddd}|j d\}}}||j z}|jdd|z} | jd} | |jddzjdd}|jddj|||}|j|}|S) Nr?r)rrrKrsrtru) rUrprvrirwrxrryrzr2r{s r8r>zChannelAttention.forwards''1ahhqk!!!Q4>>1;NOWWXY[\^_abdef**Q-1a  N{{2r"Q&|||# AKKB' ' 2 22r : KK1  % %aA . IIaLr9)rFNNr@rGs@r8rrs> " ... ."  r9rceZdZdddejej dddddf dededed ed ed e ejd e ejd ededeffd Z de fdZ xZS) ChannelBlock@FTNr"ri mlp_ratiorj drop_path act_layerrNffncpe_actv2c 4| | d} t|td|d| d| |_||_||fi| |_| rt nt}||f||d| |_|dkDr t|ntj|_ td|d| d| |_ |jr^||fi| |_td|t!||z|d| |_|dkDrt||_ytj|_yd|_d|_d|_yNr&r?)r"r#r$)rirjr) in_featureshidden_featuresr)r.r/r!cpe1rnorm1rhrrr r0r4 drop_path1cpe2norm2rrDmlp drop_path2)r5r"rirrjrrrNrrrr'r(r6 attn_layerr7s r8r/zChannelBlock.__init__s' / ?3!?B? *r* +-'3C       2;R(9-R[[]?3!?B? 88#C.2.DJ #C)O 4# DH 6?^hy1DODODJDH"DOr9r:c`|j\}}}}|j|jdjdd}|j |}|j |}||j |z}|j|jddj||||}|jw|jdjdd}||j|j|j|z}|jddj||||}|S)Nr)r) rUrflattenryrrrrviewrrr)r5r:rXrYrZr[curs r8r>zChannelBlock.forwardsWW 1a IIaL  # - -a 3jjmiin $ $ IIakk!Q',,Q1a8 9 88  ! &&q!,ADOODHHTZZ]$;<rFrGs@r8rrs ""!)+*,,,!+#+#+# +#  +#  +#BII+#RYY+#+#+#+#Zr9rr: window_sizec|j\}}}}|j|||dz|d||dz|d|}|jddddddjjd|d|d|}|S)z Args: x: (B, H, W, C) window_size (int): window size Returns: windows: (num_windows*B, window_size, window_size, C) rrr?r)rKrsrUrrw contiguous)r:rrXrZr[rYwindowss r8window_partitionrsJAq!Q q!{1~%{1~qKN7JKXYN\]^Aii1aAq)446;;B AP[\]P^`abG Nr9rrZr[c|jd}|jd||dz||dz|d|d|}|jddddddjjd|||}|S)z Args: windows: (num_windows*B, window_size, window_size, C) window_size (int): Window size H (int): Height of image W (int): Width of image Returns: x: (B, H, W, C) rsrrr?r)rKrr)rrrZr[rYr:s r8window_reverser(s  bA Rk!n,a;q>.A;q>S^_`SacdeA !Q1a#..055b!QBA Hr9c eZdZUdZej j eed< d de de e e fde deffd Z de fd Z xZS) WindowAttentiona Window based multi-head self attention (W-MSA) module with relative position bias. It supports both of shifted and non-shifted window. Args: dim (int): Number of input channels. window_size (tuple[int]): The height and width of the window. num_heads (int): Number of attention heads. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True fused_attnr"rrirjcB||d}t |||_||_||_||z}|dz|_t |_tj||dzfd|i||_ tj||fi||_ tjd|_ y)Nr&rrr?rmrsru)r.r/r"rrirrrr0rorpr2Softmaxrz) r5r"rrirjr'r(r6rnr7s r8r/zWindowAttention.__init__Ds / &")#% (*99S#'??B?IIc3-"- zzb) r9r:c|j\}}}|j|j||d|j||jzj ddddd}|j d\}}}|j rtj|||}n:||jz}||jddz} |j| } | |z}|jddj|||}|j|}|S)Nr?r)rrrKrtrs) rUrprvrirwrxrrVscaled_dot_product_attentionrryrzr2) r5r:B_r|rYrpr}r#r~rs r8r>zWindowAttention.forward[s77Aqhhqk!!"aDNNArFrGs@r8rr9sg %%"**sCx* *  *.r9rceZdZdZddddej ej ddddf ded ed ed ed e d ede ejde ejde de ffd Z de fdZxZS) SpatialBlocka< Windows Block. Args: dim (int): Number of input channels. num_heads (int): Number of attention heads. window_size (int): Window size. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True drop_path (float, optional): Stochastic depth rate. Default: 0.0 act_layer (nn.Module, optional): Activation layer. Default: nn.GELU norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm rPrTrFNr"rirrrjrrrNrrc | | d} t|||_| |_||_t ||_||_td|d| d| |_ ||fi| |_ t||j f||d| |_ |dkDr t|ntj|_td|d| d| |_|jr`||fi| |_t'||z}t)d|||d| |_|dkDrt||_ytj|_yd|_d|_d|_yr)r.r/r"rrirrrr!rrrrr r0r4rrrrDrrr)r5r"rirrrjrrrNrrr'r(r6mlp_hidden_dimr7s r8r/zSpatialBlock.__init__{sN / "$[1"?3!?B? *r* #           2;R(9-R[[]?3!?B? 88#C.2.DJ y1N .# DH 6?^hy1DODODJDH"DOr9r:c 4|j\}}}}|j|jdjdd}|j |}|j ||||}dx}}|j d||j dzz |j dz} |j d||j dzz |j dz} tj|dd|| || f}|j\} } } } t||j }|j d|j d|j dz|}|j|}|j d|j d|j d|}t||j | | }|ddd|d|ddfj}|j |||z|}||j|z}|j|jddj ||||}|jw|jdjdd}||j!|j|j#|z}|jddj ||||}|S)Nr)rrrs)rUrrryrrrrVrWrrrrrrrrr)r5r:rXrYrZr[shortcutpad_lpad_tr\r]_HpWp x_windows attn_windowss r8r>zSpatialBlock.forwardsyWW 1a99Q<''*44Q: JJx  FF1aA !!!$q4+;+;A+>'>>$BRBRSTBUU!!!$q4+;+;A+>'>>$BRBRSTBUU EE!aE5%7 8ww 2r1$Q(8(89 NN2t'7'7':T=M=Ma=P'PRST yy+ $((T-=-=a-@$BRBRSTBUWXY <)9)92r B a!RaRlO & & ( FF1a!eQ  tq) ) IIakk!Q',,Q1a8 9 88  ! &&q!,ADOODHHTZZ]$;<rFrGs@r8rrns  !!!!)+*,,,!0#0#0# 0#  0#  0#0#BII0#RYY0#0#0#d%%r9rc#*eZdZddddddddeej ddd ddd d fd ed ed ededee dfdedede dedee dfde ejde ejdededededef"fd Z ejj d!dZdefd ZxZS)" DaVitStagerTspatialchannelr?rPr)rrFr)NrLrMdepth downsample attn_types.rirrrjdrop_path_ratesrN norm_layer_clrrdown_kernel_size named_blockschannel_attn_v2c||d}t|d|_|rt||f|| d||_nt j |_ g}t|D]}ddlm }g}t|D]c\}}|dk(r*|jdtd |||| | || | ||d |f5|d k(s;|jd td |||| | || | ||d |fe|r+|jt j|||jt j|Dcgc]}|d  c}t j||_ycc}w)Nr&F)r*rNr) OrderedDictr spatial_block) r"rirrjrrNrrrr channel_block) r"rirrjrrNrrrrr)r.r/grad_checkpointingrarr0r4range collectionsr enumerateappendrr Sequentialblocks)r5rLrMrrrrirrrjrrNrrrrrrr'r(r6 stage_blocks block_idxrdual_attention_blockattn_idx attn_typebr7s r8r/zDaVitStage.__init__s, / "' (tFVcmtqstDO kkmDO  u! ZI /#% '0'< #) )(//, C#"+"+!)"1)"<#0 '$/ C C 1 )+(//, C#"+"+!)"1)"<#0 '* C C 1  6##BMM+>R2S$TU##BMMBV3WQAaD3W$XYC! ZDmm\2 4Xs" Ec||_yr<)r)r5enables r8set_grad_checkpointingz!DaVitStage.set_grad_checkpointing#s "(r9r:c|j|}|jr6tjj st |j |}|S|j |}|Sr<)rrrr is_scriptingrrr5r:s r8r>zDaVitStage.forward'sS OOA   " "599+A+A+Ct{{A.A AAr9T)rArBrCrr0rrDrErstrrrr_r/rrignorerr r>rFrGs@r8rrsG #*@ !!17*5-/\\!$%!&$))K3K3K3 K3  K3 c3h K3K3K3K3K3#5#:.K3RYYK3  ?K3K3K3 "!K3"#K3$"%K3Z YY))r9rc+eZdZdZ d-dedeedfdeedfdeedfdeded ed ed ed ed eedfdedededededededededef*fd Z dZ e jjd.dZe jjd/dZe jjdej"fdZd0dedeefdZ d1de j*d eeeeefd!ed"ed#ed$edeee j*ee j*ee j*fffd%Z d2d eeeefd&ed'efd(Zd)Zd.d*efd+Zd,ZxZS)3ra DaViT A PyTorch implementation of `DaViT: Dual Attention Vision Transformers` - https://arxiv.org/abs/2204.03645 Supports arbitrary input sizes and pyramid feature extraction Args: in_chans (int): Number of input image channels. Default: 3 num_classes (int): Number of classes for classification head. Default: 1000 depths (tuple(int)): Number of blocks in each stage. Default: (1, 1, 3, 1) embed_dims (tuple(int)): Patch embedding dimension. Default: (96, 192, 384, 768) num_heads (tuple(int)): Number of attention heads in different layers. Default: (3, 6, 12, 24) window_size (int): Window size. Default: 7 mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4 qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True drop_path_rate (float): Stochastic depth rate. Default: 0.1 norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm. in_chansdepths. embed_dimsrirrrjrNrnorm_epsrrrrrr drop_ratedrop_path_rate num_classes global_poolhead_norm_firstct |||d}t|}|t|cxk(rt|k(sJJtt || }tt | | } ||_||_|dx|_|_||_ d|_ g|_ t||dfd|i||_ |d}t||d}g}t|D]n}||}t!||f|||dkD| ||||||||| | | |||d |}|}|j#||xjt%|d |d zzd | gz c_ pt'j(||_|rB||jfi||_t/|j|f||jd ||_nCt'j2|_t5|j|f||j|d||_|j7|j8y)Nr&)epsrsFrrNT) stagewise)rrrrirrrjrrNrrrrrrr)zstages.)num_chs reductionmodule) pool_typer)rrrN)r.r/lenrrrr num_featureshead_hidden_sizerr feature_inforIstemrrrrdictr0rstagesnorm_prerheadr4rapply _init_weights)!r5rrrrirrrjrNrrrrrrrrrrrrrr'r(r6 num_stagesrLdprrirMstager7s! r8r/zDaVit.__init__BsQ4  /_ S^:s6{:::::^J7XF } =8L &  4>rNBD1""':a=NZN2N A'$Oz" cA mGQiq5%#A,'#! #A%+!1 /)#$%E(F MM%   $w!ac(U\]^\_S`"a!b b 1 c4mmV, &t'8'8?B?DM&!!&..   DIKKMDM-!!&..%   DI 4%%&r9c t|tjrjt|jdt|tjr8|j +tj j|j dyyyy)Ng{Gz?)stdr) isinstancer0rorweightrminit constant_)r5ms r8rzDaVit._init_weightssZ a # !(( ,!RYY'AFF,>!!!&&!,-?' $r9c2td|rdSgdS)Nz^stemz^stages\.(\d+)))z^stages\.(\d+).downsample)r)z^stages\.(\d+)\.blocks\.(\d+)N)z ^norm_pre)i)rr)r)r5coarses r8 group_matcherzDaVit.group_matchers'(.$  5  r9cX||_|jD]}|j|y)N)r)rrr)r5rr s r8rzDaVit.set_grad_checkpointings."([[ 8E  ( ( ( 7 8r9returnc.|jjSr<)rfc)r5s r8get_classifierzDaVit.get_classifiersyy||r9cJ||_|jj||yr<)rrreset)r5rrs r8reset_classifierzDaVit.reset_classifiers&  [1r9r:indicesrS stop_early output_fmtintermediates_onlycr|dvsJdg}tt|j|\}} |j|}t|jdz } tj j s|s |j} n|jd| dz} t| D]u\} } |jr+tj j s t| |}n| |}| |vsJ|r| | k(r|j|}n|}|j|w|r|S | k(r|j|}||fS)a Forward features that returns intermediates. Args: x: Input image tensor indices: Take last n blocks if int, all if None, select matching indices if sequence norm: Apply norm layer to compatible intermediates stop_early: Stop iterating over blocks when last desired intermediate hit output_fmt: Shape of intermediate feature outputs intermediates_only: Only return intermediate features Returns: )NCHWzOutput shape must be NCHW.rN) rrrrrrrrrrrr)r5r:rrSrr r! intermediates take_indices max_indexlast_idxrfeat_idxr x_inters r8forward_intermediateszDaVit.forward_intermediatess,*Y&D(DD& "6s4;;7G"Q i IIaLt{{#a' 99 ! ! #:[[F[[)a-0F(0 .OHe&&uyy/E/E/Gua(!H<'H0"mmA.GG$$W- .  x  a A-r9 prune_norm prune_headctt|j|\}}|jd|dz|_|rtj|_|r|j dd|S)z@ Prune layers not required for specified intermediates. Nrr)rrrr0r4rr)r5rr+r,r%r&s r8prune_intermediate_layerszDaVit.prune_intermediate_layerss]#7s4;;7G"Q ikk.9q=1 KKMDM   ! !!R (r9c|j|}|jr5tjj st |j |}n|j |}|j|}|Sr<)rrrrrrrrrs r8forward_featureszDaVit.forward_featuressW IIaL  " "599+A+A+Ct{{A.A AA MM! r9 pre_logitscN|r|j|dS|j|S)NT)r2)r)r5r:r2s r8 forward_headzDaVit.forward_heads$0:tyyty,L ! Lr9cJ|j|}|j|}|Sr<)r1r4rs r8r>z DaVit.forwards'  ! !! $   a r9)r?rrr?rrJr? rPrKT layernorm2d layernormgh㈵>rTFr)FFrravgFNNFrr<)NFFr#F)rFT)rArBrCr^rDrrrErr/rrrrrrr0r_rrrr rrr*r/r1r4r>rFrGs@r8rr0s&&2*=)7  !+!,"*@!$%$)!&!$&#$$)1^'^'#s(O^'c3h ^' S#X ^'  ^'^'^'^'^'^'c3h^'^'^'"^' "!^'"#^'$%^'&"'^'()^'*+^',"-^'@-  YY   YY88  YY 2C2hsm28<$$',3 ||3 eCcN343  3  3  3 !%3  tELL!5tELL7I)I#JJ K3 n./$# 3S >*  M$Mr9cNddl}i}|jD] \}}|j|r|j|d}n,|j dd|}|j dd|}|jdd}|jd d }|jd d }|jd d}|jdd }|jdd}|jdd}|jdd}|jdd}|jdd}|||< |S)Nrr.zconvs.([0-9]+)stages.\1.downsamplezblocks.([0-9]+)stages.\1.blocksdownsample.projdownsample.convstages.0.downsamplerzwindow_attn.norm.znorm1.zwindow_attn.fn.zattn.zchannel_attn.norm.zchannel_attn.fn.z ffn.norm.znorm2.z ffn.fn.net.zmlp.z conv1.fn.dwz cpe1.projz conv2.fn.dwz cpe2.proj)reitems startswithreplacesub) state_dictmodelprefixrJout_dictr#r~s r8_convert_florence2rSs& H  "1 <<  &"%A  FF$&=q A FF%':A > II'): ; II+V 4 II)8 4 II' 1 II*H 5 II(' 2 IIk8 , IImV , IIm[ 1 IIm[ 1 '* Or9cd|vr|Sd|vr|d}d|vr t||Sddl}i}|jD]\}}|jdd|}|jdd |}|j d d }|j d d }|j dd}|j dd}|j dd}|j dd}|||<|S)z Remap MSFT checkpoints -> timm zhead.fc.weightrOz vision_tower.convs.0.proj.weightrNzpatch_embeds.([0-9]+)rEzmain_blocks.([0-9]+)rFrGrHrIrzhead.zhead.fc.znorms.z head.norm.zcpe.0rzcpe.1r)rSrJrKrNrM)rOrPrJrRr#r~s r8checkpoint_filter_fnrU4s:%z! - )Z7!*e44 H  " 1 FF+-Da H FF*,? C II'): ; II+V 4 IIgz * IIh - IIgv & IIgv &   Or9c tdt|jddD}|jd|}|jdd}|j drd}t t ||fttd| |d |}|S) Nc3&K|] \}}| ywr<r).0r rs r8 z _create_davit..Os\da\srr6 out_indicespretrained_strictT_flF)flatten_sequentialrZ)pretrained_filter_fn feature_cfgr[) tuplergetpopendswithrrrUr)variant pretrainedkwargsdefault_out_indicesrZstrictrPs r8 _create_davitriNs\i 8\8Z.[\\**],?@K ZZ+T 2F   2DkJ    E Lr9c 2|dddddttdddd |S) NrA)r?rk)rPrPgffffff?bicubicz stem.convzhead.fcz apache-2.0) urlr input_size pool_sizecrop_pct interpolationmeanr  first_conv classifierlicenser )rmrfs r8_cfgrvcs3=v9%.B!    r9ztimm/) hf_hub_idzmicrosoft/Florence-2-base)r?r:r:)rwrrnzmicrosoft/Florence-2-large)zdavit_tiny.msft_in1kzdavit_small.msft_in1kzdavit_base.msft_in1k davit_large davit_huge davit_giantzdavit_base_fl.msft_florence2zdavit_huge_fl.msft_florence2rc Ltddd}tdd|it|fi|S)Nr6r7r;rrrire) davit_tinyrrirerf model_argss r8r}r}s0\6IUcdJ  [* [Z@ZSY@Z [[r9c Ltddd}tdd|it|fi|S)Nrr rr7r;r|re) davit_smallr~rs r8rrs0\6IUcdJ  \: \jA[TZA[ \\r9c Ltddd}tdd|it|fi|S)NrrKr r|re) davit_baser~rs r8rrs0\6KWefJ  [* [Z@ZSY@Z [[r9c Ltddd}tdd|it|fi|S)Nr)r8r9r:)r<r=r>0r|re)rxr~rs r8rxrxs0\6KWfgJ  \: \jA[TZA[ \\r9c Ltddd}tdd|it|fi|S)Nrrrrirrr@r|re)ryr~rs r8ryrys0\6LXghJ  [* [Z@ZSY@Z [[r9c Ltddd}tdd|it|fi|S)N)rrr=r?)r9r:ri )r=r>rrJr|re)rzr~rs r8rzrzs0]7MYijJ  \: \jA[TZA[ \\r9c Ttddddddd}td d|it|fi|S) Nrrrr=r?Trrrirrrrre) davit_base_flr~rs r8rrs=(=DtJ  ^Z ^4 C]V\C] ^^r9c Ttddddddd}td d|it|fi|S) Nrrrr=r?Trre) davit_huge_flr~rs r8rrs?(>/DtJ  ^Z ^4 C]V\C] ^^r9)z vision_tower.rC)r.)Fr^ functoolsrtypingrrrrrrtorch.nnr0torch.nn.functional functionalrVr timm.datar r timm.layersr rrrrrrrrr_builderr _featuresr _features_fxr _manipulaterr _registryrr__all__r_r!rIrarhrrrDrrrrrrrSrUrirv default_cfgsr}rrrxryrzrrrr9r8rs 55 AHHH=*+33< ):&299&R##L$$PryyD?299?D  U38_   F  sCx  S  S    2bii2jd299dNXXveBIIeP84* % ! 6&6$(--%1%).-%1& (\e\\ ]u]] \e\\ ]u]] \e\\ ]u]] ______r9