^j^dZddlZddlmZddlmZmZmZddlm Z ddl m Z m Z m Z mZmZmZmZmZddlZddlmZddlmZdd lmZmZdd lmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3d d l4m5Z5d d l6m7Z7d dl8m9Z9d dl:m;Z;mZ>m?Z?gdZ@eGddZAeGddZBeGddZCGddejZEGddejZFGddejZGddejd eHd!eHd"dfd#ZIGd$d%ejZJddejd eHd!eHd"dfd&ZKd'eeLd(eLd"eLfd)ZMGd*d+ejZNGd,d-ejZOd.ejd/eeLd"ejfd0ZQe9d1ejd/eeLd2eeLd"ejfd3ZRd.ejd4eeLd"ejfd5ZSe9d1ejd4eeLd2eeLd"ejfd6ZTd7eAd/eeLeLfd"ee fd8ZUGd9d:ejZVGd;dZYd.ejd4eeLd"ejfd?ZZe9d1ejd4eeLd2eeLd"ejfd@Z[GdAdBejZ\GdCdDejZ]GdEdFejZ^GdGdHejZ_GdIdJejZ`d7eAd2eeLeLfd"eAfdKZad7eCdLe d"eCfdMZbGdNdOejZc ddUeHdVeHdWeddXeddYeHdZeHd[edd\eHd]eHd^eeed_eHd`eLd"e eHe ffdaZf ddUeHdVeHdWeddceedZeHd\eHd]eHd/eeeLeLfddeLd^eeed_eHd`eLd"e eHe ffdeZg ddUeHdVeHdZeHdheHd\eHd]eHd/eeeLeLfdiedd^eeeeeeeeffd_eHd`eLd"e eHe ffdjZhd"e eHe ffdkZiejdidleCddmdndodpegdRdqrdseCddmdtdodpegdPdRdqudveCddwdxdodpefdRdQydzeCddwd{dodpefdbdRdQ|d}eCdd~d{ddpefdbddeCddd{ddpefdbddfdeCddwdxdodpefdbdRdQddeCddmdtdodpegdRdqdgddeCddwdxdodpefdbdgdeCddwd{dodpefddRdQdgddeCddwd{dodpefdbdgdTdeCdd~d{ddpefdbddfdgdeCddd{ddpefdbddfdgdeCddmdtdoddefdeCddmdtdoddehdSddeCdwdndddeCdwd{dddeCd~d{dddeCdd{dddeCdddddeCdddddeCddddddegdeCddmdddodegdeCddmdddodegdeCddmdddodegdeCddddddegdgdeCddmdddodegdgdeCddmdddodegdgdeCddwdddodegdgdfdeCddwd{ddoddegdgdeCddmdddoddehdeCddmdddodehdeCddwddddehdeCddwdddddehdRdSƫdeCdd~ddddehdRɫdeCdddddddehdRɫdeCddmddddRdTdМeideCddwddddRddМeideCddwd{dddRddМeideCdd~d{dddRddМeideCddd{dddRddМeiZkde eHejfdejd"e eHejffdׄZlddeHdeeHdeddLe d"ecf dۄZmddeHdLe d"e eHe ffd݄Zne>idend߫denddddendddendddenddenddenddenddddddenddeeddenddddend߫dendddendddend߫dend߫denddddenddidendddendddenddd end߫d end߫d end߫d end߫d end߫dend߫denddddendddddendddenddddenddddendddddendddddenddddid endd!dd"endddd#endd$endddddd%enddd&endd'ddd(endddd)endd*ddd+enddd,d-endd.endddddd/end߫d0enddd1endee2d3endddddd4endd5d6ddd7endee2id8endddddd9endd5d6ddd:endee2d;endddddd<endd5d6ddd=endee2d>endddddd?endd5d6ddd@enddAdBenddddddCendd5d6dddDenddAdEenddddddFendd5ddGdHenddAdIenddddddJendd5d6ddZoe?ddeddLe d"ecfdKZpe?ddeddLe d"ecfdLZqe?ddeddLe d"ecfdMZre?ddeddLe d"ecfdNZse?ddeddLe d"ecfdOZte?ddeddLe d"ecfdPZue?ddeddLe d"ecfdQZve?ddeddLe d"ecfdRZwe?ddeddLe d"ecfdSZxe?ddeddLe d"ecfdTZye?ddeddLe d"ecfdUZze?ddeddLe d"ecfdVZ{e?ddeddLe d"ecfdWZ|e?ddeddLe d"ecfdXZ}e?ddeddLe d"ecfdYZ~e?ddeddLe d"ecfdZZe?ddeddLe d"ecfd[Ze?ddeddLe d"ecfd\Ze?ddeddLe d"ecfd]Ze?ddeddLe d"ecfd^Ze?ddeddLe d"ecfd_Ze?ddeddLe d"ecfd`Ze?ddeddLe d"ecfdaZe?ddeddLe d"ecfdbZe?ddeddLe d"ecfdcZe?ddeddLe d"ecfddZe?ddeddLe d"ecfdeZe?ddeddLe d"ecfdfZe?ddeddLe d"ecfdgZe?ddeddLe d"ecfdhZe?ddeddLe d"ecfdiZe?ddeddLe d"ecfdjZe?ddeddLe d"ecfdkZe?ddeddLe d"ecfdlZe?ddeddLe d"ecfdmZe?ddeddLe d"ecfdnZe?ddeddLe d"ecfdoZe?ddeddLe d"ecfdpZe?ddeddLe d"ecfdqZe?ddeddLe d"ecfdrZe?ddeddLe d"ecfdsZe?ddeddLe d"ecfdtZe?ddeddLe d"ecfduZe?ddeddLe d"ecfdvZe?ddeddLe d"ecfdwZe?ddeddLe d"ecfdxZe?ddeddLe d"ecfdyZe?ddeddLe d"ecfdzZe?ddeddLe d"ecfd{Ze?ddeddLe d"ecfd|Ze?ddeddLe d"ecfd}Ze?ddeddLe d"ecfd~Ze?ddeddLe d"ecfdZe?ddeddLe d"ecfdZe?ddeddLe d"ecfdZe?ddeddLe d"ecfdZy(a MaxVit and CoAtNet Vision Transformer - CNN Hybrids in PyTorch This is a from-scratch implementation of both CoAtNet and MaxVit in PyTorch. 99% of the implementation was done from papers, however last minute some adjustments were made based on the (as yet unfinished?) public code release https://github.com/google-research/maxvit There are multiple sets of models defined for both architectures. Typically, names with a `_rw` suffix are my own original configs prior to referencing https://github.com/google-research/maxvit. These configs work well and appear to be a bit faster / lower resource than the paper. The models without extra prefix / suffix' (coatnet_0_224, maxvit_tiny_224, etc), are intended to match paper, BUT, without any official pretrained weights it's difficult to confirm a 100% match. Papers: MaxViT: Multi-Axis Vision Transformer - https://arxiv.org/abs/2204.01697 @article{tu2022maxvit, title={MaxViT: Multi-Axis Vision Transformer}, author={Tu, Zhengzhong and Talebi, Hossein and Zhang, Han and Yang, Feng and Milanfar, Peyman and Bovik, Alan and Li, Yinxiao}, journal={ECCV}, year={2022}, } CoAtNet: Marrying Convolution and Attention for All Data Sizes - https://arxiv.org/abs/2106.04803 @article{DBLP:journals/corr/abs-2106-04803, author = {Zihang Dai and Hanxiao Liu and Quoc V. Le and Mingxing Tan}, title = {CoAtNet: Marrying Convolution and Attention for All Data Sizes}, journal = {CoRR}, volume = {abs/2106.04803}, year = {2021} } Hacked together by / Copyright 2022, Ross Wightman N) OrderedDict) dataclassreplacefield)partial)AnyCallableDictListOptionalSetTupleUnion)nn)Final)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)MlpConvMlpDropPathcalculate_drop_path_rates LayerNorm LayerScale LayerScale2dClassifierHeadNormMlpClassifierHead create_attn get_act_layerget_norm_layerget_norm_act_layer create_conv2d create_pool2dtrunc_normal_tf_ to_2tuple extend_tuplemake_divisible_assert RelPosMlp RelPosBias RelPosBiasTfuse_fused_attnresize_rel_pos_bias_table)build_model_with_cfg)feature_take_indices)register_notrace_function) named_applycheckpoint_seq)generate_default_cfgsregister_model) MaxxVitCfgMaxxVitConvCfgMaxxVitTransformerCfgMaxxVitcdeZdZUdZdZeed<dZeed<dZ e ed<dZ eed<dZ eed <dZ eed <d Ze ed <d Ze ed <dZeed<dZeed<dZeed<dZeed<dZeeeefed<dZeeeefed<dZeed<dZeed<dZee ed<dZeed<dZeed<d Zeed!<d"Ze ed#<d$Z y)%r7z-Configuration for MaxxVit transformer blocks. dim_headT head_first@ expand_ratio expand_first shortcut_bias attn_bias attn_drop proj_dropavg2 pool_typebias rel_pos_type rel_pos_dimpartition_ratioN window_size grid_sizeF no_block_attn use_nchw_attn init_valuesgelu act_layer layernorm2d norm_layer layernorm norm_layer_clư>norm_epsc|jt|j|_|j9t|j|_|j|j|_yyyN)rMr$rLselfs ^/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/maxxvit.py __post_init__z#MaxxVitTransformerCfg.__post_init__ms\ >> %&t~~6DN    '()9)9:D ~~%!%!1!1& ()!__name__ __module__ __qualname____doc__r;int__annotations__r<boolr>floatr?r@rArCrDrFstrrHrJrKrLr rrMrNrOrPrRrTrVrXr^r_r]r7r7Ts7HcJL%L$M4ItIuIuIsL#KOS-1K%S/*1+/Ixc3h(/M4M4#'K%'Is#J#$M3$He2r_r7c<eZdZUdZdZeed<dZeed<dZ e ed<dZ e ed <d Z e ed <d Ze ed <dZe ed<dZeed<dZeed<dZeed<dZeed<d Ze ed<dZeed<dZeed<dZeed<dZeeed<dZeed <dZeed!<dZeed"<d#Zeeed$<d%Zy#)&r6z-Configuration for MaxxVit convolution blocks.mbconv block_typer=r>T expand_output kernel_sizer- group_sizeF pre_norm_act output_biasdw stride_moderErFdownsample_pool_typepadding attn_earlyse attn_layersiluattn_act_layer? attn_ratiorWrPrQrRrTrVNrXc|jdvsJ|jdk(}|js |rdnd|_|js |sd|_|j |rdnd|_|jxs |j |_y)N)rkconvnextrk batchnorm2drSrUh㈵>rW)rlrTrVrXrurF)r\ use_mbconvs r]r^zMaxxVitConvCfg.__post_init__st"8888__0 /9m}DO!!*!,D  == $.DDDM$($=$=$O!r_)r`rarbrcrlrhrer>rgrmrfrordrprqrrrtrFrurwrxrzr|r~rPr rRrTrVrXr^rir_r]r6r6vs7JL%M4KJL$KKIs &#&GSJJ NC J#'K%'IsJM3 $Hhuo$ Pr_r6ceZdZUdZdZeedfed<dZeedfed<dZ ee e ee dffdfed<d Z e eeeeffed <d Z eed <ee Zeed<ee Zeed<dZeeed<dZe ed<y)r5z!Configuration for MaxxVit models.`. embed_dimrnrdepths)CrTrrl@ stem_widthF stem_bias)default_factoryconv_cfgtransformer_cfgNhead_hidden_sizevit_eff weight_init)r`rarbrcrrrdrerrlrrhrrrfrr6rr7rrr rrir_r]r5r5s+!4IuS#X4*FE#s(O*:NJeCsCx01367N.0Jc5c?*+0It$^DHnD-2CX-YO*Y&*hsm* K r_r5ceZdZUdZeeed< ddedeededededed ee d e d e ffd Z dd e jdee jde jfdZxZS) Attention2dz)Multi-head attention for 2D NCHW tensors. fused_attndimdim_outr;rGr?r< rel_pos_clsrCrDc | | d} t||xs|}|r|n|} | |z|_||_||_|dz|_t |_tj|| dzdfd|i| |_ |r|dd|ji| nd|_ tj||_ tj| |dfd|i| |_tj| |_y)  Args: dim: Input dimension. dim_out: Output dimension (defaults to input dimension). dim_head: Dimension per attention head. bias: Whether to use bias in qkv and projection. expand_first: Whether to expand channels before or after qkv. head_first: Whether heads are first in tensor layout. rel_pos_cls: Relative position class to use. attn_drop: Attention dropout rate. proj_drop: Projection dropout rate. devicedtypernr-rG num_headsNri)super__init__rr;r<scaler+rrConv2dqkvrel_posDropoutrCprojrDr\rrr;rGr?r<rrCrDrrdddim_attn __class__s r]rzAttention2d.__init__s4 / .S*7!X-  $% (*99S(Q,CCCFQ{BT^^BrBW[ I.IIhDDD I.r_xshared_rel_posreturnc|j\}}}}|jrP|j|j||j|j dzdj dd\}}} nK|j|j|d|j|j djd\}}} |jrd} |j|jj} n||} tjjj|j!ddj#|j!ddj#| j!ddj#| |j$r|j&j(ndj!ddj|d||}n||j*z}|j!dd|z} |j|j| } n|| |z} | j-d} |j'| } | | j!ddzj|d||}|j/|}|j1|}|S) Nrnrrr-rB attn_mask dropout_p)shaper<rviewrr;chunkreshapeunbindrrget_biastorchr functionalscaled_dot_product_attention transpose contiguoustrainingrCprsoftmaxrrD) r\rrBrHWqkvrAattns r]forwardzAttention2d.forwardsWW 1a ??hhqk&&q$..$--!:KRPVVWX^_V`GAq!hhqk))!Q rRYYZ[\GAq! ??I||' LL113 +* ##@@ B#..0 B#..0 B#..0#.2mm$..** A iB2q! 4 DJJA;;r2&*D||'||D)+n,<>$'DT^^B++11!RA>A IIaL NN1 r_ Nr:TTTNrBrBNNrZr`rarbrcrrfrerdr r rgrrTensorr __classcell__rs@r]rrs3d  &*!%#.2!!(/(/c](/ (/  (/  (/(/"(+(/(/(/T##x 7M#Y^YeYe#r_rceZdZUdZeeed< ddedeededededed ee d e d e ffd Z dd e jdee jde jfdZxZS) AttentionClz/Channels-last multi-head attention (B, ..., C).rrrr;rGr?r<rrCrDc | | d} t||xs|}|r||kDr|n|} | |zdk(sJd| |z|_||_||_|dz|_t |_tj|| dzfd|i| |_ |r|d d|ji| nd|_ tj||_ tj| |fd|i| |_tj| |_y) rrrz(attn dim should be divisible by head_dimrrnrGrNri)rrrr;r<rr+rrLinearrrrrCrrDrs r]rzAttentionCl.__init__s4 / .S*w}7#("a'S)SS'!X-  $% (*99S(Q,@T@R@FQ{BT^^BrBW[ I.IIhAdAbA I.r_rrrcV|jd}|jdd}|jr`|j|j|d|j|j dzj ddjdd\}}}n[|j|j|dd|j|j j ddjd\}}}|jr~d}|j|jj}n||}tjjj!|||||j"r|j$j&nd}ns||j(z}||j d dz} |j|j| | } n|| |z} | j+d} |j%| } | |z}|j ddj|d z}|j-|}|j/|}|S) Nrrrnr-rrrBrrr)r)rr<rrrr;rrrrrrrrrrrrrCrrrrrD) r\rrr restore_shaperrrrArs r]rzAttentionCl.forward,s GGAJ ??hhqk&&q"dnndmma>OPZZ[\^_`ffghnofpGAq!hhqk))!RDNNDMMR\\]^`abiijklGAq! ??I||' LL113 +* ##@@1a#.2mm$..**AA DJJAq{{2r**D||'||D|H+n,<>$'DqA KK1  % %me&; < IIaL NN1 r_rrZrrs@r]rrs9d  &*!%#.2!!)/)/c])/ )/  )/  )/)/"(+)/)/)/V##x 7M#Y^YeYe#r_rc |eZdZdZ d dededededef fd Zdejd ejfd Z xZ S) Downsample2da5A downsample pooling module supporting several maxpool and avgpool modes. * 'max' - MaxPool2d w/ kernel_size 3, stride 2, padding 1 * 'max2' - MaxPool2d w/ kernel_size = stride = 2 * 'avg' - AvgPool2d w/ kernel_size 3, stride 2, padding 1 * 'avg2' - AvgPool2d w/ kernel_size = stride = 2 rrrFrwrGct||dvsJ|dk(rtddd|xsd|_nS|dk(rtdd|xsd |_n6|d k(rtd ddd |xsd |_ntd d|xsd |_||k7r!t j ||d||| |_yt j|_y)z Args: dim: Input dimension. dim_out: Output dimension. pool_type: Type of pooling operation. padding: Padding mode. bias: Whether to use bias in expansion conv. )maxmax2avgrErrnrr-)rostriderwrr)rwrF)rorcount_include_padrw)rGrrN)rrr"poolrrexpandIdentity) r\rrrFrwrGrrrs r]rzDownsample2d.__init__[s$ ::::  %e1glYZ[DI & %eQ 1EDI % %1Q%QXQ]\]_DI&eQ 1EDI '>))C!$vUZ[DK++-DKr_rrcJ|j|}|j|}|SrZ)rrr\rs r]rzDownsample2d.forward~s! IIaL KKNr_)rErvTNN) r`rarbrcrdrhrfrrrrrrs@r]rrRsj$!(!(!( !(  !(  !(F%,,r_rrvmodulenameschemerct|tjtjfr|dk(rbtjj |j d|j*tjj|jyy|dk(rNt|j d|j*tjj|jyy|dk(r`tjj|j |j*tjj|jyytjj|j |jZd|vr,tjj |jdytjj|jyyy) z&Initialize transformer module weights.normal{Gz?stdN trunc_normal xavier_normalmlprW) isinstancerrrinitnormal_weightrGzeros_r#xavier_normal_xavier_uniform_)rrrs r]_init_transformerrs5&299bii01 X  GGOOFMMsO 3{{&v{{+' ~ % V]] 4{{&v{{+'  & GG " "6== 1{{&v{{+' GG # #FMM 2{{&D=GGOOFKKTO:GGNN6;;/ '!2r_c eZdZdZddedddfdedededeed ed ef fd Z dd e d dfdZ dde jdee jd e jfdZxZS)TransformerBlock2daYTransformer block with 2D downsampling. '2D' NCHW tensor layout Some gains can be seen on GPU using a 1D / CL block, BUT w/ the need to switch back/forth to NCHW for spatial pooling, the benefit is minimal so ended up using just this variant for CoAt configs. This impl was faster on TPU w/ PT XLA than the 1D experiment. r-NrBrrrrcfg drop_pathc ||d} t |tt|j|j } t |j} |dk(rqt||f|j|jd| |_ tjtd| |fi| fdt||fd|ji| fg|_n.||k(sJtj |_ | |fi| |_t#||f|j$|j&|j(||j*|j,d| |_|j0rt3|fd |j0i| ntj |_|d kDr t7|ntj |_| |fi| |_t=d |t?||j@z| |j,d | |_!|j0rt3|fd |j0i| ntj |_"|d kDrt7||_#y tj |_#y )a Args: dim: Input dimension. dim_out: Output dimension. stride: Stride for downsampling. rel_pos_cls: Relative position class. cfg: Transformer block configuration. drop_path: Drop path rate. repsr)rFrGnormdownrF)r;r?rGrrCrDrPrB in_featureshidden_featuresrRdropNri)$rrrrrTrXrrRrrFr@shortcutr Sequentialrnorm1rrr;r?rArCrDrrPrls1r drop_path1norm2rrdr>rls2 drop_path2) r\rrrrrrrrrrTrRrs r]rzTransformerBlock2d.__init__s( / ^CNN;N !#--0 Q;(gmUXUfUfmjlmDM{C.2./c3N#--N2NO4(DJ '> !>KKMDM#C.2.DJ    \\))#mmmm     PS<KS__KKdfdododq1:R(9-R[[].2.  #*:*: :;    PS<KS__KKdfdododq1:R(9-R[[]r_rrc:ttt||yNr)r1rrr\rs r] init_weightszTransformerBlock2d.init_weightssG-f=tDr_rrc ,|j||j|j|j|j ||z}||j |j |j|j|z}|S)Nr) r rrrrrrrr)r\rrs r]rzTransformerBlock2d.forwardso MM! ttxx $**Q-`n 8o/pq q $**Q-)@ AB Br_rvrZ)r`rarbrcr7rdr r rgrrhrrrrrrs@r]rrs.2)>)@!;S;S;S ;S "(+ ;S ' ;S;SzE3EEx 7MY^YeYer_rct|tjr|dk(rbtjj |j d|j *tjj|j yy|dk(rNt|j d|j *tjj|j yy|dk(r`tjj|j |j *tjj|j yy|jd|jdz|jz}||jz}tjj |j dtjd |z |j *tjj|j yyy) z&Initialize convolution module weights.rrrNrrrr-g@)rrrrrrrGrr#rro out_channelsgroupsmathsqrt)rrrfan_outs r] _init_convr!sU&"))$ X  GGOOFMMsO 3{{&v{{+' ~ % V]] 4{{&v{{+'  & GG " "6== 1{{&v{{+'((+f.@.@.CCfFYFYYG  %G GGOOFMM1diig .F G{{&v{{+'%%r_rpchannelsc&|sy||zdk(sJ||zS)z3Calculate number of groups for grouped convolution.r-rri)rpr"s r] num_groupsr$s( *$))):%%r_c eZdZdZddedddfdededed eeefd ed ef fd Zdd e ddfdZ de jde jfdZ xZS) MbConvBlockzGPre-Norm Conv Block - 1x1 - kxk - 1x1, w/ inverted bottleneck (expand).r-r-r-rBNin_chsout_chsrdilationrrc ||d} t|tt|j|j |j } t|jr|n||jz} t|j| } |dk(r5t||f|j|j|jd| |_nt#j$|_|j&dvsJd\} }}|j&dk(r||d}} n|j&d k(r||d}}n||d }}| |fd |j(i| |_| dkDr*t||f|j,|jd | |_nt#j$|_t1|| dfd |i| |_| | fi| |_t1| | |j6f||| |jd| |_i}t;|j<t>rV|j<dk(s|j<dk(r8|j@|d<tC|jD|jr|n| z|d<|jFr4tI|j<| fi|| |_%| | fi| |_&d|_'n3d|_%| | fi| |_&tI|j<| fi|| |_'t1| |dfd|ji| |_(|dkDrtS||_*yt#j$|_*y)a Args: in_chs: Input channels. out_chs: Output channels. stride: Stride for conv. dilation: Dilation for conv. cfg: Convolution block configuration. drop_path: Drop path rate. rrr)rFrGrw)r1x1rs)r-r-r-rr-r,r apply_act)rFrwr)rr*rrwryecarR rd_channelsNrGrB)+rrrr rTrRrXr&rmr>r$rprrFrrrwr rrrtrqpre_normrurr! conv1_1x1rro conv2_kxkrrzrhr|rdr~rxrse_earlyrry conv3_1x1rr)r\r(r)rr*rrrrrnorm_act_layermid_chsr stride_poolstride_1stride_2 dilation_2 attn_kwargsrs r]rzMbConvBlock.__init__s( /  !3CNNCMM!RX[XdXde S->->'FcN^N^!^_CNNG4 Q;(k+.==sX[XcXckgikDMKKMDM"7777*1' Xx ??f $&,hqkK __ %#)8A;jH#)8A;jH&vP9I9IPRP ?$VVss?W?WadalalsprsDI DI&vwQ(QbQ#G2r2 &   OO  KK     cnnc *~~%5)@+.+=+= K(-0cN_N_7el1m-n M* >>'U;URTUDM'626DJDG DM'626DJ!#..'O[OBODG&wWWTVW09B),BKKMr_rrc:ttt||yrr1rr!rs r]rzMbConvBlock.init_weightseGJv6=r_rc|j|}|j|}|j|}|j|}|j |}|j |}|j |j |}|j|}|j|j|}|j|}|j||z}|SrZ) r r0rr1rr2r3rryr4rr\rr s r]rzMbConvBlock.forwardhs==# MM!  IIaL NN1  JJqM NN1  == $ a A JJqM 77  A NN1  NN1  (r_r)r`rarbrcr6rdrrgrrhrrrrrrs@r]r&r&sQ (."0"2!PRPRPR PR CHo PR  PRPRd>3>>%,,r_r&ceZdZdZddddeddddf ded eed ed ed eeefd edede ffd Z de jde jfdZ xZS) ConvNeXtBlockzConvNeXt Block.Nr-r'TrBr(r)rorr*rconv_mlprc | | d} t||xs|}t|j} |r1t t |j |j} t}nd|j vsJt} t}||_ |dk(rt||fi| |_ nH||k7r*tj||fd|j d| |_ ntj"|_ |j$dvsJd\}}|j$d k(r|}n|}|dk(rt||fd |j&i| |_ntj"|_t+||f|||dd |j d | |_| |fi| |_||t1|j2|zf|j | d | |_|r=|j6rt9||j6fi| ntj"|_n<|j6rt=||j6fi| ntj"|_|dkDrt?||_ ytj"|_ y)ay Args: in_chs: Input channels. out_chs: Output channels. kernel_size: Kernel size for depthwise conv. stride: Stride for conv. dilation: Dilation for conv. cfg: Convolution block configuration. conv_mlp: Whether to use convolutional MLP. drop_path: Drop path rate. rrrUrr-)rorG)rrsr'rrFT)rorr* depthwiserG)rGrRrBN)!rrrrRrrrTrXrrr use_conv_mlprr rrrrrrtrurr!conv_dwrrdr>rrPrlsrrr)r\r(r)rorr*rrDrrrrrRrT mlp_layerr7 stride_dwrs r]rzConvNeXtBlock.__init__s.0 / #V!#--0  !?S\\RJI#..0 00"JI$ Q;(?B?DM w IIfga13??a^`aDMKKMDM.000!% Y ??f $ KI ! $VV^s?W?W^[]^DI DI$    $a[     w-"-     7* +      FIool7COOBrB[][f[f[hDGDGOOj#//@R@Y[YdYdYfDG09B),BKKMr_rrc|j|}|j|}|j|}|jr4|j |}|j |}|j |}n[|jdddd}|j |}|j |}|j |}|jdddd}|j||z}|SNrrrnr-) r rrHrGrrrIpermuterr@s r]rzConvNeXtBlock.forwards==# IIaL LLO    ! A A A !Q1%A ! A A A !Q1%A NN1  (r_)r`rarbrcr6rdr rrfrgrrrrrrs@r]rBrBs &* (."0"2!!ORORc]OR OR  OR CHo OR ORORORb%,,r_rBrrLcl|j\}}}}t||dzdk(d|d|ddt||dzdk(d|d|dd|j|||dz|d||dz|d|}|jddddd d j jd |d|d|}|S) z'Partition into non-overlapping windows.rheight () must be divisible by window ()r-width (rnrrrrr'rrNr)rrLrrrrwindowss r]window_partitionrWsJAq!Q A A ! #xs2QR]^_R`Qaab%cd A A ! #wqc1PQ\]^Q_P``a%bc q!{1~%{1~qKN7JKXYN\]^Aii1aAq)446;;B AP[\]P^`abG Nr_rVimg_sizec|\}}|jd}|jd||dz||dz|d|d|}|jddddddjjd|||}|S)zReverse window partition.rrr-rnrrTrrrrNrrVrLrXrrrrs r]window_reverser\s DAq bA Rk!n,a;q>.A;q>S^_`SacdeA !Q1a#..055b!QBA Hr_rMc h|j\}}}}t||dzdk(d|d|dt||dzdk(d|d|d|j||d||dz|d||dz|}|jdddddd j jd |d|d|}|S) z6Partition into overlapping windows with grid striding.rheight  must be divisible by grid r-width rrTrnrrrU)rrMrrrrrVs r]grid_partitionrasJAq!Q A !  !WQC/J9UV<.#YZ A !  !VA3.I)TU,#XY q)A,Yq\ 19Q<iPQlARTUVAii1aAq)446;;B ! iXYl\]^G Nr_c|\}}|jd}|jd||dz||dz|d|d|}|jddddddjjd|||}|S)zReverse grid partition.rrr-rnrTrrrZrVrMrXrrrrs r] grid_reverserds DAq bA Ril*A1,=y|YWX\[\]A !Q1a#..055b!QBA Hr_rcd}|jdk(rtt||j}|S|jdk(rtt|}|S|jdk(rtt |}|S)z,Get relative position class based on config.Nr)rL hidden_dimrG)rLbias_tf)rHrr(rJr)r*)rrLrs r]get_rel_pos_clsrhsuK 5 i[S__]   V #jkB    Y &l D r_c VeZdZdZdedddfdedededeffd Zd Z d Z xZ S) PartitionAttentionClzRGrid or Block partition + Attn + FFN. NxC 'channels last' tensor layout. blockrBNrpartition_typerrc *||d}t |tt|j|j }t |j} |dk(|_t|jr |jn |j|_ t||j} ||fi||_t||f|j |j"|j$| |j&|j(d||_|j,rt/|fd|j,i|nt1j2|_|dkDr t7|nt1j2|_||fi||_t=d|t?||j@z| |j(d||_!|j,rt/|fd|j,i|nt1j2|_"|dkDrt7||_#yt1j2|_#y) Nrrrkr;rGr<rrCrDrPrBrri)$rrrrrVrXrrRpartition_blockr$rLrMpartition_sizerhrrr;rAr<rCrDrrPrrrrrrrrrdr>rrr r\rrlrrrrrrTrRrrs r]rzPartitionAttentionCl.__init__ s / ^C,=,=>CLLQ !#--0 -8'4;O;OUXUbUbc%c4+>+>? *r*     \\~~#mmmm     JM:cEsE"E^`^i^i^k1:R(9-R[[]*r*  c&6&6 67    JM:cEsE"E^`^i^i^k1:R(9-R[[]r_c0|jdd}|jrt||j}nt ||j}|j |}|jrt ||j|}|St||j|}|S)Nr-rn)rrorWrprarr\rdr\rrX partitioneds r]_partition_attnz$PartitionAttentionCl._partition_attnLs771Q<   *1d.A.ABK(D,?,?@Kii ,   {D,?,?JA[$*=*=xHAr_c  ||j|j|j|j|z}||j |j |j |j|z}|SrZrrrurrrrrrs r]rzPartitionAttentionCl.forward[c )=)=djjm)L MN N $**Q-)@ AB Br_) r`rarbrcr7rdrhrgrrurrrs@r]rjrjsW#*)>)@!*S*S *S' *S  *SX r_rjceZdZdZedddfdededeffd Zdejd ejfd Z dejd ejfd Z xZ S) ParallelPartitionAttentionzQExperimental. Grid and Block partition + single FFN. NxC tensor layout. rBNrrrc ||d}t ||dzdk(sJtt|j|j }t |j}|j|jk(sJt|j|_ t||j} ||fi||_ t||dzf|j|j |j"| |j$|j&d||_t||dzf|j|j |j"| |j$|j&d||_|j,rt/|fd|j,i|nt1j2|_|dkDr t7|nt1j2|_||fi||_t=d |t?||j@z|||j&d||_!|j,rt/|fd|j,i|nt1j2|_"|dkDrt7||_#y t1j2|_#y ) z Args: dim: Input dimension. cfg: Transformer block configuration. drop_path: Drop path rate. rrrrrnrPrB)r r  out_featuresrRr Nri)$rrrrrVrXrrRrLrMr$rprhrrr;rAr<rCrD attn_block attn_gridrPrrrrrrrrrdr>rrr) r\rrrrrrrTrRrrs r]rz#ParallelPartitionAttention.__init__gs  / Qw!||^C,=,=>CLLQ !#--0 #--///'8%c4+>+>? *r* %  1H  \\~~#mmmm    %  1H  \\~~#mmmm    JM:cEsE"E^`^i^i^k1:R(9-R[[]*r*  c&6&6 67    JM:cEsE"E^`^i^i^k1:R(9-R[[]r_rrcJ|jdd}t||j}|j|}t ||j|}t ||j}|j |}t||j|}tj||gdS)Nr-rnrr) rrWrpr}r\rar~rdrcat)r\rrXpartitioned_blockx_windowpartitioned_gridx_grids r]ruz*ParallelPartitionAttention._partition_attns771Q<,Q0C0CD OO,=>!"3T5H5H(S)!T-@-@A>>*:;.0C0CXNyy(F+44r_c  ||j|j|j|j|z}||j |j |j |j|z}|SrZrwrs r]rz"ParallelPartitionAttention.forwardrxr_) r`rarbrcr7rdrgrrrrurrrs@r]rzrzasu*?)@! <S<S'<S <S| 5 5%,, 5%,,r_rzc l|j\}}}}t||dzdk(d|d|ddt||dzdk(d|d|dd|j||||dz|d||dz|d}|jddddd d j jd ||d|d}|S) z#Partition windows for NCHW tensors.rrPrQrRr-rSrrTrnrrrU)rrLrrrrrVs r]window_partition_nchwrsJAq!Q A A ! #xs2QR]^_R`Qaab%cd A A ! #wqc1PQ\]^Q_P``a%bc q!Q+a.(+a.!{1~:M{[\~^Aii1aAq)446;;B;q>S^_`SabG Nr_c |\}}|jd}|jd||dz||dz||d|d}|jddddddjjd|||}|S)z*Reverse window partition for NCHW tensors.r-rrrnrTrrrZr[s r]window_reverse_nchwrs DAq aA Rk!n,a;q>.A1kRSnVabcVdeA !Q1a#..055b!QBA Hr_c h|j\}}}}t||dzdk(d|d|dt||dzdk(d|d|d|j|||d||dz|d||dz}|jdddddd j jd ||d|d}|S) z Grid partition for NCHW tensors.rr^r_r-r`rnrrrTrrU)rrMrrrrrVs r]grid_partition_nchwrsJAq!Q A !  !WQC/J9UV<.#YZ A !  !VA3.I)TU,#XY q!Yq\1 ! #4ilASTDUVAii1aAq)446;;B9Qrrrrqs r]rzPartitionAttention2d.__init__s  / ^CNN;N !#--0 -8'4;O;OUXUbUbc%c4+>+>? *r*     \\~~#mmmm     LO??<GGBG`b`k`k`m1:R(9-R[[]*r*  c&6&6 67    LO??<GGBG`b`k`k`m1:R(9-R[[]r_rrc0|jdd}|jrt||j}nt ||j}|j |}|jrt ||j|}|St||j|}|S)Nr)rrorrprrrrrss r]ruz$PartitionAttention2d._partition_attns7723<   /43F3FGK-a1D1DEKii ,   #K1D1DhOA"+t/B/BHMAr_c  ||j|j|j|j|z}||j |j |j |j|z}|SrZrwrs r]rzPartitionAttention2d.forward(rxr_)r`rarbrcr7rdrhrgrrrrurrrs@r]rrs#*)>)@!1S1S 1S' 1S  1Sf  %,, %,,r_rc leZdZdZdeedddfdedededed ed ef fd Zdd Z d Z xZ S) MaxxVitBlockz;MaxVit conv, window partition + FFN , grid partition + FFN.r-rBNrrrrrrc L||d} t ||j|_|jdk(rt nt } | ||f|||d| |_td|||d| } |jrtnt} |jrdn| di| |_ | dddi| |_ y) a^Initialize MaxxVitBlock. Args: dim: Input channel dimension. dim_out: Output channel dimension. stride: Stride for downsampling. conv_cfg: Configuration for convolutional blocks. transformer_cfg: Configuration for transformer blocks. drop_path: Drop path rate. rrrrrrrrNrlgridri)rrrO nchw_attnrlrBr&convdictrrjrNr}r~)r\rrrrrrrrrconv_clsr;partition_layerrs r]rzMaxxVitBlock.__init__1s* / (66$,$7$7:$E=;S'b&hR[b_ab WwOyWTVW 26...FZ"1"?"?$_EcWbEc(NN+Nr_c|j%ttt||jttt||jttt ||j yr)r}r1rrr~r!rrs r]rzMaxxVitBlock.init_weightsRsM ?? &  1&A4?? SG-f=t~~NGJv6 Br_c|j|}|js|jdddd}|j|j|}|j |}|js|jdddd}|SrM)rrrNr}r~rs r]rzMaxxVitBlock.forwardXsp IIaL~~ !Q1%A ?? &"A NN1 ~~ !Q1%Ar_r) r`rarbrcr6r7rdrgrrrrrs@r]rr.srE '5'75J5L!OOO O % O 3 OOBC r_rceZdZdZddeedddfdededed ed ed ed effd Zdde ddfdZ de jde jfdZ xZS)ParallelMaxxVitBlockzYMaxVit block with parallel cat(window + grid), one FF. Experimental timm block. r-rrBNrrrnum_convrrrc 6|| d} t ||jdk(rtnt} |dkDr>| ||f|||d| g} | | ||f||d| g|dz zz } t j | |_n| ||f|||d| |_td|||d| |_ y) aa Args: dim: Input dimension. dim_out: Output dimension. stride: Stride for first conv block. num_conv: Number of convolution blocks. conv_cfg: Convolution block configuration. transformer_cfg: Transformer block configuration. drop_path: Drop path rate. rrr-r)rrrNri) rrrlrBr&rr rrzr)r\rrrrrrrrrrrconvsrs r]rzParallelMaxxVitBlock.__init__ls, / $,$7$7:$E=; a<c7c6xS\c`bcdE hwYXYVXYZ^fij^jk kE u-DI gff(V_fcefDI.k7[dkhjk r_rrcttt||jttt||j yr)r1rrrr!rrs r]rz!ParallelMaxxVitBlock.init_weightss-G-f=tyyIGJv6 Br_rc|j|}|jdddd}|j|}|jdddd}|SrM)rrNrrs r]rzParallelMaxxVitBlock.forwardsI IIaL IIaAq ! IIaL IIaAq !r_r)r`rarbrcr6r7rdrgrrhrrrrrrs@r]rrfs'5'75J5L! l l l l  l % l3 l lDC3CC%,,r_rceZdZdZddddeedddf ded ed ed ed eeefd ee ee fdededee e e fffd Z de jde jfdZxZS) MaxxVitStagezEMaxxVit stage consisting of mixed convolution and transformer blocks.rrT)rrrBNr(r)rdepth feat_size block_typesrrrc | | d} t|d|_t||}g} t |D]\}}|dk(r|nd}|dvsJ|dk(r1|j dk(rt nt}| |||f||| |d| gz } nh|d k(r&t||}| t||f|||| |d | gz } n=|d k(r| t||f|||| |d | gz } n|d k(r| t||f|||| |d | gz } |}tj| |_y)a Args: in_chs: Input channels. out_chs: Output channels. stride: Stride for first block. depth: Number of blocks in stage. feat_size: Feature map size. block_types: Block types ('C' for conv, 'T' for transformer, etc). transformer_cfg: Transformer block configuration. conv_cfg: Convolution block configuration. drop_path: Drop path rate(s). rFrr-)rrMPMrrrr)rrrrr)rrrrrN)rrgrad_checkpointingr% enumeraterlrBr&rhrrrrr blocks)r\r(r)rrrrrrrrrrrit block_striderrrs r]rzMaxxVitStage.__init__s4 / "'";6 k*, DAq%&!V6L-- --Cx,4,?,?:,M=S^8( 'l   c-oyI -( +''l c<(%$3'l d/(%$3'l FY, ZmmV, r_rrc|jr6tjjst |j |}|S|j |}|SrZ)rrjit is_scriptingr2rrs r]rzMaxxVitStage.forwardsE  " "599+A+A+Ct{{A.A AAr_)r`rarbrcr7r6rdrrrhrgr rrrrrrs@r]rrsO )1255J5L'5'735M-M-M- M-  M- S#X M-sE#J/M-3M-%M-UDK/0M-^%,,r_rceZdZdZ ddedededededed ed effd Zdd ed dfdZ de jd e jfdZ xZ S)Stemz"Stem layer for feature extraction.Nr(r)rorwrGrRrTrXc N| | d} t |t|ttfs t |}t t|||} |d|_d|_ t||d|fd||d| |_ | |dfi| |_ t|d|d|fd||d| |_ y) ae Args: in_chs: Input channels. out_chs: Output channels. kernel_size: Kernel size for convolutions. padding: Padding mode. bias: Whether to use bias. act_layer: Activation layer. norm_layer: Normalization layer. norm_eps: Normalization epsilon. rrrrr)rrwrGr-N)rrrlisttupler$rr r)rr!conv1rconv2)r\r(r)rorwrGrRrTrXrrrr5rs r]rz Stem.__init__s0 / 'D%=1(G !3J !JPXYr{  "671:{o1V]dholno #GAJ5"5 "71:wqz;sqZahlsprs r_rrc:ttt||yrr=rs r]rzStem.init_weightsr>r_rcl|j|}|j|}|j|}|SrZ)rrrrs r]rz Stem.forwards. JJqM JJqM JJqMr_)rnrvFrQrrNNr)r`rarbrcrdrhrfrgrrrrrrrs@r]rrs, !#+"#t#t#t #t  #t  #t#t#t#tJ>3>>%,,r_rc|j|jsJ|S|d|jz|d|jzf}t|||}|S)z>Configure window size based on image size and partition ratio.rr-)rLrM)rLrMrKr)rrXrps r]cfg_window_sizer&sX "}}} a[C$7$77!H[H[9[[N #>^ LC Jr_kwargsc Vi}i}i}|jD]X\}}|jdr|||jdd<-|jdr|||jdd<T|||<Zt|ft|jfi|t|jfi|d|}|S)z-Overlay keyword arguments onto configuration. transformer_rvconv_)rr)items startswithrrr)rrtransformer_kwargs conv_kwargs base_kwargsrrs r]_overlay_kwargsr0sKK 1 << '@A qyy< = \\' "23K '2. /KN     3 3J7IJ55    C Jr_czeZdZdZ d'dedeeeeeffdededede d e d e ffd Z d(d e jd ededdfdZej j"deefdZej j"d)dedeee ffdZej j"d*deddfdZej j"de jfdZd+dedeeddfdZ d,dej6deeeeefdededededeeej6eej6eej6fffdZ d-deeeefded edeed!ffd"Zdej6dej6fd#Zd)dej6d$edej6fd%Z dej6dej6fd&Z!xZ"S).r8z{CoaTNet + MaxVit base model. Highly configurable for different block compositions, tensor layouts, pooling types. NrrXin_chans num_classes global_pool drop_ratedrop_path_raterc t||| d} t|}| r t|fi| }t |j |} ||_||_||_|jdx|_ |_ ||_ d|_ g|_ td||j|j j"|j$|j j&|j j(|j j*d| |_|j,j.} |xjt1|j,j2ddgz c_ t5t7|t| Dcgc] \}}||z c}}}t9|j}t9|j:|k(sJt=||j:d }|j,j2}g}t?|D]}d}|j|}t5|Dcgc] }|d z |zd zc}}|tA||f|j:||jB||j | |||d | gz }| |z} |}|xjt1|| d |gz c_ tEjF||_$tKtM|j j(|j j* }|jNrVtEjP|_)|jN|_'tU|j|f|jN|||d| |_+nH|j|_'||jfi| |_)tY|j|f||d| |_+|jZdvsJ|jZr,t]tK|j^|jZ|yycc}}wcc}w)a Args: cfg: Model configuration. img_size: Input image size. in_chans: Number of input channels. num_classes: Number of classification classes. global_pool: Global pooling type. drop_rate: Dropout rate. drop_path_rate: Drop path rate. **kwargs: Additional keyword arguments to overlay on config. rrF)r(r)rwrGrRrTrXrstem)num_chs reductionrT) stagewiser-)rrrrrrzstages.r) hidden_sizerFrrT)rFr)rvrrrrrNri)0rrr$rrrrrrr num_featuresrr feature_inforrrrwrrRrTrXrrrr)rziplenrrrangerrlrr stagesrrrrrrheadrrr1 _init_weights)r\rrXrrrrrrrrrrrrsr num_stagesdprr(r stage_strider)rfinal_norm_layerrs r]rzMaxxVit.__init__Ks0  /X& !#00C)#*=*=xH&  &-0]]2->>DN""'  NNLL((ll,,||..\\**     !! d499+<+<RXYZZc(If#2E2E2P2P#QWZWjWjWsWst    DI$'$8$8D !-!!!11%#+ DI%)$5$5D !():):AbADI&!!&#   DI"\\\\ ??  2 23??KT R mPOs .O O rrrrct|dr |j|yy#t$r|jYywxYw)Nrr)hasattrr TypeError)r\rrrs r]rzMaxxVit._init_weightssD 6> * &##6#2 + &##% &s ">>c|jDchc]\}tfddDrc}}Scc}}w)Nc3&K|]}|v ywrZri).0nrs r] z*MaxxVit.no_weight_decay..sSa16Ss)relative_position_bias_tablez rel_pos.mlp)named_parametersany)r\r_s ` r]no_weight_decayzMaxxVit.no_weight_decaysK//1UU!QS#RSS U UUs :coarsec$tdddg}|S)Nz^stem)z^stages\.(\d+)N)z^norm)i)rr)r)r\rmatchers r] group_matcherzMaxxVit.group_matchers-/CD r_enablec4|jD] }||_ yrZ)rr)r\rrs r]set_grad_checkpointingzMaxxVit.set_grad_checkpointings *A#)A  *r_c.|jjSrZ)rfcr[s r]get_classifierzMaxxVit.get_classifiersyy||r_cJ||_|jj||yrZ)rrreset)r\rrs r]reset_classifierzMaxxVit.reset_classifiers&  [1r_rindicesr stop_early output_fmtintermediates_onlyc|dvsJdg}tt|jdz|\}} d} |j|}| |vr|j |t|j} t j js|s |j} n|jd| } | D]@} | dz } | |}| |vs|r| | k(r|j|}n|}|j |B|r|S| | k(r|j|}||fS)a Forward features that returns intermediates. Args: x: Input image tensor indices: Take last n blocks if int, all if None, select matching indices if sequence norm: Apply norm layer to compatible intermediates stop_early: Stop iterating over blocks when last desired intermediate hit output_fmt: Shape of intermediate feature outputs intermediates_only: Only return intermediate features Returns: )NCHWzOutput shape must be NCHW.r-rN) r/rrrappendrrrr)r\rrrrrr intermediates take_indices max_indexfeat_idxlast_idxrstagex_inters r]forward_intermediateszMaxxVit.forward_intermediatess*Y&D(DD& "6s4;;7G!7KW"U i IIaL | #   #t{{# 99 ! ! #:[[F[[),F .E MHaA<'H0"iilGG$$W- .  x  ! A-r_ prune_norm prune_head.ctt|jdz|\}}|jd||_|rtj|_|r|j dd|_|S)z6Prune layers not required for specified intermediates.r-Nrrv)r/rrrrrrr)r\rrrr r s r]prune_intermediate_layersz!MaxxVit.prune_intermediate_layerssb#7s4;;7G!7KW"U ikk*9-  DI --a4DIr_cl|j|}|j|}|j|}|SrZ)rrrrs r]forward_featureszMaxxVit.forward_featuress. IIaL KKN IIaLr_ pre_logitscN|r|j||S|j|S)N)r)r)r\rrs r] forward_headzMaxxVit.forward_head#s%6@tyyzy2RdiiPQlRr_cJ|j|}|j|}|SrZ)rrrs r]rzMaxxVit.forward&s'  ! !! $   a r_)rnrrBrBNNrF)TrZ)NFFrF)r-FT)#r`rarbrcr5rrdrrhrgrrrModulerrrignorer rrfr rrrr rrr rrrrrrrs@r]r8r8Es58#$!$&iSiSCsCx01iS iS  iS  iSiS"iSiSV&BII&S&#&t& YYUSUU  YYDT#s(^ YY*T*T** YY 2C2hsm2W[28<$$',4 ||4 eCcN344  4  4  4 !%4  tELL!5tELL7I)I#JJ K4 p./$# 3S >*    sCx %,,5<< SellSSS%,,r_r8rFTrGrIrtrFconv_output_biasconv_attn_earlyconv_attn_act_layerconv_norm_layertransformer_shortcut_biastransformer_norm_layertransformer_norm_layer_clrPrHrJc btt||dd|||d| td||| ||| | S)asRW variant configuration for CoAtNet models. These models were created and trained before seeing https://github.com/google-research/maxvit Common differences for initial timm models: - pre-norm layer in MZBConv included an activation after norm - mbconv expansion calculated from input instead of output chs - mbconv shortcut and final 1x1 conv did not have a bias - SE act layer was relu, not silu - mbconv uses silu in timm, not gelu - expansion in attention block done via output proj, not input proj Variable differences (evolved over training initial models): - avg pool with kernel_size=2 favoured downsampling (instead of maxpool for coat) - SE attention was between conv2 and norm/act - default to avg pool for mbconv downsample instead of 1x1 or dw conv - transformer block shortcut has no bias TFr{) rtrFrqrmrrrxr|rRrT)r?r@rFrPrTrVrHrJrrrr6r7) rtrFr r!r"r#r$r%r&rPrHrJs r] _rw_coat_cfgr*,sW@ #(&.&  .3#-3%#   r_rsconv_attn_ratior;c `tt||d||d|td|||| ||| |  S)aRW variant configuration for MaxViT models. These models were created and trained before seeing https://github.com/google-research/maxvit Differences of initial timm models: - mbconv expansion calculated from input instead of output chs - mbconv shortcut and final 1x1 conv did not have a bias - mbconv uses silu in timm, not gelu - expansion in attention block done via output proj, not input proj Fr{)rtrFrmrrr~rRrT) r?rFr;rLrPrTrVrHrJr(r)) rtrFr r+r#r%r&rLr;rPrHrJs r] _rw_max_cfgr-esS0 #(&& .##-3%#   r_rWrconv_norm_layer_clrNc t|}ttd||d|d||td||||d||| |  S)z=Configuration for experimental ConvNeXt-based MaxxViT models.rFr)rlrtrFrmrPrTrVr-) r?rFrLrNrPrTrVrHrJr()r$rr6r7) rtrFr#r.r%r&rLrNrPrHrJs r] _next_cfgr0seK(K !##A&, .#'#A-3%#   r_c Nttdddtdddd S) z0Configuration matching TensorFlow MaxViT models.gMbP? gelu_tanhsame)rXrRrwrFrg)rXrRr<rHr(r)rir_r]_tf_cfgr4s6 ! .!"   r_coatnet_pico_rw)rrIr)r:r)rrrr})r r+coatnet_nano_rw)rnrTrn)rtr r+ coatnet_0_rwr)rrnrCr)r!r$ coatnet_1_rw)rr9rr)rtr!r$ coatnet_2_rw)r6r7rI)rr6r{)rtr" coatnet_3_rw)rrr)rr)rtr"rPcoatnet_bn_0_rwr)rtr!r$r%coatnet_rmlp_nano_rwr)r r+rHrJcoatnet_rmlp_0_rw)rtrHcoatnet_rmlp_1_rwr)rFr!r$rHrJcoatnet_rmlp_1_rw2)rtrHrJcoatnet_rmlp_2_rw)rtr"rPrHcoatnet_rmlp_3_rwcoatnet_nano_cc)rrrrrH)rrrrlcoatnext_nano_rwr)rrrr)rN)rHrP coatnet_0rr)rrrr coatnet_1 coatnet_2r6r= coatnet_3rr? coatnet_4)r r coatnet_5)r7rIrSmaxvit_pico_rw)r:rr6r7)rrrr)rrrr)r:)rrrlrmaxvit_nano_rw)r-rrnr-maxvit_tiny_rwmaxvit_tiny_pm)rrrrmaxvit_rmlp_pico_rw)rHmaxvit_rmlp_nano_rwmaxvit_rmlp_tiny_rwmaxvit_rmlp_small_rwmaxvit_rmlp_base_rw)rrrlrrmaxxvit_rmlp_nano_rw)rrrlrrmaxxvit_rmlp_tiny_rwmaxxvit_rmlp_small_rw)0rmaxxvitv2_nano_rw)rNrHmaxxvitv2_rmlp_base_rw)rr9rOr)rNmaxxvitv2_rmlp_large_rw)i@irR)rr9r)PrerRmaxvit_tiny_tf)rrrlrrrmaxvit_small_tfmaxvit_base_tfmaxvit_large_tfmaxvit_xlarge_tf state_dictmodelcr|j}i}|jD]\}}|jdr|j|dd}|j|j jk7s|j d|j dk7r,t||j |j j}||vrn|j||jk7rR|j||jk(r.|jdvsJ|j||j}|||<|S)z/Filter checkpoint state dict for compatibility.rNirr-)new_window_sizenew_bias_shape)rrT) rmrendswith get_submodulerrrLr,ndimnumelr)rmrnmodel_state_dictout_dictrrms r]checkpoint_filter_fnry>s)'')H  "1 ::4 5##AdsG,Aww!88>>>!--PQBRVWVcVcdeVfBf-$%MM#$#A#A#G#G  QVV/?/B/G/G%GAGGIYijkYlYrYrYtLt66V# ## *1-334A  Or_variant cfg_variant pretrainedc |.|tvr|}n#dj|jddd}tt||ft|t dt d|S)zCreate a MaxxVit model variant.NrrT)flatten_sequential) model_cfg feature_cfgpretrained_filter_fn) model_cfgsjoinsplitr.r8rry)rzr{r|rs r]_create_maxxvitrTsi j !K((7==#5cr#:;K * [)D11     r_urlc $|dddddddddd d d |S) z$Create a default configuration dict.r)rnrr)rCrCffffff?bicubic)?rrz stem.conv1zhead.fcTz apache-2.0) rr input_size pool_sizecrop_pct interpolationmeanr first_conv classifierfixed_input_sizelicenseri)rrs r]_cfgrcs74}SY9")    $*  r_zcoatnet_pico_rw_224.untrained)rzcoatnet_nano_rw_224.sw_in1kztimm/zyhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_nano_rw_224_sw-f53093b4.pthg?) hf_hub_idrrzcoatnet_0_rw_224.sw_in1kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_0_rw_224_sw-a6439706.pth)rrzcoatnet_1_rw_224.sw_in1kzvhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_1_rw_224_sw-5cae1ea8.pthz!coatnet_2_rw_224.sw_in12k_ft_in1k)rz'coatnet_rmlp_1_rw2_224.sw_in12k_ft_in1kz&coatnet_rmlp_2_rw_224.sw_in12k_ft_in1kz&coatnet_rmlp_2_rw_384.sw_in12k_ft_in1k)rnrr)rOrOg?squash)rrrr crop_modezcoatnet_bn_0_rw_224.sw_in1kzyhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_bn_0_rw_224_sw-c228e218.pthr)rrrrrz coatnet_rmlp_nano_rw_224.sw_in1kz~https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_rmlp_nano_rw_224_sw-bd1d51b3.pthzcoatnet_rmlp_0_rw_224.untrainedzcoatnet_rmlp_1_rw_224.sw_in1kz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_rmlp_1_rw_224_sw-9051e6c3.pthzcoatnet_rmlp_2_rw_224.sw_in1kz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnet_rmlp_2_rw_224_sw-5ccfac55.pthzcoatnet_rmlp_3_rw_224.untrainedzcoatnet_nano_cc_224.untrainedzcoatnext_nano_rw_224.sw_in1kzzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/coatnext_nano_rw_224_ad-22cb71c2.pthzcoatnet_2_rw_224.sw_in12ki-.)rrzcoatnet_3_rw_224.sw_in12kzcoatnet_rmlp_1_rw2_224.sw_in12kzcoatnet_rmlp_2_rw_224.sw_in12kzcoatnet_0_224.untrainedzcoatnet_1_224.untrainedzcoatnet_2_224.untrainedzcoatnet_3_224.untrainedzcoatnet_4_224.untrainedzcoatnet_5_224.untrainedzmaxvit_pico_rw_256.untrained)rnr7r7)r)rrrzmaxvit_nano_rw_256.sw_in1kzxhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_nano_rw_256_sw-fb127241.pth)rrrrzmaxvit_tiny_rw_224.sw_in1kzxhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_tiny_rw_224_sw-7d0dffeb.pthzmaxvit_tiny_rw_256.untrainedzmaxvit_tiny_pm_256.untrainedzmaxvit_rmlp_pico_rw_256.sw_in1kz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_pico_rw_256_sw-8d82f2c6.pthzmaxvit_rmlp_nano_rw_256.sw_in1kz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_nano_rw_256_sw-c17bb0d6.pthzmaxvit_rmlp_tiny_rw_256.sw_in1kz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_tiny_rw_256_sw-bbef0ff5.pthz maxvit_rmlp_small_rw_224.sw_in1kz~https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxvit_rmlp_small_rw_224_sw-6ef0ae4f.pthz"maxvit_rmlp_small_rw_256.untrainedz(maxvit_rmlp_base_rw_224.sw_in12k_ft_in1kz(maxvit_rmlp_base_rw_384.sw_in12k_ft_in1kz maxvit_rmlp_base_rw_224.sw_in12kz maxxvit_rmlp_nano_rw_256.sw_in1kz~https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxxvit_rmlp_nano_rw_256_sw-0325d459.pthz"maxxvit_rmlp_tiny_rw_256.untrainedz!maxxvit_rmlp_small_rw_256.sw_in1kzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-maxx/maxxvit_rmlp_small_rw_256_sw-37e217ff.pthzmaxxvitv2_nano_rw_256.sw_in1k)rrrz+maxxvitv2_rmlp_base_rw_224.sw_in12k_ft_in1kz+maxxvitv2_rmlp_base_rw_384.sw_in12k_ft_in1kz%maxxvitv2_rmlp_large_rw_224.untrainedz#maxxvitv2_rmlp_base_rw_224.sw_in12kzmaxvit_tiny_tf_224.in1k)rrrzmaxvit_tiny_tf_384.in1kzmaxvit_tiny_tf_512.in1k)rnrIrI)rfrfzmaxvit_small_tf_224.in1kzmaxvit_small_tf_384.in1kzmaxvit_small_tf_512.in1kzmaxvit_base_tf_224.in1kzmaxvit_base_tf_384.in1kzmaxvit_base_tf_512.in1kzmaxvit_large_tf_224.in1kzmaxvit_large_tf_384.in1kzmaxvit_large_tf_512.in1kzmaxvit_base_tf_224.in21kiSUz maxvit_base_tf_384.in21k_ft_in1kz maxvit_base_tf_512.in21k_ft_in1kzmaxvit_large_tf_224.in21kz!maxvit_large_tf_384.in21k_ft_in1kz!maxvit_large_tf_512.in21k_ft_in1k)rrrrzmaxvit_xlarge_tf_224.in21kz"maxvit_xlarge_tf_384.in21k_ft_in1kz"maxvit_xlarge_tf_512.in21k_ft_in1kc tdd|i|S)z)CoatNet Pico model with RW configuration.r|)coatnet_pico_rw_224rr|rs r]rrJ   RZ R6 RRr_c tdd|i|S)z)CoatNet Nano model with RW configuration.r|)coatnet_nano_rw_224rrs r]rrP rr_c tdd|i|S)z&CoatNet-0 model with RW configuration.r|)coatnet_0_rw_224rrs r]rrV   O* O OOr_c tdd|i|S)z&CoatNet-1 model with RW configuration.r|)coatnet_1_rw_224rrs r]rr\ rr_c tdd|i|S)z&CoatNet-2 model with RW configuration.r|)coatnet_2_rw_224rrs r]rrb rr_c tdd|i|S)z&CoatNet-3 model with RW configuration.r|)coatnet_3_rw_224rrs r]rrh rr_c tdd|i|S)z4CoatNet-0 model with BatchNorm and RW configuration.r|)coatnet_bn_0_rw_224rrs r]rrn rr_c tdd|i|S)z.CoatNet Nano model with Relative Position MLP.r|)coatnet_rmlp_nano_rw_224rrs r]rrt   W* WPV WWr_c tdd|i|S)z+CoatNet-0 model with Relative Position MLP.r|)coatnet_rmlp_0_rw_224rrs r]rrz   Tz TV TTr_c tdd|i|S)z+CoatNet-1 model with Relative Position MLP.r|)coatnet_rmlp_1_rw_224rrs r]rr rr_c tdd|i|S)z.CoatNet-1 model with Relative Position MLP v2.r|)coatnet_rmlp_1_rw2_224rrs r]rr s  U Uf UUr_c tdd|i|S)z+CoatNet-2 model with Relative Position MLP.r|)coatnet_rmlp_2_rw_224rrs r]rr rr_c tdd|i|S)z6CoatNet-2 model with Relative Position MLP at 384x384.r|)coatnet_rmlp_2_rw_384rrs r]rr rr_c tdd|i|S)z+CoatNet-3 model with Relative Position MLP.r|)coatnet_rmlp_3_rw_224rrs r]rr rr_c tdd|i|S)z(CoatNet Nano model with ConvNeXt blocks.r|)coatnet_nano_cc_224rrs r]rr rr_c tdd|i|S)z*CoAtNeXt Nano model with RW configuration.r|)coatnext_nano_rw_224rrs r]rr s  Sj SF SSr_c tdd|i|S)zCoatNet-0 model.r|) coatnet_0_224rrs r]rr   Lz LV LLr_c tdd|i|S)zCoatNet-1 model.r|) coatnet_1_224rrs r]rr rr_c tdd|i|S)zCoatNet-2 model.r|) coatnet_2_224rrs r]rr rr_c tdd|i|S)zCoatNet-3 model.r|) coatnet_3_224rrs r]rr rr_c tdd|i|S)zCoatNet-4 model.r|) coatnet_4_224rrs r]rr rr_c tdd|i|S)zCoatNet-5 model.r|) coatnet_5_224rrs r]rr rr_c tdd|i|S)z(MaxViT Pico model with RW configuration.r|)maxvit_pico_rw_256rrs r]rr   QJ Q& QQr_c tdd|i|S)z(MaxViT Nano model with RW configuration.r|)maxvit_nano_rw_256rrs r]rr rr_c tdd|i|S)z(MaxViT Tiny model with RW configuration.r|)maxvit_tiny_rw_224rrs r]rr rr_c tdd|i|S)z3MaxViT Tiny model with RW configuration at 256x256.r|)maxvit_tiny_rw_256rrs r]rr rr_c tdd|i|S)z3MaxViT Relative Position MLP Pico RW 256x256 model.r|)maxvit_rmlp_pico_rw_256rrs r]rr   V Vv VVr_c tdd|i|S)z3MaxViT Relative Position MLP Nano RW 256x256 model.r|)maxvit_rmlp_nano_rw_256rrs r]rr rr_c tdd|i|S)z3MaxViT Relative Position MLP Tiny RW 256x256 model.r|)maxvit_rmlp_tiny_rw_256rrs r]rr rr_c tdd|i|S)z4MaxViT Relative Position MLP Small RW 224x224 model.r|)maxvit_rmlp_small_rw_224rrs r]rr rr_c tdd|i|S)z9MaxViT Small model with Relative Position MLP at 256x256.r|)maxvit_rmlp_small_rw_256rrs r]rr rr_c tdd|i|S)z-MaxViT Base model with Relative Position MLP.r|)maxvit_rmlp_base_rw_224rrs r]rr rr_c tdd|i|S)z8MaxViT Base model with Relative Position MLP at 384x384.r|)maxvit_rmlp_base_rw_384rrs r]rr rr_c tdd|i|S)z'MaxViT Tiny model with parallel blocks.r|)maxvit_tiny_pm_256rrs r]rr rr_c tdd|i|S)z4MaxxViT Relative Position MLP Nano RW 256x256 model.r|)maxxvit_rmlp_nano_rw_256rrs r]rr rr_c tdd|i|S)z.MaxxViT Tiny model with Relative Position MLP.r|)maxxvit_rmlp_tiny_rw_256rrs r]rr rr_c tdd|i|S)z/MaxxViT Small model with Relative Position MLP.r|)maxxvit_rmlp_small_rw_256rrs r]rr" s  X: XQW XXr_c tdd|i|S)zMaxxViT-V2 Nano model.r|)maxxvitv2_nano_rw_256rrs r]rr( rr_c tdd|i|S)z1MaxxViT-V2 Base model with Relative Position MLP.r|)maxxvitv2_rmlp_base_rw_224rrs r]rr.   YJ YRX YYr_c tdd|i|S)zr%s "H #11III A6++34< N 22 2B !P!P !PH  ! !  !Q"))QhR"))Rj/299/d0bii0s0C002NNb,ryy,,S,$,0&8C=&C&C&l"))l^dBIIdN 49 ELL tCy DQTI Z_ZfZf  elltCyU\\ %,, 49 S  V[VbVb   . U38_ QYZbQc D299DNTTnU\\S ell  DI QUVYQZ _d_k_k  5<<DI%,, u|| S  TRUY [`[g[g  K299K\52995p12991hW299Wt/299/d.%S/Ncsz*dbiidP"!& %#)!*.&3)4'+"6666 6 ! 6  6$(6!$6$'6e_666 #s(^6t !&!'!&3)415'+"---- -  - !$ -$'-eCHo.--e_--- #s(^-b ,"-&3)415#9=!$$$$ $ !$ $ $' $eCHo.$$5%u "556$$$ #s(^$Nc3h"n% !  n %  !  n*%  &+ +n< %   &+ =nP '   & Qnd '   & en| %   &+#0  }nR$ %  !   Snh!%  inz! %   &+  {nR" %   Snf! '   &  gn|! '   &  }nT%5  . Unb %  $ cnz% {nF% GnR' Sn^' _nj' knv( wnF$  - GnT%  - Unb%  - cnp%  - qn@#$  5 ) AnN#%  5 ) On\#%  5 ) ]nj$ %   kn~# %   nT$%  + Und$%  + enr%%  + snB ! %   C nX &'   Y nj ' '   k nB %  )C nT %  )U nf %  )g nx '  )y nJ  '  )K n b T#u||*;%<RYYSWX[]b]i]iXiSj, S x} QU il qx  c # $sCx. %X&#Tb\X&"4 H$X& E!FX& E!X& (*!X&*.t0+X&.-d//X&2-d Hsh/X3X&<"4 H "(< $=X&F' M)GX&N&t|OX&P$T J&KQX&V$T J&KWX&\&t|]X&^$Tb\_X&`#D I%aX&l "mX&r "sX&x&t(yX&~%d'X&Ht|IX&Jt|KX&Lt|MX&Nt|OX&Pt|QX&Rt|SX&X#DRMU[$\YX&Z!$ G F#4[X&b!$ G#HcX&h#D  F%4iX&n#DRMU[$\oX&t&t L F(4uX&|&t L F(4}X&D&t L F(4EX&L' M)MX&V)$  F+4WX&`/1aX&f/ Hsh1XgX&p')qX&|' M F)4}X&D)$2-[a*bEX&F( N F*4GX&R$T F&4SX&X244YX&\24 Hsh4X]X&b,Tb\cX&f*4,gX&pt "(< >qX&vt Hsh XwX&|t Hsh X}X&B "(<!>CX&H Hsh!XIX&N Hsh!XOX&Tt "(< >UX&Zt Hsh X[X&`t Hsh XaX&f "(<!>gX&l Hsh!XmX&r Hsh!XsX&z!{X&@' Hsh)XAX&F' Hsh)XGX&L "MX&R( Hsh*XSX&X( 3(*DYX&^!$#_X&d)$ Hsh+XeX&j)$ Hsh+XkX&X vSDSCSGSS SDSCSGSS PPPPP PPPPP PPPPP PPPPP SDSCSGSS XXXXX UdUcUgUU UdUcUgUU VtVsVwVV UdUcUgUU UdUcUgUU UdUcUgUU SDSCSGSS TTTSTWTT MdMcMgMM MdMcMgMM MdMcMgMM MdMcMgMM MdMcMgMM MdMcMgMM R4R3R7RR R4R3R7RR R4R3R7RR R4R3R7RR WWWWW WWWWW WWWWW XXXXX XXXXX WWWWW WWWWW R4R3R7RR XXXXX XXXXX Y$Y#Y'YY UdUcUgUU Z4Z3Z7ZZ Z4Z3Z7ZZ [D[C[G[[ d4d3d7dd d4d3d7dd d4d3d7dd fDfCfGff fDfCfGff fDfCfGff d4d3d7dd d4d3d7dd d4d3d7dd fDfCfGff fDfCfGff fDfCfGff hThShWhh hThShWhh hThShWhhr_