
    ^j_                        d Z ddlmZ ddlmZmZmZmZmZ ddl	Z	ddl	m
Z
 ddlmZmZ ddlmZmZmZmZmZmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZ  G d de
j>                        Z  G d de
j>                        Z! G d de
j>                        Z" G d de
j>                        Z# G d de
j>                        Z$ G d de
j>                        Z% G d de
j>                        Z&d Z'd9dZ( e e(d       e(d       e(d       e(d       e(d       e(d       e(ddd !       e(ddd !       e(ddd !       e(d       e(dd"d"d#d$%       e(dd&'       e(d(d)d*+      d,      Z)d:d-Z*ed:d.       Z+ed:d/       Z,ed:d0       Z-ed:d1       Z.ed:d2       Z/ed:d3       Z0ed:d4       Z1ed:d5       Z2ed:d6       Z3ed:d7       Z4ed:d8       Z5y);z
MambaOut models for image classification.
Some implementations are modified from:
timm (https://github.com/rwightman/pytorch-image-models),
MetaFormer (https://github.com/sail-sg/metaformer),
InceptionNeXt (https://github.com/sail-sg/inceptionnext)
    )OrderedDict)ListOptionalTupleTypeUnionN)nnIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)trunc_normal_DropPathcalculate_drop_path_rates	LayerNorm
LayerScaleClNormMlpClassifierHeadget_act_layer   )build_model_with_cfg)feature_take_indices)checkpoint_seq)register_modelgenerate_default_cfgsc                        e Zd ZdZdddej
                  eddfdededed	e	ej                     d
e	ej                     f
 fdZd Z xZS )StemzV Code modified from InternImage:
        https://github.com/OpenGVLab/InternImage
       `   TNin_chsout_chsmid_norm	act_layer
norm_layerc                    ||d}t         	|           t        j                  ||dz  fdddd|| _        |r ||dz  fi |nd | _         |       | _        t        j                  |dz  |fdddd|| _         ||fi || _        y )Ndevicedtype   r   r   kernel_sizestridepadding)	super__init__r	   Conv2dconv1norm1actconv2norm2)
selfr   r   r    r!   r"   r%   r&   dd	__class__s
            _/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/mambaout.pyr-   zStem.__init__   s     /YYqL
 
 

 8@Z133T
;YYqL
 
 

  .2.
    c                 @   | j                  |      }| j                  9|j                  dddd      }| j                  |      }|j                  dddd      }| j                  |      }| j	                  |      }|j                  dddd      }| j                  |      }|S )Nr   r'   r   r   )r/   r0   permuter1   r2   r3   r4   xs     r7   forwardzStem.forward;   s    JJqM::!		!Q1%A

1A		!Q1%AHHQKJJqMIIaAq!JJqMr8   )__name__
__module____qualname____doc__r	   GELUr   intboolr   Moduler-   r=   __classcell__r6   s   @r7   r   r      sn     !)+*3// / 	/
 BII/ RYY/@
r8   r   c                   X     e Zd Zddeddfdededeej                     f fdZd Z	 xZ
S )	DownsampleNormFirstr      Nr   r   r"   c                     ||d}t         |            ||fi || _        t        j                  ||fdddd|| _        y Nr$   r   r'   r   r(   )r,   r-   normr	   r.   convr4   r   r   r"   r%   r&   r5   r6   s          r7   r-   zDownsampleNormFirst.__init__J   s[     /v,,	II
 
 
	r8   c                     | j                  |      }|j                  dddd      }| j                  |      }|j                  dddd      }|S Nr   r   r   r'   )rM   r:   rN   r;   s     r7   r=   zDownsampleNormFirst.forward^   sI    IIaLIIaAq!IIaLIIaAq!r8   r>   r?   r@   r   rC   r   r	   rE   r-   r=   rF   rG   s   @r7   rI   rI   H   sB     *3

 
 RYY	
(r8   rI   c                   X     e Zd Zddeddfdededeej                     f fdZd Z	 xZ
S )	
Downsampler   rJ   Nr   r   r"   c                     ||d}t         |           t        j                  ||fdddd|| _         ||fi || _        y rL   )r,   r-   r	   r.   rN   rM   rO   s          r7   r-   zDownsample.__init__h   s]     /II
 
 
	 w-"-	r8   c                     |j                  dddd      }| j                  |      }|j                  dddd      }| j                  |      }|S rQ   )r:   rN   rM   r;   s     r7   r=   zDownsample.forward|   sI    IIaAq!IIaLIIaAq!IIaLr8   rR   rG   s   @r7   rT   rT   f   sB     *3.. . RYY	.(r8   rT   c                        e Zd ZdZddej
                  deddddf	ded	ed
ede	ej                     dee   de	ej                     dedef fdZdd	ed
ee   defdZddefdZ xZS )MlpHeadz MLP classification head
      avg           TNin_featuresnum_classes	pool_typer!   	mlp_ratior"   	drop_ratebiasc                 D   |	|
d}t         |           |t        ||z        }nd }|| _        || _        |xs || _         ||fi || _        |rUt        j                  t        dt        j                  ||fi |fd |       fd ||fi |fg            | _        || _        n || _        t        j                         | _        |dkD  r#t        j                  | j                  |fd|i|nt        j                         | _        t        j                  |      | _        y )Nr$   fcr1   rM   r   rb   )r,   r-   rC   r_   r]   hidden_sizerM   r	   
Sequentialr   Linear
pre_logitsnum_featuresIdentityrd   Dropouthead_dropout)r4   r]   r^   r_   r!   r`   r"   ra   rb   r%   r&   r5   re   r6   s                r7   r-   zMlpHead.__init__   s    / i+56KK"&&5+{1b1	 mmKryyk@R@A	$K62679 - DO
 !,D +D kkmDOP[^_P_"))D--{LLLegepeperJJy1r8   reset_otherc                 (   ||| _         |rCt        j                         | _        t        j                         | _        | j
                  | _        |dkD  r&t        j                  | j                  |      | _        y t        j                         | _        y )Nr   )	r_   r	   rj   rM   rh   r]   ri   rg   rd   )r4   r^   r_   rm   s       r7   resetzMlpHead.reset   sh     &DNDI kkmDO $ 0 0D?JQ"))D--{;TVT_T_Tar8   rh   c                     | j                   dk(  r|j                  d      }| j                  |      }| j                  |      }| j	                  |      }|r|S | j                  |      }|S )NrZ   )r   r'   )r_   meanrM   rh   rl   rd   r4   r<   rh   s      r7   r=   zMlpHead.forward   s`    >>U"vAIIaLOOAa HGGAJr8   )NFF)r>   r?   r@   rA   r	   rB   r   rC   strr   rE   r   floatrD   r-   ro   r=   rF   rG   s   @r7   rX   rX      s      $")+'(*3!$2$2 $2 	$2
 BII$2  }$2 RYY$2 $2 $2Lb b# bTX b	T 	r8   rX   c                        e Zd ZdZddddeej                  dddf	deded	ed
ede	e   de
ej                     de
ej                     def fdZd Z xZS )GatedConvBlocka   Our implementation of Gated CNN Block: https://arxiv.org/pdf/1612.08083
    Args:
        conv_ratio: control the number of channels to conduct depthwise convolution.
            Conduct convolution on partial channels can improve paraitcal efficiency.
            The idea of partial channels is from ShuffleNet V2 (https://arxiv.org/abs/1807.11164) and
            also used by InceptionNeXt (https://arxiv.org/abs/2303.16900) and FasterNet (https://arxiv.org/abs/2303.03667)
    UUUUUU@         ?Nr\   dimexpansion_ratior)   
conv_ratiols_init_valuer"   r!   	drop_pathc                    |	|
d}t         |            ||fi || _        t        ||z        }t	        j
                  ||dz  fi || _         |       | _        t        ||z        }|||z
  |f| _        t	        j                  ||f||dz  |d|| _
        t	        j
                  ||fi || _        |t        |fi |nt	        j                         | _        |dkD  rt        |      | _        y t	        j                         | _        y )Nr$   r'   )r)   r+   groupsr\   )r,   r-   rM   rC   r	   rg   fc1r1   split_indicesr.   rN   fc2r   rj   lsr   r   )r4   r{   r|   r)   r}   r~   r"   r!   r   r%   r&   kwargsr5   hiddenconv_channelsr6   s                  r7   r-   zGatedConvBlock.__init__   s    /s)b)	_s*+99S&1*33;J,-$f}&<mLII
 $1$ 
 
	 99VS/B/+8+D*S'B'"++-09B),BKKMr8   c                    |}| j                  |      }| j                  |      }t        j                  || j                  d      \  }}}|j                  dddd      }| j                  |      }|j                  dddd      }| j                  | j                  |      t        j                  ||fd      z        }| j                  |      }| j                  |      }||z   S )N)r{   r   r   r   r'   )rM   r   torchsplitr   r:   rN   r   r1   catr   r   )r4   r<   shortcutgics         r7   r=   zGatedConvBlock.forward   s    IIaLHHQK++a!3!3<1aIIaAq!IIaLIIaAq!HHTXXa[599aV#<<=GGAJNN18|r8   )r>   r?   r@   rA   r   r	   rB   rC   ru   r   r   rE   r-   r=   rF   rG   s   @r7   rw   rw      s     &+  #-1*3)+! R R # R 	 R
  R $E? R RYY R BII R  RDr8   rw   c                        e Zd Zdddddddeej
                  dddfded	ee   d
edededede	dee   de
ej                     de
ej                     def fdZd Z xZS )MambaOutStageNr[   rx   ry   rz    r\   r{   dim_outdepthr|   r)   r}   
downsampler~   r"   r!   r   c                    ||d}t         |           |xs |}d| _        |dk(  rt        ||fd|	i|| _        n:|dk(  rt        ||fd|	i|| _        n ||k(  sJ t        j                         | _        t        j                  t        |      D cg c]1  }t        d||||||	|
t        |t        t        f      r||   n|d|3 c} | _        y c c}w )Nr$   FrN   r"   conv_nf)r{   r|   r)   r}   r~   r"   r!   r    )r,   r-   grad_checkpointingrT   r   rI   r	   rj   rf   rangerw   
isinstancelisttupleblocks)r4   r{   r   r   r|   r)   r}   r   r~   r"   r!   r   r%   r&   r5   jr6   s                   r7   r-   zMambaOutStage.__init__   s      /.S"'(gS*SPRSDO9$1#w\:\Y[\DO'>!> kkmDOmm 5\&
   
 /'%+%#*4Yu*N)A,T]
 
&
  &
s   6Cc                     | j                  |      }| j                  r6t        j                  j	                         st        | j                  |      }|S | j                  |      }|S N)r   r   r   jitis_scriptingr   r   r;   s     r7   r=   zMambaOutStage.forward*  sS    OOA""599+A+A+Ct{{A.A  AAr8   )r>   r?   r@   r   r	   rB   rC   r   ru   rt   r   rE   r-   r=   rF   rG   s   @r7   r   r      s    
 &*%*  # -1*3)+!** c]* 	*
 #* * * * $E?* RYY* BII* *Xr8   r   c            !           e Zd ZdZdddddeej                  ddd	d
dddddddfdededede	edf   de	edf   de
ej                     de
ej                     dededededee   dedededef  fd Zd! Zej$                  j&                  d5d"       Zej$                  j&                  d6d#       Zej$                  j&                  d$ej                  fd%       Zd7dedee   fd&Z	 	 	 	 	 d8d'ej0                  d(eeeee   f      d)ed*ed+ed,ed$eeej0                     e	ej0                  eej0                     f   f   fd-Z	 	 	 d9d(eeee   f   d.ed/efd0Zd1 Zd5d2efd3Zd4 Z xZ S ):MambaOuta<   MetaFormer
        A PyTorch impl of : `MetaFormer Baselines for Vision`  -
          https://arxiv.org/abs/2210.13452

    Args:
        in_chans (int): Number of input image channels. Default: 3.
        num_classes (int): Number of classes for classification head. Default: 1000.
        depths (list or tuple): Number of blocks at each stage. Default: [3, 3, 9, 3].
        dims (int): Feature dimension at each stage. Default: [96, 192, 384, 576].
        downsample_layers: (list or tuple): Downsampling layers before each stage.
        drop_path_rate (float): Stochastic depth rate. Default: 0.
        output_norm: norm before classifier head. Default: partial(nn.LayerNorm, eps=1e-6).
        head_fn: classification head. Default: nn.Linear.
        head_dropout (float): dropout for MLP classifier. Default: 0.
    r   rY   rZ   r   r   	   r   r        i@  rz   rx   ry   TNrN   r\   defaultin_chansr^   global_pooldepths.dimsr"   r!   r}   r|   r)   stem_mid_normr~   r   drop_path_ratera   head_fnc                    t         |           ||d}|| _        || _        || _        d| _        t        |t        t        f      s|g}t        |t        t        f      s|g}t        |      }t        |      }|| _        g | _        t        ||d   f|||d|| _        |d   }t        ||d      }d}d}t!        j"                         | _        t'        |      D ]  }||   }|dk(  s|dkD  rdnd	}||z  }t)        d||||   |
||	|dkD  r|nd
|||||   d|}| j$                  j+                  |       |}| xj                  t-        ||d|       gz  c_        |||   z  } |dk(  rt/        ||f|||d|| _        n#t3        ||ft5        |dz        |||d|| _        || _        | j0                  j6                  | _        | j;                  | j<                         y )Nr$   NHWCr   )r    r!   r"   T)	stagewiser[   r'   r   r   )r{   r   r   r)   r}   r|   r   r~   r"   r!   r   zstages.)num_chs	reductionmoduler   )r_   ra   r"   )re   r_   r"   ra   r   )r,   r-   r^   r   ra   
output_fmtr   r   r   r   len	num_stagefeature_infor   stemr   r	   rf   stagesr   r   appenddictrX   headr   rC   ri   head_hidden_sizeapply_init_weights)r4   r   r^   r   r   r   r"   r!   r}   r|   r)   r   r~   r   r   ra   r   r%   r&   r5   r   prev_dimdp_ratescurcurr_strider   r{   r*   stager6   s                                r7   r-   zMambaOut.__init__D  sT   * 	/& " &4-0XF$u.6D!),	K	"G
 #!
 
	 7,^VtTmmoy! 	Aq'C%*a!eQF6!K! Qi'% /)*Q:B+%#"1+ E KKu%H$x;Y`ab`cWd"e!ff6!9C-	0 i &#% DI 0  1-%%# DI % $		 6 6

4%%&r8   c                     t        |t        j                  t        j                  f      rOt	        |j
                  d       |j                  +t        j                  j                  |j                  d       y y y )Ng{Gz?)stdr   )	r   r	   r.   rg   r   weightrb   init	constant_)r4   ms     r7   r   zMambaOut._init_weights  sS    a"))RYY/0!((,vv!!!!&&!, " 1r8   c                 2    t        d|rd      S ddg      S )Nz^stemz^stages\.(\d+))z^stages\.(\d+)\.downsample)r   )z^stages\.(\d+)\.blocks\.(\d+)N)r   r   )r   )r4   coarses     r7   group_matcherzMambaOut.group_matcher  s/    (.$
 	
 685
 	
r8   c                 4    | j                   D ]	  }||_         y r   )r   r   )r4   enabless      r7   set_grad_checkpointingzMambaOut.set_grad_checkpointing  s     	*A#)A 	*r8   returnc                 .    | j                   j                  S r   )r   rd   )r4   s    r7   get_classifierzMambaOut.get_classifier  s    yy||r8   c                 J    || _         | j                  j                  ||       y r   )r^   r   ro   )r4   r^   r   s      r7   reset_classifierzMambaOut.reset_classifier  s    &		[1r8   r<   indicesrM   
stop_earlyr   intermediates_onlyc           	         |dv sJ d       |dk(  }g }t        t        | j                        |      \  }	}
| j                  |      }t        j
                  j                         s|s| j                  }n| j                  d|
dz    }t        |      D ]#  \  }} ||      }||	v s|j                  |       % |r/|D cg c]$  }|j                  dddd      j                         & }}|r|S ||fS c c}w )	a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWr   z*Output format must be one of NCHW or NHWC.r   Nr   r   r   r'   )r   r   r   r   r   r   r   	enumerater   r:   
contiguous)r4   r<   r   rM   r   r   r   channel_firstintermediatestake_indices	max_indexr   feat_idxr   ys                  r7   forward_intermediateszMambaOut.forward_intermediates  s   * --[/[[-"f,"6s4;;7G"Qi IIaL99!!#:[[F[[)a-0F(0 	(OHeaA<'$$Q'	(
 IVWAQYYq!Q2==?WMW  - Xs   <)C/
prune_norm
prune_headc                     t        t        | j                        |      \  }}| j                  d|dz    | _        |r| j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r   )r   r   r   r   )r4   r   r   r   r   r   s         r7   prune_intermediate_layersz"MambaOut.prune_intermediate_layers  sM     #7s4;;7G"Qikk.9q=1!!!R(r8   c                 J    | j                  |      }| j                  |      }|S r   )r   r   r;   s     r7   forward_featureszMambaOut.forward_features  s!    IIaLKKNr8   rh   c                 V    |r| j                  ||      }|S | j                  |      }|S )N)rh   )r   rr   s      r7   forward_headzMambaOut.forward_head  s1    3=DIIaJI/ DH99Q<r8   c                 J    | j                  |      }| j                  |      }|S r   )r   r   r;   s     r7   r=   zMambaOut.forward  s'    !!!$a r8   rs   )Tr   )NFFr   F)r   FT)!r>   r?   r@   rA   r   r	   rB   rC   rt   r   r   rE   ru   rD   r   r-   r   r   r   ignorer   r   r   r   Tensorr   r   r   r   r   r   r=   rF   rG   s   @r7   r   r   3  s   $ #$&2$7*3)+ #%( "&-1$$&!$'b'b' b' 	b'
 #s(Ob' S/b' RYYb' BIIb' b' #b' b'  b' $E?b' b' "b'  !b'" #b'H- YY
 
 YY* * YY		  2C 2hsm 2 8<$$',- ||-  eCcN34-  	- 
 -  -  !%-  
tELL!5tELL7I)I#JJ	K- b ./$#	3S	>*  	
$ r8   r   c                    d| v r| d   } d| v r| S dd l }i }| j                         D ]  \  }}|j                  dd      }|j                  dd|      }|j                  dd	|      }|j	                  d
      r|j                  d
d      }nG|j	                  d      r6|j                  dd      }|j                  dd      }|j                  dd      }|||<    |S )Nmodelzstem.conv1.weightr   zdownsample_layers.0.zstem.zstages.([0-9]+).([0-9]+)zstages.\1.blocks.\2zdownsample_layers.([0-9]+)zstages.\1.downsampleznorm.z
head.norm.zhead.z	head.fc1.zhead.pre_logits.fc.zhead.pre_logits.norm.z	head.fc2.zhead.fc.)reitemsreplacesub
startswith)
state_dictr   r   out_dictkvs         r7   checkpoint_filter_fnr    s    *(
j(H  " 1II,g6FF.0FJFF02I1M<< 		'<0A\\'"		+'<=A		,(?@A		+z2A Or8   c                 4    | ddddddt         t        ddd	d
|S )NrY   )r      r  )r      r  )ry   ry   rz   bicubicz
stem.conv1zhead.fcz
apache-2.0)urlr^   
input_sizetest_input_size	pool_sizecrop_pctinterpolationrq   r   
first_conv
classifierlicenser
   )r  r   s     r7   _cfgr  *  s5    =]y%.B")  r8   ztimm/)	hf_hub_idgffffff?rz   )r  r  test_crop_pct)r   r   r   squash)   r  )r  r  r	  	crop_moder
  i-.  )r  r^   )r      r  )r   r   r   )   r  )r  r	  r
  )zmambaout_femto.in1kzmambaout_kobe.in1kzmambaout_tiny.in1kzmambaout_small.in1kzmambaout_base.in1kzmambaout_small_rw.sw_e450_in1kz#mambaout_base_short_rw.sw_e500_in1kz"mambaout_base_tall_rw.sw_e500_in1kz"mambaout_base_wide_rw.sw_e500_in1kz+mambaout_base_plus_rw.sw_e150_in12k_ft_in1kz0mambaout_base_plus_rw.sw_e150_r384_in12k_ft_in1kz#mambaout_base_plus_rw.sw_e150_in12ktest_mambaoutc                 N    t        t        | |ft        t        dd      d|}|S )N)r   r   r'   r   T)out_indicesflatten_sequential)pretrained_filter_fnfeature_cfg)r   r   r  r   )variant
pretrainedr   r   s       r7   _create_mambaoutr   b  s6     ':1\dK 	E Lr8   c           	      J    t        dd      }t        dd| it        |fi |S )Nr   0   r   r   r  r   r   r  )mambaout_femtor   r   r  r   
model_argss      r7   r%  r%  m  s-    \0BCJbbtJGaZ`Gabbr8   c           	      R    t        g dg d      }t        dd| it        |fi |S )N)r   r      r   r"  r$  r  )mambaout_kober&  r'  s      r7   r+  r+  s  s-    ]1CDJa
ad:F`Y_F`aar8   c           	      R    t        g dg d      }t        dd| it        |fi |S )Nr   r   r$  r  )mambaout_tinyr&  r'  s      r7   r-  r-  x  s-    \0CDJa
ad:F`Y_F`aar8   c           	      R    t        g dg d      }t        dd| it        |fi |S )Nr   r[      r   r   r$  r  )mambaout_smallr&  r'  s      r7   r1  r1  ~  s-    ]1DEJbbtJGaZ`Gabbr8   c           	      R    t        g dg d      }t        dd| it        |fi |S )Nr/        i   i   r$  r  )mambaout_baser&  r'  s      r7   r6  r6    s-    ]1EFJa
ad:F`Y_F`aar8   c           	      Z    t        g dg ddddd      }t        d	d| it        |fi |S )
Nr/  r   Fr   ư>norm_mlp)r   r   r   r   r~   r   r  )mambaout_small_rwr&  r'  s      r7   r:  r:    s>     J eJe$zJd]cJdeer8   c           
      V    t        dddddddd	      }t        dd
| it        |fi |S )N)r   r      r   r3        @      ?Fr   r8  r9  r   r   r|   r}   r   r   r~   r   r  )mambaout_base_short_rwr&  r'  s      r7   r@  r@    sE    !	J jjtT^OibhOijjr8   c           
      V    t        dddddddd	      }t        dd
| it        |fi |S )Nr   r[      r   r3  g      @r>  Fr   r8  r9  r?  r  )mambaout_base_tall_rwr&  r'  s      r7   rD  rD    sE    !	J i
idS]NhagNhiir8   c                 X    t        ddddddddd	
	      }t        dd| it        |fi |S )Nr/  r3  r=        ?Fr   r8  silur9  	r   r   r|   r}   r   r   r~   r!   r   r  )mambaout_base_wide_rwr&  r'  s      r7   rI  rI    H    !
J i
idS]NhagNhiir8   c                 X    t        ddddddddd	
	      }t        dd| it        |fi |S )NrB  r3  r=  rF  Fr   r8  rG  r9  rH  r  )mambaout_base_plus_rwr&  r'  s      r7   rL  rL    rJ  r8   c           
      V    t        dddddddd	      }t        dd
| it        |fi |S )N)r   r   r   r   )       r#  @   r   Fr   g-C6?rG  r9  )r   r   r|   r   r   r~   r!   r   r  )r  r&  r'  s      r7   r  r    sD    	J a
ad:F`Y_F`aar8   )r   rs   )6rA   collectionsr   typingr   r   r   r   r   r   r	   	timm.datar   r   timm.layersr   r   r   r   r   r   r   _builderr   	_featuresr   _manipulater   	_registryr   r   rE   r   rI   rT   rX   rw   r   r   r  r  default_cfgsr   r%  r+  r-  r1  r6  r:  r@  rD  rI  rL  r  r   r8   r7   <module>rZ     s   $ 5 5   A J  J  J * + ' </299 /d")) < <<bii <~6RYY 6r4BII 4n[ryy [|2	 % '+' ,0S, +/S+ +/S+ 484 9= -8_g9 ,0, ]M]cdQ)& )X c c
 b b b b
 c c
 b b
 	f 	f k k j j j j j j b br8   