
    ^jI                        d Z ddlZddlmZ ddlmZ ddlmZmZm	Z	m
Z
mZ ddlZddlmZ ddlmZmZmZ ddlmZmZmZmZmZ dd	lmZ dd
lmZ ddlmZmZ dgZ d*dejB                  de"de#fdZ$ G d dejB                        Z% G d dejB                        Z& G d de&      Z' G d dejB                        Z( G d dejB                        Z) G d dejB                        Z* G d dejB                        Z+ G d dejB                        Z,d  Z-d+d!Z.d,d"Z/ e e/d#$       e/d#$       e/d#$      d%      Z0ed+d&e,fd'       Z1ed+d&e,fd(       Z2ed+d&e,fd)       Z3y)-zk Sequencer

Paper: `Sequencer: Deep LSTM for Image Classification` - https://arxiv.org/pdf/2205.01972.pdf

    N)partial)
accumulate)ListOptionalTupleTypeUnion)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STDDEFAULT_CROP_PCT)lecun_normal_DropPathMlp
PatchEmbedClassifierHead   )build_model_with_cfg)named_apply)register_modelgenerate_default_cfgsSequencer2dmodulename	head_biasc                    t        | t        j                        rC|j                  d      rTt        j                  j                  | j                         t        j                  j                  | j                  |       y |rKt        | j                         | j                  t        j                  j                  | j                         y t        j                  j                  | j                         | j                  Zd|v r,t        j                  j                  | j                  d       y t        j                  j                  | j                         y y y t        | t        j                        rLt        | j                         | j                  *t        j                  j                  | j                         y y t        | t        j                  t        j                  t        j                  f      rSt        j                  j!                  | j                         t        j                  j                  | j                         y t        | t        j"                  t        j$                  t        j&                  f      rZdt)        j*                  | j,                        z  }| j/                         D ]$  }t        j                  j1                  || |       & y t3        | d      r| j5                          y y )Nheadmlpư>)stdg      ?init_weights)
isinstancennLinear
startswithinitzeros_weight	constant_biasr   xavier_uniform_normal_Conv2d	LayerNormBatchNorm2d	GroupNormones_RNNGRULSTMmathsqrthidden_size
parametersuniform_hasattrr    )r   r   r   flaxstdvr'   s         `/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/sequencer.py_init_weightsr=      s   &"))$??6"GGNN6==)GGfkk95fmm,;;*GGNN6;;/''6;;*}>v{{3	 +	 + 
FBII	&fmm$;;"GGNN6;;' #	FR\\2>>2<<H	I
fmm$
v{{#	FRVVRVVRWW5	6TYYv1122'') 	2FGGVdUD1	2		( 
)    c                   `     e Zd Z fdZdej
                  deej
                  df   fdZ xZS )RNNIdentityc                 "    t         |           y Nsuper__init__)selfargskwargs	__class__s      r<   rE   zRNNIdentity.__init__<       r>   xreturnNc                 
    |d fS rB    rF   rK   s     r<   forwardzRNNIdentity.forward?   s    $wr>   )	__name__
__module____qualname__rE   torchTensorr   rP   __classcell__rI   s   @r<   r@   r@   ;   s-     %d0B*C r>   r@   c                   R     e Zd Z	 	 	 	 	 	 	 d
dededededededef fdZd	 Z xZS )	RNN2dBase
input_sizer6   
num_layersr)   bidirectionalunionwith_fcc
                 Z   ||	d}
t         |           || _        || _        |rd|z  n|| _        || _        d| _        d| _        || _        d | _	        |r|dk(  r+t        j                  d| j                  z  |fi |
| _	        n|dk(  r(t        j                  | j                  |fi |
| _	        n[|dk(  r/t        j                  | j                  |fi |
| _	        d| _        n'|dk(  r.t        j                  | j                  |fi |
| _	        d| _        nt        d	|z         |dk(  r2	 d| j                  z  |k7  rt        d
d| j                  z   d| d      |dk(  r,	 | j                  |k7  rt        d
| j                   d| d      |dk(  r3| j                  |k7  rt        d
| j                   d| d      d| _        nF|dk(  r3| j                  |k7  rt        d
| j                   d| d      d| _        nt        d	|z         t               | _        t               | _        y )Ndevicedtype   TcataddverticalF
horizontalzUnrecognized union: zThe output channel z% is different from the input channel .)rD   rE   rZ   r6   output_sizer]   with_verticalwith_horizontalr^   fcr"   r#   
ValueErrorr@   rnn_vrnn_hrF   rZ   r6   r[   r)   r\   r]   r^   ra   rb   ddrI   s              r<   rE   zRNN2dBase.__init__E   sP    /$&.;1{?
!#~))A(8(8$8*KK%))D$4$4jGBG*$))D$4$4jGBG',$,&))D$4$4jGBG%*" !7%!?@@e^4###z1 #6q4;K;K7K6LLqr|q}}~!  A  Ae^:- #6t7G7G6HHmnxmyyz!{||j :- #6t7G7G6HHmnxmyyz!{||#(D l":- #6t7G7G6HHmnxmyyz!{||!&D3e;<< ]
 ]
r>   c                 R   |j                   \  }}}}| j                  rd|j                  dddd      }|j                  d||      }| j	                  |      \  }}|j                  |||d      }|j                  dddd      }nd }| j
                  r<|j                  d||      }| j                  |      \  }}|j                  |||d      }nd }|1|/| j                  dk(  rt        j                  ||gd      }n||z   }n	||}n||}| j                  | j                  |      }|S )Nr   rc   r      rd   )dim)shaperj   permutereshapern   rk   ro   r]   rT   rd   rl   )	rF   rK   BHWCv_hs	            r<   rP   zRNN2dBase.forward   s'   WW
1a		!Q1%A		"a#A::a=DAq		!Q2&A		!Q1%AA		"a#A::a=DAq		!Q2&AA=Q]zzU"IIq!f"-E]A]A77
Ar>   r   TTrd   TNN)	rQ   rR   rS   intboolstrrE   rP   rV   rW   s   @r<   rY   rY   C   si      "& :#:# :# 	:#
 :#  :# :# :#x r>   rY   c                   L     e Zd Z	 	 	 	 	 	 	 d	dededededededef fdZ xZS )
LSTM2drZ   r6   r[   r)   r\   r]   r^   c
                     ||	d}
t         |   |||||||||		       | j                  r!t        j                  |||fd||d|
| _        | j                  r"t        j                  |||fd||d|
| _        y y )Nr`   T)batch_firstr)   r\   )rD   rE   rj   r"   r3   rn   rk   ro   rp   s              r<   rE   zLSTM2d.__init__   s     /[*dMSXZacikpq !+ DJ  !+ DJ  r>   r   )rQ   rR   rS   r   r   r   rE   rV   rW   s   @r<   r   r      sh      "& !! ! 	!
 !  ! ! ! !r>   r   c                       e Zd Zdee eej                  d      ej                  ddddddddfd	e	d
e	de
deej                     deej                     deej                     deej                     de	dededede
de
f fdZd Z xZS )Sequencer2dBlock      @r   epsr   Trd           Nru   r6   	mlp_ratio	rnn_layer	mlp_layer
norm_layer	act_layerr[   r\   r]   r^   drop	drop_pathc                    ||d}t         |           t        ||z        } ||fi || _         |||f||	|
|d|| _        |dkD  rt        |      nt        j                         | _         ||fi || _	         |||f||d|| _
        y )Nr`   )r[   r\   r]   r^   r   )r   r   )rD   rE   r   norm1
rnn_tokensr   r"   Identityr   norm2mlp_channels)rF   ru   r6   r   r   r   r   r   r[   r\   r]   r^   r   r   ra   rb   rq   channels_dimrI   s                     r<   rE   zSequencer2dBlock.__init__   s    $ /9s?+*r*
#
 "'
 
 1:B),BKKM*r*
%c<^9SW^[]^r>   c                     || j                  | j                  | j                  |                  z   }|| j                  | j                  | j	                  |                  z   }|S rB   )r   r   r   r   r   rO   s     r<   rP   zSequencer2dBlock.forward   sQ    ttzz!}=>>t00A?@@r>   )rQ   rR   rS   r   r   r   r"   r-   GELUr   floatr   Moduler   r   rE   rP   rV   rW   s   @r<   r   r      s    
  #)/),*1",,D*I)+"& !!!_!_ !_ 	!_
 BII!_ BII!_ RYY!_ BII!_ !_  !_ !_ !_ !_ !_Fr>   r   c                   $     e Zd Z fdZd Z xZS )Shufflec                 "    t         |           y rB   rC   )rF   rI   s    r<   rE   zShuffle.__init__   rJ   r>   c                     | j                   rZ|j                  \  }}}}t        j                  ||z        }|j	                  |d|      }|d d |d d f   j	                  |||d      }|S )Nrt   )trainingrv   rT   randpermrx   )rF   rK   ry   rz   r{   r|   rs          r<   rP   zShuffle.forward   sc    ==JAq!Qq1u%A		!R#A!Q'
""1aB/Ar>   )rQ   rR   rS   rE   rP   rV   rW   s   @r<   r   r      s    r>   r   c                   8     e Zd Z	 	 ddededef fdZd Z xZS )Downsample2d	input_dim
output_dim
patch_sizec                 j    ||d}t         |           t        j                  ||f||d|| _        y )Nr`   )kernel_sizestride)rD   rE   r"   r,   down)rF   r   r   r   ra   rb   rq   rI   s          r<   rE   zDownsample2d.__init__  s:     /IIieT^ebde	r>   c                 x    |j                  dddd      }| j                  |      }|j                  dddd      }|S )Nr   rs   r   rc   )rw   r   rO   s     r<   rP   zDownsample2d.forward  s<    IIaAq!IIaLIIaAq!r>   )NN)rQ   rR   rS   r   rE   rP   rV   rW   s   @r<   r   r     s5     
f
f 
f 	
fr>   r   c            (       V    e Zd Zdeee eej                  d      ej                  ddddddddfd	e
d
e
de
de
de
dededeej                     deej                     deej                     deej                     deej                     de
dededededeeee   f   f$ fdZd Z xZS )Sequencer2dStageFr   r   r   Trd   r   Nru   dim_outdepthr   r6   r   
downsampleblock_layerr   r   r   r   r[   r\   r]   r^   r   r   c                 j   t         |           ||d}|rt        |||fi || _        n ||k(  sJ t	        j
                         | _        g }t        |      D ]B  }|j                   |||f||	|
|||||||t        |t        t        f      r||   n|d|       D t	        j                  | | _        y )Nr`   )r   r   r   r   r   r[   r\   r]   r^   r   r   )rD   rE   r   r   r"   r   rangeappendr!   listtuple
Sequentialblocks)rF   ru   r   r   r   r6   r   r   r   r   r   r   r   r[   r\   r]   r^   r   r   ra   rb   rq   r   	block_idxrI   s                           r<   rE   zSequencer2dStage.__init__  s    . 	/*3JrJDO'>!> kkmDOu 	IMM+ $##%#%+2<Yu2V)I.\e  	" mmV,r>   c                 J    | j                  |      }| j                  |      }|S rB   )r   r   rO   s     r<   rP   zSequencer2dStage.forwardI  s"    OOAKKNr>   )rQ   rR   rS   r   r   r   r   r"   r-   r   r   r   r   r   r   r   r	   r   rE   rP   rV   rW   s   @r<   r   r     s1     %+;)/),*1",,D*I)+"& 35+1-1- 1- 	1-
 1- 1- 1- 1- bii1- BII1- BII1- RYY1- BII1- 1-  1-  !1-" #1-$ %1-& UDK/0'1-fr>   r   c            -           e Zd Zddddddddd	eee eej                  d
      ej                  ddddddddddfde
de
de
dedee
df   dee
df   dee
df   dee
df   deedf   deej                     deej                     deej                     deej                     d eej                     d!e
d"ed#ed$ed%ed&ed'ed(ef, fd)Zd4d*Zej(                  j*                  d4d+       Zej(                  j*                  d5d,       Zej(                  j*                  d-ej                  fd.       Zd6de
dee   fd/Zd0 Zd4d1efd2Zd3 Z xZS )7r        rs   avg   rs      rs   )   rc   rc   r        r   r   0   `   r   r   r   r   r   r   r   r   r   Trd   r   FNnum_classesimg_sizein_chansglobal_poollayers.patch_sizes
embed_dimshidden_sizes
mlp_ratiosr   r   r   r   r   num_rnn_layersr\   r]   r^   	drop_ratedrop_path_ratenlhb	stem_normc                    t         |           ||d}|dv sJ || _        || _        || _        |d   x| _        | _        d| _        d| _        g | _	        t        dd |d   ||d   |r|nd ddd|| _        t        |      t        |      cxk(  r-t        |      cxk(  rt        |      cxk(  rt        |	      k(  sJ  J t        t        |d             }g }|d   }t        |      D ]  \  }}|t!        |||   fi d	||   d
|dkD  d||   d||   d|	|   d|
d|d|d|d|d|d|d|d|d|d||gz  }||   }| xj                  t#        |||   d|       gz  c_	         t%        j&                  | | _         ||d   fi || _        t-        | j
                  |f||| j                  d|| _        | j1                  |       y )Nr`   ) r   rt   NHWCr   F)r   r   r   	embed_dimr   flatten
output_fmtc                     | |z  S rB   rN   )rK   ys     r<   <lambda>z&Sequencer2d.__init__.<locals>.<lambda>  s
    q1u r>   r   r   r   r6   r   r   r   r   r   r   r[   r\   r]   r^   r   r   stages.)num_chs	reductionr   )	pool_typer   	input_fmt)r   rN   )rD   rE   r   r   r   num_featureshead_hidden_sizefeature_dimr   feature_infor   stemlenr   r   	enumerater   dictr"   r   stagesnormr   r   r    ) rF   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r\   r]   r^   r   r   r   r   ra   rb   rq   
reductionsr   prev_dimir~   rI   s                                   r<   rE   zSequencer2d.__init__P  s   6 	/k)))& &4>rNBD1  	
"1~ m%.zD	
 	
	 6{c+.i#j/iSEViZ]^hZiiiiii*[2DEF
a=j) 	iDAq'1 Qi q5	
 'q> )O %Q- ( $ $ & $ * ,    !" #$ )'  F* "!}H$x:a=[bcdbeYf"g!hh/	i2 mmV,z"~44	"
 "oo
 
	 	t$r>   c                     |r t        j                  | j                         nd}t        t	        t
        |      |        y )Nr   )r   )r   )r4   logr   r   r   r=   )rF   r   r   s      r<   r    zSequencer2d.init_weights  s.    37TXXd..//R	GMY?Mr>   c                 6    t        d|r	ddg      S g d      S )Nz^stem)z^stages\.(\d+)Nz^norm)i ))z^stages\.(\d+)\.blocks\.(\d+)N)z^stages\.(\d+)\.downsample)r   r   )r   r   )r   )rF   coarses     r<   group_matcherzSequencer2d.group_matcher  s3      *$

 
	



 
	
r>   c                     |rJ d       y )Nz$gradient checkpointing not supportedrN   )rF   enables     r<   set_grad_checkpointingz"Sequencer2d.set_grad_checkpointing  s    AAAz6r>   rL   c                     | j                   S rB   r   )rF   s    r<   get_classifierzSequencer2d.get_classifier  s    yyr>   c                 L    || _         | j                  j                  ||       y )N)r   )r   r   reset)rF   r   r   s      r<   reset_classifierzSequencer2d.reset_classifier  s    &		{;r>   c                 l    | j                  |      }| j                  |      }| j                  |      }|S rB   )r   r   r   rO   s     r<   forward_featureszSequencer2d.forward_features  s.    IIaLKKNIIaLr>   
pre_logitsc                 N    |r| j                  |d      S | j                  |      S )NT)r	  r  )rF   rK   r	  s      r<   forward_headzSequencer2d.forward_head  s$    0:tyyty,L		!Lr>   c                 J    | j                  |      }| j                  |      }|S rB   )r  r  rO   s     r<   rP   zSequencer2d.forward  s'    !!!$a r>   F)TrB   )rQ   rR   rS   r   r   r   r   r"   r-   r   r   r   r   r   r   r   r   rE   r    rT   jitignorer   r   r  r   r  r  r  rP   rV   rW   s   @r<   r   r   O  s@     $$&2+7*>,<,@+;)/),*1",,D*I)+"#"& !$&#3Y%Y% Y% 	Y%
 Y% #s(OY% sCxY% c3hY%  S/Y% eSj)Y% biiY% BIIY% BIIY% RYYY% BIIY%   !Y%"  #Y%$ %Y%& 'Y%( )Y%* "+Y%, -Y%. /Y%vN YY
 
 YYB B YY		  <C <hsm <M$ Mr>   c                     d| v r| S d| v r| d   } ddl }i }| j                         D ]C  \  }}|j                  dd |      }|j                  dd|      }|j                  d	d
      }|||<   E |S )z$ Remap original checkpoints -> timm zstages.0.blocks.0.norm1.weightmodelr   Nzblocks.([0-9]+).([0-9]+).downc                 D    dt        | j                  d            dz    dS )Nr   r   z.downsample.down)r   group)rK   s    r<   r   z&checkpoint_filter_fn.<locals>.<lambda>  s$    QWWUVZ[\I\H]]m?n r>   zblocks.([0-9]+).([0-9]+)zstages.\1.blocks.\2zhead.zhead.fc.)reitemssubreplace)
state_dictr  r  out_dictkr}   s         r<   checkpoint_filter_fnr    s    ':5*(
H  " 1FF35npqrFF.0FJIIgz*	 Or>   c                     t        t        d            }|j                  d|      }t        t        | |ft
        t        d|      d|}|S )Nrs   out_indicesT)flatten_sequentialr  )pretrained_filter_fnfeature_cfg)r   r   popr   r   r  r   )variant
pretrainedrH   default_out_indicesr  r  s         r<   _create_sequencer2dr%    sY    a/**],?@K  2DkJ E Lr>   c                 <    | ddd t         ddt        t        dddd|S )	Nr   )rs   r   r   bicubicTz	stem.projzhead.fcz
apache-2.0)urlr   rZ   	pool_sizecrop_pctinterpolationfixed_input_sizemeanr   
first_conv
classifierlicense)r   r
   r   )r(  rH   s     r<   _cfgr1    s6    =t$yVZ%.B!  r>   ztimm/)	hf_hub_id)zsequencer2d_s.in1kzsequencer2d_m.in1kzsequencer2d_l.in1krL   c                 x    t        g dg dg dg dg dt        ddd	      }t        d
d	| it        |fi |}|S )Nr   r   rc   r   r   r   r   r   Trd   	r   r   r   r   r   r   r\   r]   r^   r#  )sequencer2d_sr   r   r%  r#  rH   
model_argsr  s       r<   r6  r6    sK     '%'
J  eJe$zJd]cJdeELr>   c                 z    t        d
g dg dg dg dg dt        dddd	|}t        dd	| it        |fi |}|S )N)r   rs      rs   r4  r   r   r   Trd   r5  r#  rN   )sequencer2d_mr7  r8  s       r<   r<  r<    Z     
 '%'
 
J  eJe$zJd]cJdeELr>   c                 z    t        d
g dg dg dg dg dt        dddd	|}t        dd	| it        |fi |}|S )N)r   r      r   r4  r   r   r   Trd   r5  r#  rN   )sequencer2d_lr7  r8  s       r<   r@  r@  .  r=  r>   )r   Fr  )r   )4__doc__r4   	functoolsr   	itertoolsr   typingr   r   r   r   r	   rT   torch.nnr"   	timm.datar
   r   r   timm.layersr   r   r   r   r   _builderr   _manipulater   	_registryr   r   __all__r   r   r   r=   r@   rY   r   r   r   r   r   r   r  r%  r1  default_cfgsr6  r<  r@  rN   r>   r<   <module>rM     sj       5 5   S S P P * $ </")) 3 5 B")) ^		 ^B#Y #L'ryy 'T
bii 
299 (7ryy 7tF")) FR$	 %111&            r>   