
    ^j[N              	          d Z ddlmZmZmZmZmZmZmZm	Z	 ddl
Z
ddlmZ ddlmc mZ ddlmZmZ ddlmZmZmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lm Z m!Z! dgZ" G d dejF                        Z$ G d dejJ                        Z& G d dejJ                        Z' G d dejF                        Z( G d dejF                        Z) G d dejF                        Z* G d dejF                        Z+ G d dejF                        Z, G d dejF                        Z-dee.e
j^                  f   dejF                  dee.e
j^                  f   fd Z0d0d!e.d"edee.ef   fd#Z1 e! e1d$%       e1d$%       e1d$%       e1d$d&'      d(      Z2d1d)e.d*e3d"ede-fd+Z4e d1d*e3d"ede-fd,       Z5e d1d*e3d"ede-fd-       Z6e d1d*e3d"ede-fd.       Z7e d1d*e3d"ede-fd/       Z8y)2a  SHViT
SHViT: Single-Head Vision Transformer with Memory Efficient Macro Design
Code: https://github.com/ysj9909/SHViT
Paper: https://arxiv.org/abs/2401.16456

@inproceedings{yun2024shvit,
  author={Yun, Seokju and Ro, Youngmin},
  title={SHViT: Single-Head Vision Transformer with Memory Efficient Macro Design},
  booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
  pages={5756--5767},
  year={2024}
}
    )AnyDictListOptionalSetTupleTypeUnionNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)
GroupNorm1SqueezeExciteSelectAdaptivePool2d	LayerTypetrunc_normal_   )build_model_with_cfg)feature_take_indices)checkpoint_seq)register_modelgenerate_default_cfgsSHViTc                        e Zd Zdej                  f fdZdej                  dej                  fdZ ej                         dej                  fd       Z
 xZS )Residualmc                 0    t         |           || _        y N)super__init__r   )selfr   	__class__s     \/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/shvit.pyr    zResidual.__init__   s        xreturnc                 *    || j                  |      z   S r   )r   r!   r%   s     r#   forwardzResidual.forward#   s    466!9}r$   c                    t        | j                  t              r| j                  j                         }|j                  |j
                  k(  sJ t        j                  |j                  j                  d   |j                  j                  d   dd      }t        j                  |g d      }|xj                  |j                  |j                  j                        z  c_        |S | S )Nr   r   )r   r   r   r   )
isinstancer   
Conv2dNormfusegroupsin_channelstorchonesweightshapeFpadtodevice)r!   r   identitys      r#   r-   zResidual.fuse&   s    dffj)A88q}},-,zz!((.."3QXX^^A5F1MHuuXy1HHHAHHOO44HHKr$   )__name__
__module____qualname__nnModuler    r0   Tensorr)   no_gradr-   __classcell__r"   s   @r#   r   r      sU    "))  %,,  U]]_	bii 	 	r$   r   c                        e Zd Z	 	 	 	 	 	 d
dedededededef fdZ ej                         dej                  fd	       Z	 xZ
S )r,   r/   out_channelskernel_sizestridepaddingbn_weight_initc	           
         ||d}
t         |           | j                  dt        j                  |||||fddi|
|	       | j                  dt        j
                  |fi |
       t        j                  j                  | j                  j                  |       t        j                  j                  | j                  j                  d       y )Nr7   dtypecbiasFbnr   )r   r    
add_moduler<   Conv2dBatchNorm2dinit	constant_rM   r2   rL   )r!   r/   rC   rD   rE   rF   rG   r7   rJ   kwargsddr"   s              r#   r    zConv2dNorm.__init__4   s     /RYY{FGaJOaSUaY_a 	bbnn\@R@A
$''...9
$'',,*r$   r&   c                    | j                   j                         \  }}|j                  |j                  |j                  z   dz  z  }|j                  |d d d d d f   z  }|j
                  |j                  |j                  z  |j                  |j                  z   dz  z  z
  }t        j                  |j                  d      | j                  j                  z  |j                  d      |j                  dd  | j                  j                  | j                  j                  | j                  j                  | j                  j                  |j                  j                   |j                  j"                  	      }|j                  j$                  j'                  |       |j
                  j$                  j'                  |       |S )N      ?r   r      )	r/   rC   rD   rE   rF   dilationr.   r7   rJ   )_modulesvaluesr2   running_varepsrL   running_meanr<   rO   sizerK   r.   r3   rE   rF   rX   r7   rJ   datacopy_)r!   rK   rM   wbr   s         r#   r-   zConv2dNorm.fuseH   s:   $$&2II"&&0S88HHqD$,--GGboo		1R^^bff5LQT4TTTIIq	DFFMM166==FFNNVV__66==88??((..

 	
A	!r$   )r   r   r   r   NN)r9   r:   r;   intr    r0   r?   r<   rO   r-   r@   rA   s   @r#   r,   r,   3   s}    
  !"#++ + 	+
 + +  +( U]]_bii  r$   r,   c            	            e Zd Z	 	 	 	 ddedededef fdZ ej                         de	j                  fd       Z xZS )	
NormLinearin_featuresout_featuresrL   stdc                 z   ||d}t         |           | j                  dt        j                  |fi |       | j                  dt        j
                  ||fd|i|       t        | j                  j                  |       |r5t        j                  j                  | j                  j                  d       y y )NrI   rM   lrL   )rh   r   )r   r    rN   r<   BatchNorm1dLinearr   rj   r2   rQ   rR   rL   )	r!   rf   rg   rL   rh   r7   rJ   rT   r"   s	           r#   r    zNormLinear.__init___   s     /bnn[?B?@RYY{LRtRrRSdffmm-GGdffkk1- r$   r&   c                    | j                   j                         \  }}|j                  |j                  |j                  z   dz  z  }|j
                  | j                  j                  | j                  j                  z  |j                  |j                  z   dz  z  z
  }|j                  |d d d f   z  }|j
                  $|| j                  j                  j                  z  }n<|j                  |d d d f   z  j                  d      | j                  j
                  z   }t        j                  |j                  d      |j                  d      |j                  j                  |j                  j                        }|j                  j                   j#                  |       |j
                  j                   j#                  |       |S )NrV   r   r   rI   )rY   rZ   r2   r[   r\   rL   rM   r]   rj   Tviewr<   rl   r^   r7   rJ   r_   r`   )r!   rM   rj   ra   rb   r   s         r#   r-   zNormLinear.fusep   s@   $$&AII"&&0S88GGdgg**TWW^^;r~~PRPVPV?V[^>^^^HHqqz!66>DFFMMOO#AAagJ&,,R0466;;>AIIaffQi188??!((..Y	A	!r$   )Tg{Gz?NN)r9   r:   r;   rc   boolfloatr    r0   r?   r<   rl   r-   r@   rA   s   @r#   re   re   ^   sc    
 .. . 	.
 ." U]]_bii  r$   re   c                        e Zd Zej                  ddfdededeej                     f fdZde	j                  de	j                  fdZ xZS )	PatchMergingNdimout_dim	act_layerc                    ||d}t         |           t        |dz        }t        ||fi || _         |       | _        t        ||dddfd|i|| _         |       | _        t        |dfi || _	        t        ||fi || _
        y )NrI         rW   r   r.   g      ?)r   r    rc   r,   conv1act1conv2act2r   seconv3)	r!   ru   rv   rw   r7   rJ   rT   hid_dimr"   s	           r#   r    zPatchMerging.__init__   s     /cAg,W33
K	!QP'PRP
K	447B7
r$   r%   r&   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }|S r   )r{   r|   r}   r~   r   r   r(   s     r#   r)   zPatchMerging.forward   sU    JJqMIIaLJJqMIIaLGGAJJJqMr$   r9   r:   r;   r<   ReLUrc   r	   r=   r    r0   r>   r)   r@   rA   s   @r#   rt   rt      sV    
 *,88 8 BII	8$ %,, r$   rt   c                        e Zd Zej                  ddfdededeej                     f fdZde	j                  de	j                  fdZ xZS )	FFNNru   	embed_dimrw   c                     ||d}t         |           t        ||fi || _         |       | _        t        ||fddi|| _        y )NrI   rG   r   )r   r    r,   pw1actpw2)r!   ru   r   rw   r7   rJ   rT   r"   s          r#   r    zFFN.__init__   sN     /c933;iEQE"Er$   r%   r&   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )r   r   r   r(   s     r#   r)   zFFN.forward   s.    HHQKHHQKHHQKr$   r   rA   s   @r#   r   r      sZ    
 *,FF F BII	F %,, r$   r   c                        e Zd ZdZeej                  ddfdedededeej                     deej                     f
 fdZ
d	ej                  d
ej                  fdZ xZS )SHSAzSingle-Head Self-AttentionNru   qk_dimpdim
norm_layerrw   c                    ||d}t         	|           |dz  | _        || _        || _        || _         ||fi || _        t        ||dz  |z   fi || _        t        j                   |       t        ||fddi|      | _        y )NrI   g      rW   rG   r   )r   r    scaler   ru   r   pre_normr,   qkvr<   
Sequentialproj)
r!   ru   r   r   r   rw   r7   rJ   rT   r"   s
            r#   r    zSHSA.__init__   s     /t^
	"4.2.dFQJ$5<<MM)+z#s/[ST/[XZ/[\	r$   r%   r&   c                    |j                   \  }}}}t        j                  || j                  | j                  | j                  z
  gd      \  }}| j                  |      }| j                  |      }t        j                  || j                  | j                  | j                  gd      \  }	}
}|	j                  d      |
j                  d      |j                  d      }}
}	|	j                  dd      |
z  | j                  z  }|j                  d      }||j                  dd      z  j                  || j                  ||      }| j                  t        j                  ||gd            }|S )Nr   )ru   rW   rn   )r3   r0   splitr   ru   r   r   r   flatten	transposer   softmaxreshaper   cat)r!   r%   B_HWx1x2r   qkvattns                r#   r)   zSHSA.forward   s$   WW
1aQDHHtyy,@ AKB]]2hhrl++cDKKdii#HaP1a))A,		!aiila1B#a'4::5|||#$..R((11!TYY1EIIeiiR23r$   )r9   r:   r;   __doc__r   r<   r   rc   r	   r=   r    r0   r>   r)   r@   rA   s   @r#   r   r      s~    $ +5)+]] ] 	]
 RYY] BII], %,, r$   r   c                        e Zd Zeej
                  ddfdededededeej                     deej                     f fdZ
d	ej                  d
ej                  fdZ xZS )
BasicBlockNru   r   r   typer   rw   c	           	      6   ||d}	t         
|           t        t        ||dddf|dd|	      | _        |dk(  rt        t        |||||fi |	      | _        nt        j                         | _        t        t        |t        |dz        fi |	      | _        y )NrI   rz   r   r   )r.   rG   srW   )r   r    r   r,   convr   mixerr<   Identityr   rc   ffn)r!   ru   r   r   r   r   rw   r7   rJ   rT   r"   s             r#   r    zBasicBlock.__init__   s     /ZS!Q^#VW^[]^_	3;!$sFD*i"VSU"VWDJDJCSq\8R89r$   r%   r&   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )r   r   r   r(   s     r#   r)   zBasicBlock.forward   s.    IIaLJJqMHHQKr$   r9   r:   r;   r   r<   r   rc   strr	   r=   r    r0   r>   r)   r@   rA   s   @r#   r   r      s     +5)+:: : 	:
 : RYY: BII:( %,, r$   r   c                        e Zd Zeej
                  ddfdededededededeej                     d	eej                     f fd
Z
dej                  dej                  fdZ xZS )
StageBlockNprev_dimru   r   r   r   depthr   rw   c                 ,   |	|
d}t         |           d| _        ||k7  rt        j                  t        t        ||dddfd|i|      t        t        |t        |dz        |fi |      t        |||fi |t        t        ||dddfd|i|      t        t        |t        |dz        |fi |            nt        j                         | _        t        j                  t        |      D cg c]  }t        ||||||fi | c} | _        y c c}w )NrI   Frz   r   r.   rW   )r   r    grad_checkpointingr<   r   r   r,   r   rc   rt   r   
downsampleranger   blocks)r!   r   ru   r   r   r   r   r   rw   r7   rJ   rT   r   r"   s                r#   r    zStageBlock.__init__   s    /"' _ --Z(Aq!SHSPRSTS3x!|#4iF2FG3	8R8ZS!QD#DDESc#'lI<<=
 #%++- 	 mmV[\aVb&
QRJsFD$
ILL&
  &
s   /Dr%   r&   c                     | j                  |      }| j                  r6t        j                  j	                         st        | j                  |      }|S | j                  |      }|S r   )r   r   r0   jitis_scriptingr   r   r(   s     r#   r)   zStageBlock.forward  sS    OOA""599+A+A+Ct{{A.A  AAr$   r   rA   s   @r#   r   r      s     +5)+  	
    RYY BII8 %,, r$   r   c                       e Zd Zddddddddd	eej
                  d
d
fdedededeeeef   deeeef   deeeef   deeeef   deeeef   de	de
ej                     de
ej                     f fdZej                  j                  defd       Zej                  j                  d-dedeeef   fd       Zej                  j                  d.d       Zej                  j                  dej                  fd       Zd/dedefdZ	 	 	 	 	 d0dej2                  deeeee   f      d ed!ed"ed#edeeej2                     eej2                  eej2                     f   f   fd$Z	 	 	 d1deeee   f   d%ed&efd'Zdej2                  dej2                  fd(Zd-dej2                  d)edej2                  fd*Z dej2                  dej2                  fd+Z! ejD                         d,        Z# xZ$S )2r   rz     avg)      i  )    @   `   )   r   r   )r   rW   rz   )r   r   r           Nin_chansnum_classesglobal_poolr   partial_dimr   r   types	drop_rater   rw   c                    t         |           ||d}|| _        || _        |	| _        g | _        |d   }t        j                  t        ||dz  dddfi | |       t        |dz  |dz  dddfi | |       t        |dz  |dz  dddfi | |       t        |dz  |dddfi |      | _	        g }|}t        t        |            D ]i  }|j                  t        d|||   ||   ||   ||   ||   |
|d|       ||   }| j
                  j                  t        |d|dz   z  d	| 
             k t        j                  | | _        |d   x| _        | _        t%        |      | _        |rt        j(                  d      nt        j*                         | _        |dkD  rt/        | j"                  |fi || _        y t        j*                         | _        y )NrI   r      rz   rW   r   ry   )r   ru   r   r   r   r   r   rw   zstages.)num_chs	reductionmodulern   	pool_type )r   r    r   r   r   feature_infor<   r   r,   patch_embedr   lenappendr   dictstagesnum_featureshead_hidden_sizer   r   Flattenr   r   re   head)r!   r   r   r   r   r   r   r   r   r   r   rw   r7   rJ   rT   stem_chsr   prev_chsir"   s                      r#   r    zSHViT.__init__  s     	/& " Q<==xQ1a>2>Kx1}h!mQ1CCKx1}h!mQ1CCKx1}h1a>2>
 s9~& 	gAMM* 
!aLay ^1XAh%#
 
 
 !|H$$T(a!A#hY`ab`cWd%ef	g mmV, 5>bMAD1/+F(3rzz!}LWZ[OJt44kHRH	acalalan	r$   r&   c                     t               S r   )setr!   s    r#   no_weight_decayzSHViT.no_weight_decayY  s	    ur$   coarsec                 ,    t        d|rdnddg      }|S )Nz^patch_embedz^stages\.(\d+))z^stages\.(\d+).downsample)r   )z^stages\.(\d+)\.blocks\.(\d+)N)stemr   )r   )r!   r   matchers      r#   group_matcherzSHViT.group_matcher]  s'     (.$485
 r$   c                 4    | j                   D ]	  }||_         y r   )r   r   )r!   enabler   s      r#   set_grad_checkpointingzSHViT.set_grad_checkpointingh  s     	*A#)A 	*r$   c                 .    | j                   j                  S r   )r   rj   r   s    r#   get_classifierzSHViT.get_classifierm  s    yy{{r$   c                    || _         t        |      | _        |rt        j                  d      nt        j
                         | _        |dkD  rt        | j                  |      | _	        y t        j
                         | _	        y )Nr   r   r   )
r   r   r   r<   r   r   r   re   r   r   )r!   r   r   s      r#   reset_classifierzSHViT.reset_classifierq  s[    &/+F(3rzz!}FQTUoJt44kB	[][f[f[h	r$   r%   indicesnorm
stop_early
output_fmtintermediates_onlyc                 r   |dv sJ d       g }t        t        | j                        |      \  }}	| j                  |      }t        j
                  j                         s|s| j                  }
n| j                  d|	dz    }
t        |
      D ]#  \  }} ||      }||v s|j                  |       % |r|S ||fS )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWzOutput shape must be NCHW.Nr   )	r   r   r   r   r0   r   r   	enumerater   )r!   r%   r   r   r   r   r   intermediatestake_indices	max_indexr   feat_idxstages                r#   forward_intermediateszSHViT.forward_intermediatesx  s    * Y&D(DD&"6s4;;7G"Qi Q99!!#:[[F[[)a-0F(0 	(OHeaA<'$$Q'	(
   -r$   
prune_norm
prune_headc                     t        t        | j                        |      \  }}| j                  d|dz    | _        |r| j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r    )r   r   r   r   )r!   r   r  r  r  r  s         r#   prune_intermediate_layerszSHViT.prune_intermediate_layers  sM     #7s4;;7G"Qikk.9q=1!!!R(r$   c                 J    | j                  |      }| j                  |      }|S r   )r   r   r(   s     r#   forward_featureszSHViT.forward_features  s$    QKKNr$   
pre_logitsc                     | j                  |      }| j                  |      }| j                  dkD  r,t        j                  || j                  | j
                        }|r|S | j                  |      S )Nr   )ptraining)r   r   r   r4   dropoutr  r   )r!   r%   r  s      r#   forward_headzSHViT.forward_head  sZ    QLLO>>B		!t~~FAq0DIIaL0r$   c                 J    | j                  |      }| j                  |      }|S r   )r  r  r(   s     r#   r)   zSHViT.forward  s'    !!!$a r$   c                      fd |        y )Nc                     | j                         D ]?  \  }}t        |d      r&|j                         }t        | ||        |       8 |       A y )Nr-   )named_childrenhasattrr-   setattr)net
child_namechildfusedfuse_childrens       r#   r  z!SHViT.fuse.<locals>.fuse_children  sQ    %(%7%7%9 )!
E5&)!JJLECU3!%(!%()r$   r   )r!   r  s    @r#   r-   z
SHViT.fuse  s    	) 	dr$   F)T)r   )NFFr   F)r   FT)%r9   r:   r;   r   r<   r   rc   r   r   rr   r	   r=   r    r0   r   ignorer   r   rq   r   r   r   r   r   r   r>   r   r
   r   r  r
  r  r  r)   r?   r-   r@   rA   s   @r#   r   r     s    #$.=0<+7*3*9!*4)+:o:o :o 	:o
 S#s]+:o sC}-:o #sC-(:o c3':o c3':o :o RYY:o BII:ox YY   YYD T#s(^   YY* * YY		  iC ic i 8<$$',( ||(  eCcN34(  	( 
 (  (  !%(  
tELL!5tELL7I)I#JJ	K( X ./$#	3S	>*  	%,, 5<< 
1ell 1 1 1 %,, 
 U]]_
 
r$   
state_dictmodelr&   c                 *    | j                  d|       } | S )Nr!  )get)r   r!  s     r#   checkpoint_filter_fnr$    s    4J: r$   urlrS   c                 8    | dddddt         t        dddd	d
dd|S )Nr   )rz      r'  )ry   ry   g      ?bicubiczpatch_embed.0.czhead.lmitzarXiv:2401.16456zHSHViT: Single-Head Vision Transformer with Memory Efficient Macro Designz https://github.com/ysj9909/SHViT)r%  r   
input_size	pool_sizecrop_pctinterpolationmeanrh   
first_conv
classifierlicense	paper_ids
paper_name
origin_urlr   )r%  rS   s     r#   _cfgr5    s<    4}SYI%.B'x'`8
 
 
r$   ztimm/)	hf_hub_id)rz   r   r   )r6  r*  )zshvit_s1.in1kzshvit_s2.in1kzshvit_s3.in1kzshvit_s4.in1kvariant
pretrainedc                 N    t        t        | |ft        t        dd      d|}|S )N)r   r   rW   T)out_indicesflatten_sequential)pretrained_filter_fnfeature_cfg)r   r   r$  r   )r7  r8  rS   r!  s       r#   _create_shvitr>    s6     w
1Y4H 	E Lr$   c           	      N    t        dddd      }t        dd| it        |fi |S )N)r   r'  i@  rW   ry      )r   0   D   r   r   r   r   r   r   r   r8  )shvit_s1r   r>  r8  rS   
model_argss      r#   rF  rF    5    !TceJY
Yd:>XQW>XYYr$   c           	      N    t        dddd      }t        dd| it        |fi |S )N)r   i4    r@  )r   B   r   rD  rE  r8  )shvit_s2rG  rH  s      r#   rN  rN  $  rJ  r$   c           	      N    t        dddd      }t        dd| it        |fi |S )N)   i`  rL  )rz   rA  rA  )rB  K   r   rD  rE  r8  )shvit_s3rG  rH  s      r#   rR  rR  +  rJ  r$   c           	      N    t        dddd      }t        dd| it        |fi |S )N)r'  iP  rL  )ry         )rB  H   r   rD  rE  r8  )shvit_s4rG  rH  s      r#   rW  rW  2  rJ  r$   )r	  r  )9r   typingr   r   r   r   r   r   r	   r
   r0   torch.nnr<   torch.nn.functional
functionalr4   	timm.datar   r   timm.layersr   r   r   r   r   _builderr   	_featuresr   _manipulater   	_registryr   r   __all__r=   r   r   r,   re   rt   r   r   r   r   r   r   r>   r$  r5  default_cfgsrq   r>  rF  rN  rR  rW  r   r$   r#   <module>rd     sV   F E E     A a a * + ' <)ryy *( (V D299 :")) ,$299 $N 8# #LpBII pfT#u||*;%< RYY SWX[]b]i]iXiSj Bc # $sCx.  %    & *3 D C E  Z Z Z Z Z Z Z Z Z Z Z Z Z Z Z Z Z Z Z Zr$   