
    ^jWR                        d Z ddlZddlmZmZmZmZmZmZm	Z	 ddl
Z
ddlmZ ddlmc mZ ddlmZmZ ddlmZmZmZmZmZmZmZ ddlmZ ddlmZ dd	lm Z  dd
l!m"Z"m#Z# dgZ$ G d dejJ                        Z& G d dejJ                        Z' G d dejJ                        Z( G d dejJ                        Z) G d dejJ                        Z* G d dejJ                        Z+d Z,d%dZ-d&dZ. e# e.d       e.d       e.d       e.d       e.d       e.d       e.d      d      Z/e"d%de+fd       Z0e"d%de+fd       Z1e"d%de+fd        Z2e"d%de+fd!       Z3e"d%de+fd"       Z4e"d%de+fd#       Z5e"d%de+fd$       Z6y)'a   Pyramid Vision Transformer v2

@misc{wang2021pvtv2,
      title={PVTv2: Improved Baselines with Pyramid Vision Transformer},
      author={Wenhai Wang and Enze Xie and Xiang Li and Deng-Ping Fan and Kaitao Song and Ding Liang and
        Tong Lu and Ping Luo and Ling Shao},
      year={2021},
      eprint={2106.13797},
      archivePrefix={arXiv},
      primaryClass={cs.CV}
}

Based on Apache 2.0 licensed code at https://github.com/whai362/PVT

Modifications and timm support by / Copyright 2022, Ross Wightman
    N)CallableListOptionalTupleUnionTypeAnyIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPathcalculate_drop_path_rates	to_2tuple	to_ntupletrunc_normal_	LayerNormuse_fused_attn   )build_model_with_cfg)feature_take_indices)
checkpoint)register_modelgenerate_default_cfgsPyramidVisionTransformerV2c                        e Zd Zddej                  ddddfdedee   dee   deej                     de	d	e
f fd
Zdee   fdZ xZS )MlpWithDepthwiseConvN        Fin_featureshidden_featuresout_features	act_layerdrop
extra_reluc	                    t         
|           ||d}	|xs |}|xs |}t        j                  ||fi |	| _        |rt        j
                         nt        j                         | _        t        j                  ||dddfd|d|	| _	         |       | _
        t        j                  ||fi |	| _        t        j                  |      | _        y )Ndevicedtype   r   T)biasgroups)super__init__nnLinearfc1ReLUIdentityreluConv2ddwconvactfc2Dropoutr"   )selfr   r   r    r!   r"   r#   r&   r'   dd	__class__s             ]/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/pvt_v2.pyr,   zMlpWithDepthwiseConv.__init__$   s     	/#2{)8[99[/@R@!+BGGI	ii!QsPT]lsprs;99_lAbAJJt$	    	feat_sizec                    | j                  |      }|j                  \  }}}|j                  dd      j                  |||d   |d         }| j	                  |      }| j                  |      }|j                  d      j                  dd      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }|S )Nr      r   )
r/   shape	transposeviewr2   r4   flattenr5   r"   r6   )r8   xr=   BNCs         r;   forwardzMlpWithDepthwiseConv.forward:   s    HHQK''1aKK1""1a1y|DIIaLKKNIIaL""1a(HHQKIIaLHHQKIIaLr<   )__name__
__module____qualname__r-   GELUintr   r   Modulefloatboolr,   r   rH   __classcell__r:   s   @r;   r   r   #   s}     .2*.)+$%% &c]% #3-	%
 BII% % %,DI r<   r   c                        e Zd ZU ej                  j
                  e   ed<   	 	 	 	 	 	 	 	 ddededededede	de	f fd	Z
d
ee   fdZ xZS )	Attention
fused_attndim	num_headssr_ratiolinear_attnqkv_bias	attn_drop	proj_dropc
                    t         |           ||	d}
||z  dk(  sJ d| d| d       || _        || _        ||z  | _        | j                  dz  | _        t               | _        t        j                  ||fd|i|
| _
        t        j                  ||dz  fd|i|
| _        t        j                  |      | _        t        j                  ||fi |
| _        t        j                  |      | _        |sdd | _        |d	kD  rBt        j"                  ||f||d
|
| _        t        j&                  |fi |
| _        d | _        y d | _        d | _        d | _        y t        j,                  d      | _        t        j"                  ||fd	d	d
|
| _        t        j&                  |fi |
| _        t        j.                         | _        y )Nr%   r   zdim z  should be divided by num_heads .g      r)   r?   r   )kernel_sizestride   )r+   r,   rV   rW   head_dimscaler   rU   r-   r.   qkvr7   r[   projr\   poolr3   srr   normr5   AdaptiveAvgPool2drL   )r8   rV   rW   rX   rY   rZ   r[   r\   r&   r'   r9   r:   s              r;   r,   zAttention.__init__K   s    	/Y!#]tC50PQZP[[\%]]#"y(]]d*
(*39(9b9))Cq>x>2>I.IIc3-"-	I.DI!|))CZ(8ZWYZLL33	 DH  	DH,,Q/DIiiSHaHRHDGS/B/DIwwyDHr<   r=   c                 d   |j                   \  }}}|\  }}| j                  |      j                  ||| j                  d      j	                  dddd      }| j
                  |j	                  ddd      j                  ||||      }| j                  | j                  |            j                  ||d      j	                  ddd      }| j                  |      }| j                  |      }| j                  |      j                  |dd| j                  | j                        j	                  ddddd      }	n| j                  |j	                  ddd      j                  ||||      }| j                  |      j                  ||d      j	                  ddd      }| j                  |      }| j                  |      j                  |dd| j                  | j                        j	                  ddddd      }	nK| j                  |      j                  |dd| j                  | j                        j	                  ddddd      }	|	j                  d      \  }
}| j                  r<t        j                  ||
|| j                  r| j                   j"                  nd      }nL|| j$                  z  }||
j'                  d	d      z  }|j)                  d
      }| j!                  |      }||z  }|j'                  dd      j                  |||      }| j+                  |      }| j-                  |      }|S )Nr   r?   r   r(      r   )	dropout_prV   )r@   rd   reshaperW   permuterg   rh   ri   r5   re   rb   unbindrU   Fscaled_dot_product_attentiontrainingr[   prc   rA   softmaxrf   r\   )r8   rD   r=   rE   rF   rG   HWrd   re   kvattns                r;   rH   zAttention.forwardv   s   ''1a1FF1IaDNNB7??1aK99 		!Q"**1aA6A		!%--aB7??1aHA		!AA##Ar1dnndmmLTTUVXY[\^_abcBww"IIaA&..q!Q:GGAJ&&q!R088AqAIIaLWWQZ''2q$..$--PXXYZ\]_`bcefgWWQZ''2q$..$--PXXYZ\]_`bcefgyy|1??..q!QVZVcVc$..BRBRiklADJJAq{{2r**D<<B<'D>>$'DqAKK1%%aA.IIaLNN1r<   )   r   FTr   r   NN)rI   rJ   rK   torchjitFinalrP   __annotations__rM   rO   r,   r   rH   rQ   rR   s   @r;   rT   rT   H   s    		%%
  %!!!)!)! )! 	)!
 )! )! )! )!V!DI !r<   rT   c                        e Zd Zdddddddej                  eddfdededed	ed
ededededede	ej                     de	ej                     f fdZdee   fdZ xZS )Block      @r   Fr   NrV   rW   	mlp_ratiorX   rY   rZ   r\   r[   	drop_pathr!   
norm_layerc           
         t         |           ||d} ||fi || _        t        |f||||||d|| _        |	dkD  rt        |	      nt        j                         | _         ||fi || _	        t        d|t        ||z        |
||d|| _        |	dkD  rt        |	      | _        y t        j                         | _        y )Nr%   )rW   rX   rY   rZ   r[   r\   r   )r   r   r!   r"   r#    )r+   r,   norm1rT   r}   r   r-   r1   
drop_path1norm2r   rM   mlp
drop_path2)r8   rV   rW   r   rX   rY   rZ   r\   r[   r   r!   r   r&   r'   r9   r:   s                  r;   r,   zBlock.__init__   s      	/*r*
	
#	
 	
	 2;R(9-R[[]*r*
' 
i0"
 
 2;R(9-R[[]r<   r=   c                     || j                  | j                  | j                  |      |            z   }|| j                  | j	                  | j                  |      |            z   }|S N)r   r}   r   r   r   r   )r8   rD   r=   s      r;   rH   zBlock.forward   sS    		$**Q- CDDA	 BCCr<   )rI   rJ   rK   r-   rL   r   rM   rO   rP   r   rN   r,   r   rH   rQ   rR   s   @r;   r   r      s      " %"!!!)+*3(S(S (S 	(S
 (S (S (S (S (S (S BII(S RYY(STDI r<   r   c            	       \     e Zd ZdZ	 	 	 	 	 	 ddeeeeef   f   dededef fdZd Z xZ	S )	OverlapPatchEmbedz Image to Patch Embedding
    
patch_sizer`   in_chans	embed_dimc                    t         |           ||d}t        |      }t        |      |kD  sJ d       || _        t        j                  |||f||d   dz  |d   dz  fd|| _        t        j                  |fi || _	        y )Nr%   z!Set larger patch_size than strider   r?   r   )r`   padding)
r+   r,   r   maxr   r-   r3   rf   r   ri   )	r8   r   r`   r   r   r&   r'   r9   r:   s	           r;   r,   zOverlapPatchEmbed.__init__   s     	/z*
:'L)LL'$IIiSJqMQ$6
18J#KSOQS	 LL1b1	r<   c                 r    | j                  |      }|j                  dddd      }| j                  |      }|S )Nr   r?   r(   r   )rf   rr   ri   r8   rD   s     r;   rH   zOverlapPatchEmbed.forward   s5    IIaLIIaAq!IIaLr<   )ra   rm   r(   i   NN)
rI   rJ   rK   __doc__r   rM   r   r,   rH   rQ   rR   s   @r;   r   r      s^     78 2c5c?232 2 	2
 2&r<   r   c                        e Zd Zdddddddddeddfded	ed
edededededededededeee   ef   de	f fdZ
d Z xZS )PyramidVisionTransformerStageTr~   r   Fr   r   NrV   dim_outdepth
downsamplerW   rX   rY   r   rZ   r\   r[   r   r   c                 `   t         |           ||d}d| _        |rt        ddd||d|| _        n||k(  sJ d | _        t        j                  t        |      D cg c]-  }t        d||||||	|
|t        |t              r||   n||d
|/ c}      | _         ||fi || _        y c c}w )Nr%   Fr(   r?   r   r`   r   r   )
rV   rW   rX   rY   r   rZ   r\   r[   r   r   r   )r+   r,   grad_checkpointingr   r   r-   
ModuleListranger   
isinstancelistblocksri   )r8   rV   r   r   r   rW   rX   rY   r   rZ   r\   r[   r   r   r&   r'   r9   ir:   s                     r;   r,   z&PyramidVisionTransformerStage.__init__   s    $ 	/"'/ !	
 DO '>!>"DOmm <%!  &+ &
#&0D&Aily!&
 &
 %! " w-"-	%!s   2B+c                    | j                   | j                  |      }|j                  \  }}}}||f}|j                  |d|      }| j                  D ]C  }| j                  r,t
        j                  j                         st        |||      }; |||      }E | j                  |      }|j                  ||d   |d   d      j                  dddd      j                         }|S )Nrl   r   r   r(   r?   )r   r@   rq   r   r   r   r   is_scriptingr   ri   rr   
contiguous)r8   rD   rE   ry   rz   rG   r=   blks           r;   rH   z%PyramidVisionTransformerStage.forward  s    ??&"AWW
1aF	IIaQ;; 	&C&&uyy/E/E/GsAy19%		&
 IIaLIIa1y|R8@@Aq!LWWYr<   )rI   rJ   rK   r   rM   rP   rO   r   r   r   r,   rH   rQ   rR   s   @r;   r   r      s      $ %"!!!36#,!0.0. 0. 	0.
 0. 0. 0. 0. 0. 0. 0. 0. T%[%/00. !0.dr<   r   c                        e Zd Zddddddddd	d
ddddeddfdedededeedf   deedf   deedf   deedf   deedf   dedededededede	e
j                     f fdZd Zd Zej                   j"                  d         Zej                   j"                  d4d!       Zej                   j"                  d5d"       Zd#e
j                  fd$Zd6dedee   fd%Z	 	 	 	 	 d7d&ej0                  d'eeeee   f      d(ed)ed*ed+ed#eeej0                     eej0                  eej0                     f   f   fd,Z	 	 	 d8d'eeee   f   d-ed.efd/Zd0 Zd4d1efd2Zd3 Z xZ S )9r   r(     avgr(   rm      r(   )@            )r   r?   rm   r~   )r~   rm   r?   r   )       @r   r   r   TFr   Nr   num_classesglobal_pooldepths.
embed_dimsrW   	sr_ratios
mlp_ratiosrZ   linear	drop_rateproj_drop_rateattn_drop_ratedrop_path_rater   c                    t         |           ||d}|| _        || _        |dv sJ || _        || _        t        |      } t        |      |      } t        |      |      } t        |      |      }t        |      |k(  sJ g | _        t        ddd||d   d|| _
        t        ||d      }d}|d   }g }t        |      D ]l  }|t        d|||   ||   |dkD  ||   ||   ||   |
|	||||   |d	|gz  }||   }|||   z  }| xj                  t        |dd
|z  z  d|       gz  c_        n t        j                   | | _        |d   x| _        | _        t        j(                  |      | _        |dkD  rt        j,                  |d   |fi |nt        j.                         | _        | j3                  | j4                         y )Nr%   r    ra   rm   r   r   T)	stagewise)rV   r   r   r   rW   rX   r   rY   rZ   r\   r[   r   r   r?   stages.)num_chs	reductionmodulerl   r   )r+   r,   r   r   r   r   lenr   feature_infor   patch_embedr   r   r   dictr-   
Sequentialstagesnum_featureshead_hidden_sizer7   	head_dropr.   r1   headapply_init_weights)r8   r   r   r   r   r   rW   r   r   rZ   r   r   r   r   r   r   r&   r'   r9   
num_stagesdprcurprev_dimr   r   r:   s                            r;   r,   z#PyramidVisionTransformerV2.__init__0  s*   ( 	/& k)))&[
*Yz*:6
)Ij))4	)Ij))4	::---, 
 m	

 
 ($Oa=z" 	dA4 "1Qiq5#A,"1$Q-"!((a&%   F  "!}H6!9C$x1q!t8V]^_]`Ta"b!cc'	d( mmV, 5?rNBD1I.DORSOBIIjnk@R@Y[YdYdYf	

4%%&r<   c                    t        |t        j                        rjt        |j                  d       t        |t        j                        r8|j
                  +t        j                  j                  |j
                  d       y y y t        |t        j                        r|j                  d   |j                  d   z  |j                  z  }||j                  z  }|j                  j                  j                  dt        j                  d|z               |j
                  %|j
                  j                  j!                          y y y )Ng{Gz?)stdr   r   g       @)r   r-   r.   r   weightr)   init	constant_r3   r_   out_channelsr*   datanormal_mathsqrtzero_)r8   mfan_outs      r;   r   z(PyramidVisionTransformerV2._init_weights{  s    a#!((,!RYY'AFF,>!!!&&!, -?'299%mmA&q)99ANNJG GHHMM!!!TYYsW}%=>vv!!!# "	 &r<   c                 &    d| j                   _        y )NF)r   requires_gradr8   s    r;   freeze_patch_embz+PyramidVisionTransformerV2.freeze_patch_emb  s    ).&r<   c                     i S r   r   r   s    r;   no_weight_decayz*PyramidVisionTransformerV2.no_weight_decay  s    	r<   c                      t        dd      }|S )Nz^patch_embedz^stages\.(\d+))stemr   )r   )r8   coarsematchers      r;   group_matcherz(PyramidVisionTransformerV2.group_matcher  s     $
 r<   c                 4    | j                   D ]	  }||_         y r   )r   r   )r8   enabless      r;   set_grad_checkpointingz1PyramidVisionTransformerV2.set_grad_checkpointing  s     	*A#)A 	*r<   returnc                     | j                   S r   )r   r   s    r;   get_classifierz)PyramidVisionTransformerV2.get_classifier  s    yyr<   c                    || _         ||dv sJ || _        t        | j                  d      r | j                  j                  j
                  nd }t        | j                  d      r | j                  j                  j                  nd }|dkD  r)t        j                  | j                  |||      | _        y t        j                         | _        y )Nr   r   r   r%   )r   r   hasattrr   r   r&   r'   r-   r.   r   r1   )r8   r   r   r&   r'   s        r;   reset_classifierz+PyramidVisionTransformerV2.reset_classifier  s    &"+---*D,3DIIx,H!!((d*1$))X*F		  &&D]hkl]lBIId//VSXY	rtr}r}r	r<   rD   indicesri   
stop_early
output_fmtintermediates_onlyc                 r   |dv sJ d       g }t        t        | j                        |      \  }}	| j                  |      }t        j
                  j                         s|s| j                  }
n| j                  d|	dz    }
t        |
      D ]#  \  }} ||      }||v s|j                  |       % |r|S ||fS )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWzOutput shape must be NCHW.Nr   )	r   r   r   r   r   r   r   	enumerateappend)r8   rD   r   ri   r   r   r  intermediatestake_indices	max_indexr   feat_idxstages                r;   forward_intermediatesz0PyramidVisionTransformerV2.forward_intermediates  s    * Y&D(DD&"6s4;;7G"Qi Q99!!#:[[F[[)a-0F(0 	(OHeaA<'$$Q'	(
   -r<   
prune_norm
prune_headc                     t        t        | j                        |      \  }}| j                  d|dz    | _        |r| j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r   )r   r   r   r   )r8   r   r  r  r  r  s         r;   prune_intermediate_layersz4PyramidVisionTransformerV2.prune_intermediate_layers  sM     #7s4;;7G"Qikk.9q=1!!!R(r<   c                 J    | j                  |      }| j                  |      }|S r   )r   r   r   s     r;   forward_featuresz+PyramidVisionTransformerV2.forward_features  s$    QKKNr<   
pre_logitsc                     | j                   r|j                  d      }| j                  |      }|r|S | j                  |      S )N)rl   ro   rp   )r   meanr   r   )r8   rD   r  s      r;   forward_headz'PyramidVisionTransformerV2.forward_head  s>    8$ANN1q0DIIaL0r<   c                 J    | j                  |      }| j                  |      }|S r   )r  r  r   s     r;   rH   z"PyramidVisionTransformerV2.forward  s'    !!!$a r<   F)Tr   )NFFr  F)r   FT)!rI   rJ   rK   r   rM   strr   rO   rP   r   r-   rN   r,   r   r   r   r   ignorer   r   r   r   r   r   Tensorr   r   r  r  r  r  rH   rQ   rR   s   @r;   r   r   /  s    #$&2*=)5)5,<! !$&$&$&*3%I'I' I' 	I'
 #s(OI' c3hI' S#XI' S#XI' eSj)I' I' I' I' "I' "I' "I'  RYY!I'V
$/ YY  YY  YY* *		 @C @hsm @ 8<$$',( ||(  eCcN34(  	( 
 (  (  !%(  
tELL!5tELL7I)I#JJ	K( X ./$#	3S	>*  	
1$ 1r<   c                 ~   d| v r| S i }ddl }| j                         D ]  \  }}|j                  d      rH|j                  dd      }|j                  dd      }|j                  dd	      }|j                  d
d      }|j                  dd      }|j	                  dd |      }|j	                  dd |      }|||<    |S )z$ Remap original checkpoints -> timm zpatch_embed.proj.weightr   Nr   patch_embed1patch_embed2zstages.1.downsamplepatch_embed3zstages.2.downsamplepatch_embed4zstages.3.downsamplezdwconv.dwconvr4   zblock(\d+).(\d+)c                 f    dt        | j                  d            dz
   d| j                  d       S )Nr   r   z.blocks.r?   rM   grouprD   s    r;   <lambda>z&checkpoint_filter_fn.<locals>.<lambda>  s4    GC
Oa<O;PPXYZY`Y`abYcXd2e r<   z
^norm(\d+)c                 D    dt        | j                  d            dz
   dS )Nr   r   z.normr!  r#  s    r;   r$  z&checkpoint_filter_fn.<locals>.<lambda>  s!    gc!''!*o6I5J%,P r<   )reitems
startswithreplacesub)
state_dictmodelout_dictr&  r{   r|   s         r;   checkpoint_filter_fnr.    s     J.H  " 	1<<&		.-8A		.*?@A		.*?@A		.*?@AIIox0FF&(eghiFF="PRST	 Or<   c                     t        t        d            }|j                  d|      }t        t        | |ft
        t        d|      d|}|S )Nrm   out_indicesT)flatten_sequentialr0  )pretrained_filter_fnfeature_cfg)tupler   popr   r   r.  r   )variant
pretrainedkwargsdefault_out_indicesr0  r,  s         r;   _create_pvt2r:    sY    a/**],?@K " 2DkJ E Lr<   c                 4    | dddddt         t        dddd	d
|S )Nr   )r(      r<  )ra   ra   g?bicubiczpatch_embed.projr   Fz
apache-2.0)urlr   
input_size	pool_sizecrop_pctinterpolationr  r   
first_conv
classifierfixed_input_sizelicenser
   )r>  r8  s     r;   _cfgrG    s6    4}SY)%.B(TY  r<   ztimm/)	hf_hub_id)zpvt_v2_b0.in1kzpvt_v2_b1.in1kzpvt_v2_b2.in1kzpvt_v2_b3.in1kzpvt_v2_b4.in1kzpvt_v2_b5.in1kzpvt_v2_b2_li.in1kr   c           	      L    t        ddd      }t        dd| it        |fi |S )Nr?   r?   r?   r?   )    r      r   r   r?      r~   r   r   rW   r7  )	pvt_v2_b0r   r:  r7  r8  
model_argss      r;   rP  rP  (  s0    \6HT`aJY
Yd:>XQW>XYYr<   c           	      L    t        ddd      }t        dd| it        |fi |S )NrJ  r   r   i@  r   rM  rO  r7  )	pvt_v2_b1rQ  rR  s      r;   rV  rV  .  0    \6IUabJY
Yd:>XQW>XYYr<   c           	      L    t        ddd      }t        dd| it        |fi |S )Nr   rU  rM  rO  r7  )	pvt_v2_b2rQ  rR  s      r;   rY  rY  4  rW  r<   c           	      L    t        ddd      }t        dd| it        |fi |S )N)r(   rm      r(   rU  rM  rO  r7  )	pvt_v2_b3rQ  rR  s      r;   r\  r\  :  0    ]7JVbcJY
Yd:>XQW>XYYr<   c           	      L    t        ddd      }t        dd| it        |fi |S )N)r(   r~      r(   rU  rM  rO  r7  )	pvt_v2_b4rQ  rR  s      r;   r`  r`  @  r]  r<   c           	      N    t        dddd      }t        dd| it        |fi |S )N)r(   r   (   r(   rU  rM  )rm   rm   rm   rm   )r   r   rW   r   r7  )	pvt_v2_b5rQ  rR  s      r;   rc  rc  F  s6    )<amoJY
Yd:>XQW>XYYr<   c           	      N    t        dddd      }t        dd| it        |fi |S )Nr   rU  rM  T)r   r   rW   r   r7  )pvt_v2_b2_lirQ  rR  s      r;   re  re  M  s6    (;|\`bJ\:\jA[TZA[\\r<   r  )r   )7r   r   typingr   r   r   r   r   r   r	   r   torch.nnr-   torch.nn.functional
functionalrt   	timm.datar   r   timm.layersr   r   r   r   r   r   r   _builderr   	_featuresr   _manipulater   	_registryr   r   __all__rN   r   rT   r   r   r   r   r.  r:  rG  default_cfgsrP  rV  rY  r\  r`  rc  re  r   r<   r;   <module>rr     s  "  D D D     A { { { * + # <'
("299 "JO		 Od0BII 0f		 :BBII BJ~ ~B( %W-W-W-W-W-W-0&  Z-G Z Z
 Z-G Z Z
 Z-G Z Z
 Z-G Z Z
 Z-G Z Z
 Z-G Z Z ]0J ] ]r<   