
    ^jX                        d Z ddlZddlmZmZmZmZmZmZ ddl	Z	ddl
mZ ddlmZmZ ddlmZmZmZmZmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZ dgZ  G d dejB                        Z" G d dejB                        Z# G d dejB                        Z$ G d dejB                        Z%ddZ& e e&d       e&d       e&d      d      Z'd Z(ddZ)edde%fd       Z*edde%fd       Z+edde%fd       Z,y)a   Transformer in Transformer (TNT) in PyTorch

A PyTorch implement of TNT as described in
'Transformer in Transformer' - https://arxiv.org/abs/2103.00112

The official mindspore code is released and available at
https://gitee.com/mindspore/mindspore/tree/master/model_zoo/research/cv/TNT

The official pytorch code is released and available at
https://github.com/huawei-noah/Efficient-AI-Backbones/tree/master/tnt_pytorch
    N)ListOptionalTupleUnionTypeAnyIMAGENET_INCEPTION_MEANIMAGENET_INCEPTION_STD)MlpDropPathcalculate_drop_path_ratestrunc_normal__assert	to_2tupleresample_abs_pos_embed   )build_model_with_cfg)feature_take_indices)
checkpoint)generate_default_cfgsregister_modelTNTc                   P     e Zd ZdZ	 	 	 	 	 	 d
dedededededef fdZd	 Z xZ	S )	Attentionz Multi-Head Attention
    dim
hidden_dim	num_headsqkv_bias	attn_drop	proj_dropc	                    ||d}	t         |           || _        || _        ||z  }
|
| _        |
dz  | _        t        j                  ||dz  fd|i|	| _        t        j                  ||fd|i|	| _	        t        j                  |d      | _        t        j                  ||fi |	| _        t        j                  |d      | _        y )Ndevicedtypeg         biasT)inplace)super__init__r   r   head_dimscalennLinearqkvDropoutr    projr!   )selfr   r   r   r   r    r!   r$   r%   ddr+   	__class__s              Z/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/tnt.pyr*   zAttention.__init__    s     /$"* %
))CaEhE"E39(9b9It<IIc3-"-	It<    c                 |   |j                   \  }}}| j                  |      j                  ||d| j                  | j                        j                  ddddd      }|j                  d      \  }}| j                  |      j                  ||| j                  d      j                  dddd      }||j                  dd      z  | j                  z  }	|	j                  d      }	| j                  |	      }	|	|z  j                  dd      j                  ||d      }| j                  |      }| j                  |      }|S )	Nr&   r      r      r   )shaper/   reshaper   r+   permuteunbindr0   	transposer,   softmaxr    r2   r!   )
r3   xBNCr/   qkr0   attns
             r6   forwardzAttention.forward9   s   ''1aWWQZ1aGOOPQSTVWYZ\]^yy|1FF1IaDNNB7??1aKAKKB''4::5|||#~~d#AX  A&..q!R8IIaLNN1r7   )   F        rM   NN)
__name__
__module____qualname____doc__intboolfloatr*   rK   __classcell__r5   s   @r6   r   r      s`     "!!== = 	=
 = = =2r7   r   c                        e Zd ZdZdddddddej
                  ej                  dddfded	ed
edededede	dededede
ej                     de
ej                     de	f fdZd Z xZS )Blockz TNT Block
    r:            @FrM   Nr   dim_out	num_pixelnum_heads_innum_heads_out	mlp_ratior   r!   r    	drop_path	act_layer
norm_layerlegacyc           	         ||d}t         |            ||fi || _        t        ||f|||	|d|| _         ||fi || _        t        d	|t        |dz        |||d|| _        || _	        | j                  r7 ||fi || _
        t        j                  ||z  |fddi|| _        d | _        n@ |||z  fi || _
        t        j                  ||z  |fddi|| _         ||fi || _         ||fi || _        t        ||f|||	|d|| _        |
dkD  rt#        |
      nt        j$                         | _         ||fi || _        t        d	|t        ||z        |||d|| _        y )
Nr#   )r   r   r    r!   r:   )in_featureshidden_featuresout_featuresra   dropr'   TFrM    )r)   r*   norm_inr   attn_innorm_mlp_inr   rR   mlp_inrc   
norm1_projr-   r.   r2   
norm2_projnorm_outattn_outr   Identityr`   norm_mlpmlp)r3   r   r[   r\   r]   r^   r_   r   r!   r    r`   ra   rb   rc   r$   r%   r4   r5   s                    r6   r*   zBlock.__init__M   s   $ /!#,, 
 #
 
 &c0R0 
aL
 
 ;;(33DO		#	/7LLLDI"DO(y?B?DO		#	/7MM"MDI(7B7DO #71b1!
 $
 
 1:B),BKKM"71b1 
) 34 
 
r7   c                 z   || j                  | j                  | j                  |                  z   }|| j                  | j                  | j	                  |                  z   }|j                         \  }}}| j                  at        j                  |d d ddf   |d d dd f   | j                  | j                  |      j                  ||dz
  d            z   gd      }not        j                  |d d ddf   |d d dd f   | j                  | j                  | j                  |j                  ||dz
  d                        z   gd      }|| j                  | j                  | j                  |                  z   }|| j                  | j                  | j                  |                  z   }||fS )Nr   r   r;   r=   )r`   rk   rj   rm   rl   sizero   torchcatr2   rn   r?   rq   rp   rt   rs   )r3   pixel_embedpatch_embedrE   rF   rG   s         r6   rK   zBlock.forward   s   !DNN4<<[@Y3Z$[[!DNN4;;t?O?OP[?\3]$^^""$1a??"))AqsF#AqrE"TYYt{/K/S/STUWX[\W\^`/a%bb% K
  ))AqsF#AqrE"T__TYYt{ObObcdfgjkfkmoOp?q5r%ss% K "DNN4=={A[3\$]]!DNN488DMM+<V3W$XXK''r7   )rN   rO   rP   rQ   r-   GELU	LayerNormrR   rT   rS   r   Moduler*   rK   rU   rV   s   @r6   rX   rX   I   s     !"!#!"!!!)+*,,, !H
H
 H
 	H

 H
 H
 H
 H
 H
 H
 H
 BIIH
 RYYH
 H
T(r7   rX   c                       e Zd ZdZ	 	 	 	 	 	 	 	 ddeeeeef   f   deeeeef   f   dedededef fdZdd	eeeef   ef   fd
Z	deeef   d	eeef   fdZ
dej                  dej                  d	ej                  fdZ xZS )
PixelEmbedz Image to Pixel Embedding
    img_size
patch_sizein_chansin_dimstriderc   c	                 R   ||d}	t         |           t        |      }t        |      }|d   |d   z  |d   |d   z  f| _        | j                  d   | j                  d   z  }
|| _        || _        || _        |
| _        || _        |D cg c]  }t        j                  ||z         }}|| _        t        j                  || j                  fdd|d|	| _        | j                  rt        j                  ||      | _        y t        j                  ||      | _        y c c}w )Nr#   r   r      r9   )kernel_sizepaddingr   )r   r   )r)   r*   r   	grid_sizer   r   rc   num_patchesr   mathceilnew_patch_sizer-   Conv2dr2   Unfoldunfold)r3   r   r   r   r   r   rc   r$   r%   r4   r   psr   r5   s                r6   r*   zPixelEmbed.__init__   s    /X&z*
"1+A6zRS}8TU~~a(T^^A->? $&;EFR$))BK0FF,IIhcAV\c`bc	;;))~VDK))
:NDK Gs   D$returnc                 H    |rt        | j                        S | j                  S N)maxr   )r3   	as_scalars     r6   
feat_ratiozPixelEmbed.feat_ratio   s    t''??"r7   c                 V    |d   | j                   d   z  |d   | j                   d   z  fS )Nr   r   )r   )r3   r   s     r6   dynamic_feat_sizezPixelEmbed.dynamic_feat_size   s2    {dooa00(1+QRAS2SSSr7   rD   	pixel_posc                    |j                   \  }}}}t        || j                  d   k(  d| d| d| j                  d    d| j                  d    d	       t        || j                  d   k(  d| d| d| j                  d    d| j                  d    d	       | j                  rx| j	                  |      }| j                  |      }|j                  dd      j                  || j                  z  | j                  | j                  d   | j                  d         }nm| j                  |      }|j                  dd      j                  || j                  z  || j                  d   | j                  d         }| j	                  |      }||z   }|j                  || j                  z  | j                  d      j                  dd      }|S )	Nr   zInput image size (*z) doesn't match model (r   z).r&   r;   )r>   r   r   rc   r2   r   rB   r?   r   r   r   r   )r3   rD   r   rE   rG   HWs          r6   rK   zPixelEmbed.forward   s   WW
1aq!! 1QC'>t}}Q?O>PPQRVR_R_`aRbQccef	h 	q!! 1QC'>t}}Q?O>PPQRVR_R_`aRbQccef	h ;;		!AAAAq!))D$$$dkk43F3Fq3I4K^K^_`KacA AAAq!))!d.>.>*>4??STCUW[WfWfghWijA		!A	MIIa$***DKK<FFq!Lr7   )      r9   0   r:   FNNT)rN   rO   rP   rQ   r   rR   r   rS   r*   r   r   rw   TensorrK   rU   rV   s   @r6   r   r      s    
 5868 OCsCx01O c5c?23O 	O
 O O O@#E%S/32F,G #T%S/ TeCHo T %,, 5<< r7   r   c            )           e Zd ZdZdddddddd	d
d	dddddddej
                  d
dddfdeeeeef   f   deeeeef   f   dedede	dededededede
dede
de
de
de
de
d eej                     d!ed"ef( fd#Zd$ Zej"                  j$                  d%        Zej"                  j$                  d:d&       Zej"                  j$                  d;d'       Zej"                  j$                  d(ej                  fd)       Zd<dedee	   fd*Z	 	 	 	 	 	 d=d+ej2                  d,eeeee   f      d-ed.ed/ed0e	d1ed(eeej2                     eej2                  eej2                     f   f   fd2Z	 	 	 d>d,eeee   f   d3ed4efd5Zd6 Zd:d7efd8Zd9 Z xZ S )?r   zC Transformer in Transformer - https://arxiv.org/abs/2103.00112
    r   r   r9     tokeni   r   rY   r:   rZ   FrM   Nr   r   r   num_classesglobal_pool	embed_dim	inner_dimdepthnum_heads_innernum_heads_outerr_   r   	drop_ratepos_drop_rateproj_drop_rateattn_drop_ratedrop_path_raterb   first_striderc   c                 >   t         |           ||d}|dv sJ || _        || _        || _        |x| _        x| _        | _        d| _        d| _	        t        d||||||d|| _        | j                  j                  }t        | j                  d      r| j                  j                         n|}|| _        | j                  j                  }|d   |d   z  } |||z  fi || _        t#        j$                  ||z  |fi || _         ||fi || _        t#        j*                  t-        j.                  dd|fi |      | _        t#        j*                  t-        j.                  d|dz   |fi |      | _        t#        j*                  t-        j.                  d||d   |d   fi |      | _        t#        j6                  |      | _        t;        ||      }g }t=        |      D ],  }|j?                  tA        d||||	|
||||||   ||d	|       . t#        jB                  |      | _"        t=        |      D cg c]  }tG        d
| ||       c}| _$         ||fi || _%        t#        j6                  |      | _&        |dkD  rt#        j$                  ||fi |nt#        jN                         | _(        tS        | j0                  d       tS        | j2                  d       tS        | j4                  d       | jU                  | jV                         y c c}w )Nr#    r   avgr   F)r   r   r   r   r   rc   r   r   )p)r   r[   r\   r]   r^   r_   r   r!   r    r`   rb   rc   zblocks.)modulenum_chs	reduction{Gz?stdri   ),r)   r*   r   r   r   num_featureshead_hidden_sizer   num_prefix_tokensgrad_checkpointingr   ry   r   hasattrr   r   rn   r-   r.   r2   ro   	Parameterrw   zeros	cls_token	patch_posr   r1   pos_dropr   rangeappendrX   
ModuleListblocksdictfeature_infonorm	head_droprr   headr   apply_init_weights) r3   r   r   r   r   r   r   r   r   r   r   r_   r   r   r   r   r   r   rb   r   rc   r$   r%   r4   r   rr   r\   dprr   ir5   s                                   r6   r*   zTNT.__init__   s   2 	/2222& &ENNND1DN!""'% 
!
 
 &&22-4T5E5E|-TD'')Zd&))88"1%q(99	$Y%:AbAIIi)3YE"E	$Y5"5ekk!Q	&HR&HIekk![1_i&VSU&VWekk!Yq@QSabcSd&khj&kl

]3'>u 	AMM% !#,-#!((a&%  	  mmF+PUV[P\^KLD'!yAF^ y/B/	I.?JQBIIi;;TVT_T_Ta	dnn#.dnn#.dnn#.

4%%&^s   <Lc                    t        |t        j                        rjt        |j                  d       t        |t        j                        r8|j
                  +t        j                  j                  |j
                  d       y y y t        |t        j                        rUt        j                  j                  |j
                  d       t        j                  j                  |j                  d       y y )Nr   r   r   g      ?)	
isinstancer-   r.   r   weightr'   init	constant_r|   )r3   ms     r6   r   zTNT._init_weightsM  s    a#!((,!RYY'AFF,>!!!&&!, -?'2<<(GGaffa(GGahh, )r7   c                 
    h dS )N>   r   r   r   ri   r3   s    r6   no_weight_decayzTNT.no_weight_decayV  s    66r7   c                 $    t        dddg      }|S )Nz=^cls_token|patch_pos|pixel_pos|pixel_embed|norm[12]_proj|proj)z^blocks\.(\d+)N)z^norm)i )stemr   )r   )r3   coarsematchers      r6   group_matcherzTNT.group_matcherZ  s!    Q)$
 r7   c                     || _         y r   )r   )r3   enables     r6   set_grad_checkpointingzTNT.set_grad_checkpointinge  s
    "(r7   r   c                     | j                   S r   )r   r   s    r6   get_classifierzTNT.get_classifieri  s    yyr7   c                    || _         ||dv sJ || _        t        | j                  d      r | j                  j                  j
                  nd }t        | j                  d      r | j                  j                  j                  nd }|dkD  r)t        j                  | j                  |||      | _        y t        j                         | _        y )Nr   r   r   r#   )r   r   r   r   r   r$   r%   r-   r.   r   rr   )r3   r   r   r$   r%   s        r6   reset_classifierzTNT.reset_classifierm  s    &""6666*D,3DIIx,H!!((d*1$))X*F		  &&DZehiZiBIIdnnk&PUV	oqozozo|	r7   rD   indicesreturn_prefix_tokensr   
stop_early
output_fmtintermediates_onlyc                    |dv sJ d       |dk(  }g }	t        t        | j                        |      \  }
}|j                  \  }}}}| j	                  || j
                        }| j                  | j                  | j                  |j                  || j                  d                        }t        j                  | j                  j                  |dd      |fd      }|| j                  z   }| j!                  |      }t        j"                  j%                         s|s| j                  }n| j                  d|dz    }t'        |      D ]u  \  }}| j(                  r/t        j"                  j%                         st+        |||      \  }}n |||      \  }}||
v sR|	j-                  |r| j/                  |      n|       w | j0                  rD|	D cg c]  }|ddd| j0                  f    }}|	D cg c]  }|dd| j0                  df    }	}|ra| j                  j3                  ||f      \  }}|	D cg c]6  }|j                  |||d      j5                  dd	dd
      j7                         8 }	}t        j"                  j%                         s|rt9        t;        |	            }	|r|	S | j/                  |      }||	fS c c}w c c}w c c}w )a<   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if an int, if is a sequence, select by matching indices
            return_prefix_tokens: Return both prefix and spatial intermediate tokens
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWNLCz)Output format must be one of NCHW or NLC.r   r;   r   r=   Nr   r9   r&   )r   lenr   r>   ry   r   ro   r2   rn   r?   r   rw   rx   r   expandr   r   jitis_scripting	enumerater   r   r   r   r   r   r@   
contiguouslistzip)r3   rD   r   r   r   r   r   r   r?   intermediatestake_indices	max_indexrE   _heightwidthry   rz   r   r   blkyprefix_tokensr   r   s                            r6   forward_intermediateszTNT.forward_intermediatesv  s   . _,Y.YY,&"6s4;;7G"Qi  gg1fe&&q$..9oodii@S@STUW[WgWgik@l0m&noii!6!6q"b!A; OUVW!DNN2mmK099!!#:[[F[[)a-0F' 	VFAs&&uyy/E/E/G+5c;+T([+.{K+H([L $$tTYY{%;U	V !!ERSQq!D$:$:"::;SMSDQRqQq$"8"8"99:RMR##55vuoFDAq^klYZQYYq!Q3;;Aq!QGRRTlMlyy%%',@ ]M!BCM  ii,M))! TR
 ms   K4K8;K
prune_norm
prune_headc                     t        t        | j                        |      \  }}| j                  d|dz    | _        |rt        j                         | _        |r| j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r   )r   r   r   r-   rr   r   r   )r3   r   r  r  r   r   s         r6   prune_intermediate_layerszTNT.prune_intermediate_layers  s]     #7s4;;7G"Qikk.9q=1DI!!!R(r7   c                 j   |j                   d   }| j                  || j                        }| j                  | j	                  | j                  |j                  || j                  d                        }t        j                  | j                  j                  |dd      |fd      }|| j                  z   }| j                  |      }| j                  D ]I  }| j                  r/t        j                   j#                         st%        |||      \  }}> |||      \  }}K | j'                  |      }|S )Nr   r;   r   r=   )r>   ry   r   ro   r2   rn   r?   r   rw   rx   r   r   r   r   r   r   r   r   r   r   )r3   rD   rE   ry   rz   r  s         r6   forward_featureszTNT.forward_features  s	   GGAJ&&q$..9oodii@S@STUW[WgWgik@l0m&noii!6!6q"b!A; OUVW!DNN2mmK0;; 	IC&&uyy/E/E/G+5c;+T([+.{K+H([		I ii,r7   
pre_logitsc                     | j                   r=| j                   dk(  r%|d d | j                  d f   j                  d      n|d d df   }| j                  |      }|r|S | j	                  |      S )Nr   r   r=   r   )r   r   meanr   r   )r3   rD   r  s      r6   forward_headzTNT.forward_head  sq    =A=M=MQV=V!T++,,-22q29\]^_ab^b\cANN1q0DIIaL0r7   c                 J    | j                  |      }| j                  |      }|S r   )r  r  )r3   rD   s     r6   rK   zTNT.forward  s'    !!!$a r7   Fr   r   )NFFFr   F)r   FT)!rN   rO   rP   rQ   r-   r|   r   rR   r   strrT   rS   r   r}   r*   r   rw   r   ignorer   r   r   r   r   r   r   r   r  r	  r  r  rK   rU   rV   s   @r6   r   r      s   
 5868#& #$#%!"!#%$&$&$&*,,, ! /X'CsCx01X' c5c?23X' 	X'
 X' X' X' X' X' !X' !X' X' X' X' !X'  "!X'" "#X'$ "%X'& RYY'X'( )X'* +X't- YY7 7 YY  YY) ) YY		  }C }hsm } 8<).$$',G*||G* eCcN34G* #'	G*
 G* G* G* !%G* 
tELL!5tELL7I)I#JJ	KG*V ./$#	3S	>*  	 $1$ 1r7   c                 :    | ddd dddt         t        dddd	d
dd|S )Nr   )r9   r   r   g?bicubicTzpixel_embed.projr   zarXiv:2103.00112zTransformer in TransformerzMhttps://github.com/huawei-noah/Efficient-AI-Backbones/tree/master/tnt_pytorchz
apache-2.0)urlr   
input_size	pool_sizecrop_pctinterpolationfixed_input_sizer  r   
first_conv
classifier	paper_ids
paper_name
origin_urllicenser	   )r  kwargss     r6   _cfgr#    s>    =t'0F('2e  r7   ztimm/)	hf_hub_id)ztnt_s_legacy_patch16_224.in1kztnt_s_patch16_224.in1kztnt_b_patch16_224.in1kc                    | j                  dd        d| v r| }nxi }| j                         D ]b  \  }}|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd	      }|j                  d
d      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|dk(  rh|j                  j                  dk(  rO|j
                  \  }}}t        |dz        x}}	||	z  |k(  sJ |j                  ddd       j                  ||||	      }|||<   e 	 |d   j
                  |j                  j
                  k7  r(t        |d   |j                  j                  d !      |d<   |S )"Nouter_tokensr   	outer_pos	inner_posr   rz   ry   
proj_norm1rn   
proj_norm2ro   inner_norm1rj   
inner_attnrk   inner_norm2rl   	inner_mlprm   outer_norm1rp   
outer_attnrq   outer_norm2rs   	outer_mlprt   Fg      ?r   r&   r   )new_sizer   )popitemsreplacery   rc   r>   rR   r@   r?   r   r   r   )

state_dictmodelout_dictrI   r0   rE   rF   rG   r   r   s
             r6   checkpoint_filter_fnr:    s   NN>4(j $$& 	DAq		+{3A		+{3A		-7A		,5A		,5A		-3A		,	2A		-7A		+x0A		-4A		,
3A		-4A		+u-AKE$5$5$<$<$E''1aAH%A1uz!zIIaA&..q!Q:HQK'	* U""eoo&;&;; 6[!&&00!

 Or7   c                 r    |j                  dd      }t        t        | |ft        t	        |d      d|}|S )Nout_indicesr9   getter)r<  feature_cls)pretrained_filter_fnfeature_cfg)r4  r   r   r:  r   )variant
pretrainedr"  r<  r8  s        r6   _create_tntrC  1  sF    **]A.K Wj1[hG 	E
 Lr7   r   c           	      X    t        ddddddd      }t        d
d	| it        |fi |}|S )Nr        rY      FT)r   r   r   r   r   r   rc   rB  )tnt_s_legacy_patch16_224r   rC  rB  r"  	model_cfgr8  s       r6   rH  rH  ;  sA    "at%I gzgTR[Mf_eMfgELr7   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nr   rE  rF  rY   rG  Fr   r   r   r   r   r   rB  )tnt_s_patch16_224rI  rJ  s       r6   rN  rN  D  s>    "aI `
`d9F_X^F_`ELr7   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nr   i  (   rY   
   FrM  rB  )tnt_b_patch16_224rI  rJ  s       r6   rR  rR  M  s>    "bI `
`d9F_X^F_`ELr7   )r   r  )-rQ   r   typingr   r   r   r   r   r   rw   torch.nnr-   	timm.datar
   r   timm.layersr   r   r   r   r   r   r   _builderr   	_featuresr   _manipulater   	_registryr   r   __all__r}   r   rX   r   r   r#  default_cfgsr:  rC  rH  rN  rR  ri   r7   r6   <module>r]     s  
  : :   E { { { * + # <'*		 *Z`(BII `(F@ @F{")) {| %%)& # #&  "J C   S   S  r7   