
    ^jZ                     \   d Z ddlmZ ddlmZmZmZmZmZm	Z	 ddl
Z
ddlmZ ddlmZmZ ddlmZmZmZmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZmZ g dZ  G d dejB                        Z" G d dejB                        Z# G d dejB                        Z$ G d dejB                        Z% G d dejB                        Z&d5dZ'd6dZ(d7dZ) e e)ddd       e)dd       e)dd d       e)dd!       e)dd"       e)dd#d       e)dd$       e)dd%       e)dd&       e)dd'd(      d)
      Z*ed6d*e&fd+       Z+ed6d*e&fd,       Z,ed6d*e&fd-       Z-ed6d*e&fd.       Z.ed6d*e&fd/       Z/ed6d*e&fd0       Z0ed6d*e&fd1       Z1ed6d*e&fd2       Z2ed6d*e&fd3       Z3ed6d*e&fd4       Z4y)8a3   Class-Attention in Image Transformers (CaiT)

Paper: 'Going deeper with Image Transformers' - https://arxiv.org/abs/2103.17239

Original code and weights from https://github.com/facebookresearch/deit, copyright below

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
    )partial)ListOptionalTupleUnionTypeAnyNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)
PatchEmbedMlpDropPathtrunc_normal_use_fused_attn   )build_model_with_cfg)feature_take_indices)
checkpointcheckpoint_seq)register_modelgenerate_default_cfgs)Cait	ClassAttnLayerScaleBlockClassAttnLayerScaleBlockTalkingHeadAttnc                        e Zd ZU ej                  j
                  e   ed<   	 	 	 	 	 	 d	dededede	de	f
 fdZ
d Z xZS )
r   
fused_attndim	num_headsqkv_bias	attn_drop	proj_dropc                    t         
|           ||d}|| _        ||z  }	|	dz  | _        t	               | _        t        j                  ||fd|i|| _        t        j                  ||fd|i|| _	        t        j                  ||fd|i|| _
        t        j                  |      | _        t        j                  ||fi || _        t        j                  |      | _        y )Ndevicedtype      bias)super__init__r!   scaler   r   nnLinearqkvDropoutr#   projr$   selfr    r!   r"   r#   r$   r'   r(   ddhead_dim	__class__s             [/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/cait.pyr,   zClassAttn.__init__    s     	/")#%
(*39(9b939(9b939(9b9I.IIc3-"-	I.    c                    |j                   \  }}}| j                  |d d df         j                  d      j                  |d| j                  || j                  z        j                  dddd      }| j                  |      j                  ||| j                  || j                  z        j                  dddd      }| j                  |      j                  ||| j                  || j                  z        j                  dddd      }| j                  rPt        j                  j                  j                  |||| j                  r| j                  j                  nd      }nL|| j                   z  }||j#                  dd      z  }	|	j%                  d	      }	| j                  |	      }	|	|z  }|j#                  dd      j                  |d|      }| j'                  |      }| j)                  |      }|S )
Nr   r                 )	dropout_pr    )shaper0   	unsqueezereshaper!   permuter1   r2   r   torchr.   
functionalscaled_dot_product_attentiontrainingr#   pr-   	transposesoftmaxr4   r$   )
r6   xBNCr0   r1   r2   x_clsattns
             r:   forwardzClassAttn.forward8   s   ''1aFF1QT7O%%a(00At~~qDNNGZ[ccdeghjkmnoFF1IaDNNA4GHPPQRTUWXZ[\FF1IaDNNA4GHPPQRTUWXZ[\??HH''DD1a.2mm$..** E E
 DJJAq{{2r**D<<B<'D>>$'D1HE1%--aA6		% u%r;      Fr?   r?   NN)__name__
__module____qualname__rH   jitFinalbool__annotations__intfloatr,   rU   __classcell__r9   s   @r:   r   r      sf     		%%
 "!!// / 	/
 / /0r;   r   c                        e Zd Zdddddej                  ej
                  eedddfdedede	d	e
d
e	de	de	deej                     deej                     deej                     deej                     de	f fdZd Z xZS )r         @Fr?   -C6?Nr    r!   	mlp_ratior"   r$   r#   	drop_path	act_layer
norm_layer
attn_block	mlp_blockinit_valuesc                    t         |           ||d} |	|fi || _         |
|f||||d|| _        |dkD  rt	        |      nt        j                         | _         |	|fi || _        t        ||z        } |d||||d|| _
        t        j                  |t        j                  |fi |z        | _        t        j                  |t        j                  |fi |z        | _        y Nr&   )r!   r"   r#   r$   r?   )in_featureshidden_featuresrh   drop r+   r,   norm1rT   r   r.   Identityrg   norm2r_   mlp	ParameterrH   onesgamma_1gamma_2r6   r    r!   rf   r"   r$   r#   rg   rh   ri   rj   rk   rl   r'   r(   r7   mlp_hidden_dimr9   s                    r:   r,   z!LayerScaleBlockClassAttn.__init__T       " 	/*r*


 
	 1:B),BKKM*r*
S9_- 
*	

 
 ||K%**S2GB2G$GH||K%**S2GB2G$GHr;   c           	      4   t        j                  ||fd      }|| j                  | j                  | j	                  | j                  |            z        z   }|| j                  | j                  | j                  | j                  |            z        z   }|S )Nr   rC   )	rH   catrg   rz   rT   rt   r{   rw   rv   )r6   rO   rS   us       r:   rU   z LayerScaleBlockClassAttn.forward}   sv    IIuaja(t||dii

16N'NOOt||dhhtzz%?P6Q'QRRr;   )rX   rY   rZ   r.   GELU	LayerNormr   r   r_   r`   r]   r   Moduler,   rU   ra   rb   s   @r:   r   r   Q   s      ""!!!)+*,,,*3),!%'I'I 'I 	'I
 'I 'I 'I 'I BII'I RYY'I RYY'I BII'I 'IRr;   r   c                   H     e Zd Z	 	 	 	 	 	 ddededededef
 fdZd Z xZS )	r   r    r!   r"   r#   r$   c                    t         
|           ||d}|| _        ||z  }	|	dz  | _        t	        j
                  ||dz  fd|i|| _        t	        j                  |      | _        t	        j
                  ||fi || _	        t	        j
                  ||fi || _
        t	        j
                  ||fi || _        t	        j                  |      | _        y )Nr&   r)   r>   r*   )r+   r,   r!   r-   r.   r/   qkvr3   r#   r4   proj_lproj_wr$   r5   s             r:   r,   zTalkingHeadAttn.__init__   s     	/")#%
99S#'??B?I.IIc3-"-	ii	9;;ii	9;;I.r;   c                    |j                   \  }}}| j                  |      j                  ||d| j                  || j                  z        j	                  ddddd      }|d   | j
                  z  |d   |d   }}}||j                  dd      z  }	| j                  |	j	                  dddd            j	                  dddd      }	|	j                  d      }	| j                  |	j	                  dddd            j	                  dddd      }	| j                  |	      }	|	|z  j                  dd      j                  |||      }| j                  |      }| j                  |      }|S )	Nr>   r=   r   r      rA   rB   rC   )rD   r   rF   r!   rG   r-   rM   r   rN   r   r#   r4   r$   )
r6   rO   rP   rQ   rR   r   r0   r1   r2   rT   s
             r:   rU   zTalkingHeadAttn.forward   sL   ''1ahhqk!!!Q4>>1;NOWWXY[\^_abdefa&4::%s1vs1va11;;r2&&{{4<<1a34<<Q1aH|||#{{4<<1a34<<Q1aH~~d#AX  A&..q!Q7IIaLNN1r;   rV   )	rX   rY   rZ   r_   r]   r`   r,   rU   ra   rb   s   @r:   r   r      sQ     "!!// / 	/
 / /:r;   r   c                        e Zd Zdddddej                  ej
                  eedddfdedede	d	e
d
e	de	de	deej                     deej                     deej                     deej                     de	f fdZd Z xZS )r   rd   Fr?   re   Nr    r!   rf   r"   r$   r#   rg   rh   ri   rj   rk   rl   c                    t         |           ||d} |	|fi || _         |
|f||||d|| _        |dkD  rt	        |      nt        j                         | _         |	|fi || _        t        ||z        } |d||||d|| _
        t        j                  |t        j                  |fi |z        | _        t        j                  |t        j                  |fi |z        | _        y rn   rs   r|   s                    r:   r,   zLayerScaleBlock.__init__   r~   r;   c           	         || j                  | j                  | j                  | j                  |            z        z   }|| j                  | j                  | j                  | j                  |            z        z   }|S N)rg   rz   rT   rt   r{   rw   rv   r6   rO   s     r:   rU   zLayerScaleBlock.forward   sa    t||dii

1.FFGGt||dhhtzz!}.EEFFr;   )rX   rY   rZ   r.   r   r   r   r   r_   r`   r]   r   r   r,   rU   ra   rb   s   @r:   r   r      s      ""!!!)+*,,,*9),!%'I'I 'I 	'I
 'I 'I 'I 'I BII'I RYY'I RYY'I BII'I 'IRr;   r   c            7           e Zd Zdddddddddd	d
d
d
d
d
eee eej                  d      ej                  e
edeeddddfdedededededededededededededededeej$                     d eej$                     d!eej$                     d"eej$                     d#eej$                     d$eej$                     d%eej$                     d&ed'eej$                     d(eej$                     d)ed*ef6 fd+Zd, Zej,                  j.                  d-        Zej,                  j.                  dAd.       Zej,                  j.                  dBd/       Zej,                  j.                  d0ej$                  fd1       ZdCdedee   fd2Z	 	 	 	 	 dDd3ej<                  d4eeee e   f      d5ed6ed7ed8ed0ee ej<                     e!ej<                  e ej<                     f   f   fd9Z"	 	 	 dEd4eee e   f   d:ed;efd<Z#d= Z$dBd>efd?Z%d@ Z& xZ'S )Fr         r>     token      rd   Tr?   ư>)epsre   r=   Nimg_size
patch_sizein_chansnum_classesglobal_pool	embed_dimdepthr!   rf   r"   	drop_ratepos_drop_rateproj_drop_rateattn_drop_ratedrop_path_rateblock_layersblock_layers_tokenpatch_layerri   rh   rj   rk   rl   attn_block_token_onlymlp_block_token_onlydepth_token_onlymlp_ratio_token_onlyc                 L   t         $|           ||d}|dv sJ || _        || _        || _        |x| _        x| _        | _        d| _         |d||||d|| _	        | j                  j                  }t        | j                  d      r| j                  j                         n|} t        j                  t        j                   dd|fi |      | _        t        j                  t        j                   d||fi |      | _        t        j&                  |      | _        t+        |      D !cg c]  }!| }"}!t        j,                  t+        |      D !cg c]  }! |d|||	|
|||"|!   |||||d| c}! | _        t+        |      D !cg c]  }!t1        || d	|! 
       c}!| _        t        j4                  t+        |      D #cg c]  }# |d||||
|||||d	| c}#      | _         ||fi || _        t        j&                  |      | _        |dkD  rt        j<                  ||fi |nt        j>                         | _         tC        | j$                  d       tC        | j"                  d       | jE                  | jF                         y c c}!w c c}!w c c}!w c c}#w )Nr&    r   avgF)r   r   r   r   
feat_ratior   )rL   )r    r!   rf   r"   r$   r#   rg   ri   rh   rj   rk   rl   blocks.)num_chs	reductionmodule)	r    r!   rf   r"   ri   rh   rj   rk   rl   r   {Gz?stdrr   )$r+   r,   r   r   r   num_featureshead_hidden_sizer   grad_checkpointingpatch_embednum_patcheshasattrr   r.   rx   rH   zeros	cls_token	pos_embedr3   pos_droprange
Sequentialblocksdictfeature_info
ModuleListblocks_token_onlynorm	head_dropr/   ru   headr   apply_init_weights)%r6   r   r   r   r   r   r   r   r!   rf   r"   r   r   r   r   r   r   r   r   ri   rh   rj   rk   rl   r   r   r   r   r'   r(   r7   r   ridpr_r9   s%                                       r:   r,   zCait.__init__   s   @ 	/2222& &ENNND1DN"'& 
!	

 
 &&22-4T5E5E|-TD'')Zdekk!Q	&HR&HIekk![)&Rr&RS

]3',U|4!~44mm <&!  '3 '
$$!f!!#'
 '
 &! " bgglamn\]T)q7STRUWn!# )*0,  1C 1
*!,*#1
 1
 0, "- y/B/	I.?JQBIIi;;TVT_T_Ta	dnn#.dnn#.

4%%&M 5&! o0,s   .	JJ
JJ!c                    t        |t        j                        rjt        |j                  d       t        |t        j                        r8|j
                  +t        j                  j                  |j
                  d       y y y t        |t        j                        rUt        j                  j                  |j
                  d       t        j                  j                  |j                  d       y y )Nr   r   r         ?)	
isinstancer.   r/   r   weightr*   init	constant_r   )r6   ms     r:   r   zCait._init_weightsM  s    a#!((,!RYY'AFF,>!!!&&!, -?'2<<(GGaffa(GGahh, )r;   c                 
    ddhS )Nr   r   rr   r6   s    r:   no_weight_decayzCait.no_weight_decayV  s    [))r;   c                     || _         y r   )r   )r6   enables     r:   set_grad_checkpointingzCait.set_grad_checkpointingZ  s
    "(r;   c                       fd}|S )Nc                    t        dD cg c]  }| j                  |       c}      ry| j                  d      r t        | j                  d      d         dz   S | j                  d      rNt	        j
                        t	        j                        z
  dz   }t        | j                  d      d         |z   S | j                  d      rt	        j
                        S t        d      S c c}w )	N)r   r   r   r   r   .r   zblocks_token_only.znorm.inf)any
startswithr_   splitlenr   r   r`   )namen	to_offsetr6   s      r:   _matcherz$Cait.group_matcher.<locals>._matcher`  s    0YZ1DOOA&Z[+4::c?1-.22!56,s43I3I/JJQN	4::c?1-.::)4;;''U|# [s   C,rr   )r6   coarser   s   `  r:   group_matcherzCait.group_matcher^  s    	$ r;   returnc                     | j                   S r   )r   r   s    r:   get_classifierzCait.get_classifiero  s    yyr;   c                     || _         ||dv sJ || _        |dkD  r&t        j                  | j                  |      | _        y t        j
                         | _        y )Nr   r   )r   r   r.   r/   r   ru   r   )r6   r   r   s      r:   reset_classifierzCait.reset_classifiers  sU    &""6666*DALqBIId//=	VXVaVaVc	r;   rO   indicesr   
stop_early
output_fmtintermediates_onlyc           	      6   |dv sJ d       |dk(  }g }t        t        | j                        |      \  }	}
|j                  \  }}}}| j	                  |      }|| j
                  z   }| j                  |      }t        j                  j                         s|s| j                  }n| j                  d|
dz    }t        |      D ]m  \  }}| j                  r+t        j                  j                         st        ||      }n ||      }||	v sJ|j                  |r| j                  |      n|       o |ra| j                  j                  ||f      \  }}|D cg c]6  }|j!                  |||d      j#                  dddd	      j%                         8 }}|r|S | j&                  j)                  |j                  d   dd      }t        | j*                        D ]  \  }} |||      } t        j,                  ||fd
      }| j                  |      }||fS c c}w )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        )NCHWNLCz)Output format must be one of NCHW or NLC.r   Nr   rB   r   r>   r=   rC   )r   r   r   rD   r   r   r   rH   r[   is_scripting	enumerater   r   appendr   dynamic_feat_sizerF   rG   
contiguousr   expandr   r   )r6   rO   r   r   r   r   r   rF   intermediatestake_indices	max_indexrP   r   heightwidthr   r   blkHWy
cls_tokenss                         r:   forward_intermediateszCait.forward_intermediatesz  s    & _,Y.YY,&"6s4;;7G"Qi  gg1feQMM!99!!#:[[F[[)a-0F' 	BFAs&&uyy/E/E/GsA&FL $$TTYYq\qA	B ##55vuoFDAq^klYZQYYq!Q3;;Aq!QGRRTlMl   ^^**1771:r2>
 6 67 	,FAsQ
+J	,IIz1o1-IIaL- ms   ;H
prune_norm
prune_headc                    t        t        | j                        |      \  }}| j                  d|dz    | _        |rt        j                         | _        |r+t        j                         | _        | j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r   )	r   r   r   r.   ru   r   r   r   r   )r6   r   r  r  r  r  s         r:   prune_intermediate_layerszCait.prune_intermediate_layers  sk     #7s4;;7G"Qikk.9q=1DI%']]_D"!!!R(r;   c                     | j                  |      }|| j                  z   }| j                  |      }| j                  r5t        j
                  j                         st        | j                  |      }n| j                  |      }| j                  j                  |j                  d   dd      }t        | j                        D ]  \  }} |||      } t	        j                  ||fd      }| j                  |      }|S )Nr   rB   r   rC   )r   r   r   r   rH   r[   r   r   r   r   r   rD   r   r   r   r   )r6   rO   r	  r   r  s        r:   forward_featureszCait.forward_features  s    QMM!""599+A+A+Ct{{A.AAA^^**1771:r2>
 6 67 	,FAsQ
+J	,IIz1o1-IIaLr;   
pre_logitsc                     | j                   r3| j                   dk(  r|d d dd f   j                  d      n|d d df   }| j                  |      }|r|S | j                  |      S )Nr   r   rC   r   )r   meanr   r   )r6   rO   r  s      r:   forward_headzCait.forward_head  s_    (,(8(8E(A!QR%!$qAwANN1q0DIIaL0r;   c                 J    | j                  |      }| j                  |      }|S r   )r  r  r   s     r:   rU   zCait.forward  s'    !!!$a r;   )TFr   )NFFr   F)r   FT)(rX   rY   rZ   r   r   r   r   r.   r   r   r   r   r   r_   strr`   r]   r   r   r,   r   rH   r[   ignorer   r   r   r   r   r   Tensorr   r   r   r
  r  r  r  rU   ra   rb   s   @r:   r   r      sm   
   #& !!!#%$&$&$&,;2J+5*1",,D*I)+*9),!%5>47$%*-=^'^' ^' 	^'
 ^' ^' ^' ^' ^' ^' ^' ^' !^' "^' "^'  "!^'" ryy/#^'$ !%RYY%^'& bii'^'( RYY)^'* BII+^', RYY-^'. BII/^'0 1^'2 $(		?3^'4 #'ryy/5^'6 "7^'8 #(9^'@- YY* * YY) ) YY   YY		  dC dhsm d 8<$$',; ||;  eCcN34;  	; 
 ;  ;  !%;  
tELL!5tELL7I)I#JJ	K; ~ ./$#	3S	>*  	"1$ 1r;   r   c                 v    d| v r| d   } i }| j                         D ]  \  }}|||j                  dd      <    |S )Nmodelzmodule.r   )itemsreplace)
state_dictr  checkpoint_no_moduler1   r2   s        r:   checkpoint_filter_fnr     sR    *(
  " ;19:QYYy"56;r;   c                 r    |j                  dd      }t        t        | |ft        t	        |d      d|}|S )Nout_indicesr>   getter)r"  feature_cls)pretrained_filter_fnfeature_cfg)popr   r   r   r   )variant
pretrainedkwargsr"  r  s        r:   _create_caitr+    sJ    **]A.K  2[hG E Lr;   c                 4    | ddd dddt         t        dddd	|S )
Nr   )r>     r-  r   bicubicTzpatch_embed.projr   z
apache-2.0)urlr   
input_size	pool_sizecrop_pctinterpolationfixed_input_sizer  r   
first_conv
classifierlicenser
   )r/  r*  s     r:   _cfgr8    s5    =t)%.B(  r;   ztimm/z1https://dl.fbaipublicfiles.com/deit/XXS24_224.pth)r>   r   r   )	hf_hub_idr/  r0  z1https://dl.fbaipublicfiles.com/deit/XXS24_384.pth)r9  r/  z1https://dl.fbaipublicfiles.com/deit/XXS36_224.pthz1https://dl.fbaipublicfiles.com/deit/XXS36_384.pthz0https://dl.fbaipublicfiles.com/deit/XS24_384.pthz/https://dl.fbaipublicfiles.com/deit/S24_224.pthz/https://dl.fbaipublicfiles.com/deit/S24_384.pthz/https://dl.fbaipublicfiles.com/deit/S36_384.pthz/https://dl.fbaipublicfiles.com/deit/M36_384.pthz/https://dl.fbaipublicfiles.com/deit/M48_448.pth)r>     r:  )
zcait_xxs24_224.fb_dist_in1kzcait_xxs24_384.fb_dist_in1kzcait_xxs36_224.fb_dist_in1kzcait_xxs36_384.fb_dist_in1kzcait_xs24_384.fb_dist_in1kzcait_s24_224.fb_dist_in1kzcait_s24_384.fb_dist_in1kzcait_s36_384.fb_dist_in1kzcait_m36_384.fb_dist_in1kzcait_m48_448.fb_dist_in1kr   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	Nr         r   h㈵>r   r   r   r!   rl   r)  )cait_xxs24_224r   r+  r)  r*  
model_argsr  s       r:   r@  r@  5  7    s"W[\J_j_DD^W]D^_ELr;   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	Nr   r<  r=  r   r>  r?  r)  )cait_xxs24_384rA  rB  s       r:   rF  rF  <  rD  r;   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	Nr   r<  $   r   r>  r?  r)  )cait_xxs36_224rA  rB  s       r:   rI  rI  C  rD  r;   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	Nr   r<  rH  r   r>  r?  r)  )cait_xxs36_384rA  rB  s       r:   rK  rK  J  rD  r;   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	Nr   i   r=     r>  r?  r)  )cait_xs24_384rA  rB  s       r:   rN  rN  Q  s7    s"W[\J^Z^4
C]V\C]^ELr;   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	Nr   r-  r=  rW   r>  r?  r)  )cait_s24_224rA  rB  s       r:   rP  rP  X  7    s"W[\J]J]$zB\U[B\]ELr;   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	Nr   r-  r=  rW   r>  r?  r)  )cait_s24_384rA  rB  s       r:   rS  rS  _  rQ  r;   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	Nr   r-  rH  rW   r   r?  r)  )cait_s36_384rA  rB  s       r:   rU  rU  f  rQ  r;   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )Nr   r   rH  r   r?  r)  )cait_m36_384rA  rB  s       r:   rW  rW  m  7    s"X\]J]J]$zB\U[B\]ELr;   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )Nr   r   0   r   r?  r)  )cait_m48_448rA  rB  s       r:   r[  r[  t  rX  r;   r   r  )r   )5__doc__	functoolsr   typingr   r   r   r   r   r	   rH   torch.nnr.   	timm.datar   r   timm.layersr   r   r   r   r   _builderr   	_featuresr   _manipulater   r   	_registryr   r   __all__r   r   r   r   r   r   r   r+  r8  default_cfgsr@  rF  rI  rK  rN  rP  rS  rU  rW  r[  rr   r;   r:   <module>rh     sf    : :   A P P * + 3 <
a3		 3l0ryy 0f1bii 1h/bii /dv299 vr 
	 %#'? $
 $(?$ $(? $
 $(?$ #'># "&= "
 "&=" "&=" "&=" "&= "Q-& -` $   $   $   $                    r;   