
    ^j                     	   d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlm Z m!Z!m"Z" ddl#m$Z$ dgZ%e G d dejL                               Z'ddZ( G d dejL                        Z) G d dejL                        Z* G d dejL                        Z+ G d dejL                        Z, G d dejL                        Z- G d dejL                        Z.d Z/ddZ0ddZ1 e!i d  e1d!d"#      d$ e1d!d%#      d& e1d!d'd()      d* e1d!d+#      d, e1d!d-#      d. e1d!d/d()      d0 e1d!d1#      d2 e1d!d3#      d4 e1d!d5d()      d6 e1d!d7#      d8 e1d!d9#      d: e1d!d;d()      d< e1d!d=#      d> e1d!d?#      d@ e1d!dAd()      dB e1d!dC#      dD e1d!dE#      i dF e1d!dGd()      dH e1d!dI#      dJ e1d!dK#      dL e1d!dMd()      dN e1d!dO#      dP e1d!dQ#      dR e1d!dSd()      dT e1d!dU#      dV e1d!dW#      dX e1d!dYd()      dZ e1d!d[#      d\ e1d!d]#      d^ e1d!d_d()      d` e1d!da#      db e1d!dc#      dd e1d!ded()      df e1d!dg#       e1d!dh#       e1d!did()       e1d!dj#       e1d!dk#       e1d!dld()       e1d!dm#       e1d!dn#       e1d!dod()      dp      Z2e ddqe.fdr       Z3e ddqe.fds       Z4e ddqe.fdt       Z5e ddqe.fdu       Z6e ddqe.fdv       Z7e ddqe.fdw       Z8e ddqe.fdx       Z9e ddqe.fdy       Z:e ddqe.fdz       Z;e ddqe.fd{       Z<e ddqe.fd|       Z=e ddqe.fd}       Z>e ddqe.fd~       Z?e ddqe.fd       Z@e ddqe.fd       ZAe ddqe.fd       ZBe ddqe.fd       ZCe ddqe.fd       ZDe ddqe.fd       ZEe ddqe.fd       ZFe ddqe.fd       ZGe ddqe.fd       ZHe ddqe.fd       ZIe ddqe.fd       ZJe ddqe.fd       ZKe ddqe.fd       ZLe ddqe.fd       ZMe ddqe.fd       ZN e"eOi dd$dd&dd,dd.dd2dd4dd8dd:dd>dd@ddDddFddJddLddPddRddVdXd\d^dbddddddddd       y)a]   Cross-Covariance Image Transformer (XCiT) in PyTorch

Paper:
    - https://arxiv.org/abs/2106.09681

Same as the official implementation, with some minor adaptations, original copyright below
    - https://github.com/facebookresearch/xcit/blob/master/xcit.py

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
    N)partial)ListOptionalTupleUnionTypeAnyIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPathtrunc_normal_	to_2tupleuse_fused_attnMlp   )build_model_with_cfg)feature_take_indices)register_notrace_module)
checkpoint)register_modelgenerate_default_cfgsregister_model_deprecations)	ClassAttnXcitc                   P     e Zd ZdZ	 	 	 	 	 d
dededef fdZdededefd	Z xZS )PositionalEncodingFourierz
    Positional encoding relying on a fourier kernel matching the one used in the "Attention is all you Need" paper.
    Based on the official XCiT code
        - https://github.com/facebookresearch/xcit/blob/master/xcit.py
    
hidden_dimdimtemperaturec                     ||d}t         |           t        j                  |dz  |fddi|| _        dt
        j                  z  | _        || _        || _	        || _
        d| _        y )Ndevicedtype   kernel_sizer   ư>)super__init__nnConv2dtoken_projectionmathpiscaler    r   r   eps)selfr   r   r    r#   r$   dd	__class__s          [/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/xcit.pyr)   z"PositionalEncodingFourier.__init__)   sh     / "		*q.# S1 SPR S[
&$    BHWc           
         | j                   j                  j                  }| j                   j                  j                  }t	        j
                  d|dz   |      j                  t        j                        j                  d      j                  dd|      }t	        j
                  d|dz   |      j                  t        j                        j                  d|d      }||d d dd d d f   | j                  z   z  | j                  z  }||d d d d dd f   | j                  z   z  | j                  z  }t	        j
                  | j                  |      j                  t        j                        }| j                  dt	        j                  |dd      z  | j                  z  z  }|d d d d d d d f   |z  }	|d d d d d d d f   |z  }
t	        j                  |	d d d d d d dd df   j!                         |	d d d d d d dd df   j#                         gd	      j%                  d
      }	t	        j                  |
d d d d d d dd df   j!                         |
d d d d d d dd df   j#                         gd	      j%                  d
      }
t	        j&                  |
|	fd
	      j)                  dd
dd      }| j                  |j                  |            }|j                  |ddd      S )Nr   )r#   r%   floor)rounding_moder      r      )r,   weightr#   r$   torcharangetofloat32	unsqueezerepeatr0   r/   r   r    divstacksincosflattencatpermute)r1   r6   r7   r8   r#   r$   y_embedx_embeddim_tpos_xpos_yposs               r4   forwardz!PositionalEncodingFourier.forward:   s   &&--44%%,,22,,q!a%7::5==ISSTUV]]^_abdef,,q!a%7::5==IPPQRTUWXYWQQY/$((:;djjHWQ23Y/$((:;djjHT__V<??N  Q5!7)S%SVZVeVe%ef1a&.1a&.U1aADqD=1557q!Q1}9M9Q9Q9STZ[\ddefgU1aADqD=1557q!Q1}9M9Q9Q9STZ[\ddefgiiA.66q!QB##CFF5M2zz!Q1%%r5   )       i'  NN)	__name__
__module____qualname____doc__intfloatr)   rT   __classcell__r3   s   @r4   r   r   !   sT     !!&  	"& & & &r5   r   c           
          ||d}t         j                  j                  t        j                  | |fd|ddd|t        j                  |fi |      S )z3x3 convolution + batch normr"   r?   r   F)r&   stridepaddingbias)rA   r*   
Sequentialr+   BatchNorm2d)	in_planes
out_planesr`   r#   r$   r2   s         r4   conv3x3rg   L   sT    U	+B88
		)ZcQvqW\c`bc
z(R( r5   c                   |     e Zd ZdZddddej
                  ddfdeded	ed
edeej                     f
 fdZ	d Z
 xZS )ConvPatchEmbedz<Image to Patch Embedding using multiple convolutional layers      r?   rV   Nimg_size
patch_sizein_chans	embed_dim	act_layerc                 b   ||d}t         
|           t        |      }|d   |z  |d   |z  z  }	|| _        || _        |	| _        |dk(  r|t        j                  j                  t        ||dz  dfi | |       t        |dz  |dz  dfi | |       t        |dz  |dz  dfi | |       t        |dz  |dfi |      | _
        y |dk(  rct        j                  j                  t        ||dz  dfi | |       t        |dz  |dz  dfi | |       t        |dz  |dfi |      | _
        y d)	Nr"   r   r   rk      r%   r=   z=For convolutional projection, patch size has to be in [8, 16])r(   r)   r   rl   rm   num_patchesrA   r*   rc   rg   proj)r1   rl   rm   rn   ro   rp   r#   r$   r2   rs   r3   s             r4   r)   zConvPatchEmbed.__init__X   sT    /X&{j0Xa[J5NO $&++)q.!:r:	Q	Q@R@	Q	Q@R@	Q	1;;DI 1_++)q.!:r:	Q	Q@R@	Q	1;;DI RRr5   c                     | j                  |      }|j                  d   |j                  d   }}|j                  d      j                  dd      }|||ffS )Nr%   r?   r   )rt   shaperK   	transpose)r1   xHpWps       r4   rT   zConvPatchEmbed.forward   sP    IIaLQWWQZBIIaL""1a(2r({r5   )rW   rX   rY   rZ   r*   GELUr[   r   Moduler)   rT   r]   r^   s   @r4   ri   ri   U   sj    F    )+%S%S %S 	%S
 %S BII%SNr5   ri   c            	            e Zd ZdZdej
                  dddfdedee   deej                     def fdZ
d	ed
efdZ xZS )LPIa  
    Local Patch Interaction module that allows explicit communication between tokens in 3x3 windows to augment the
    implicit communication performed by the block diagonal scatter attention. Implemented using 2 layers of separable
    3x3 convolutions with GeLU and BatchNorm2d
    Nr?   in_featuresout_featuresrp   r&   c                 8   t         	|           ||d}|xs |}|dz  }t        j                  j                  ||f|||d|| _         |       | _        t        j                  |fi || _        t        j                  j                  ||f|||d|| _	        y )Nr"   r%   )r&   ra   groups)
r(   r)   rA   r*   r+   conv1actrd   bnconv2)
r1   r   r   rp   r&   r#   r$   r2   ra   r3   s
            r4   r)   zLPI.__init__   s     	/#2{"XX__j2=wWbjfhj
;..33XX__l3>Xdlhjl
r5   r7   r8   c                 @   |j                   \  }}}|j                  ddd      j                  ||||      }| j                  |      }| j	                  |      }| j                  |      }| j                  |      }|j                  |||      j                  ddd      }|S )Nr   r%   r   )rv   rM   reshaper   r   r   r   )r1   rx   r7   r8   r6   NCs          r4   rT   zLPI.forward   s    ''1aIIaA&&q!Q2JJqMHHQKGGAJJJqMIIaA&&q!Q/r5   )rW   rX   rY   rZ   r*   r{   r[   r   r   r|   r)   rT   r]   r^   s   @r4   r~   r~      sl     +/)+ ll #3-l BII	l
 l*C C r5   r~   c                        e Zd ZdZdddddej
                  ej                  ddddfdeded	ed
e	dededede
ej                     de
ej                     dee   de	f fdZd Z xZS )ClassAttentionBlockzAClass Attention Layer as in CaiT https://arxiv.org/abs/2103.17239      @F              ?Nr   	num_heads	mlp_ratioqkv_bias	proj_drop	attn_drop	drop_pathrp   
norm_layeretatokens_normc                 z   ||d}t         |            |	|fi || _        t        |f||||d|| _        |dkD  rt        |      nt        j                         | _         |	|fi || _	        t        d|t        ||z        ||d|| _        |dkD  rt        |      nt        j                         | _        |
jt        j                  |
t        j                   |fi |z        | _        t        j                  |
t        j                   |fi |z        | _        || _        y d\  | _        | _        || _        y )Nr"   r   r   r   r   r   r   hidden_featuresrp   drop)r   r    )r(   r)   norm1r   attnr   r*   Identity
drop_path1norm2r   r[   mlp
drop_path2	ParameterrA   onesgamma1gamma2r   )r1   r   r   r   r   r   r   r   rp   r   r   r   r#   r$   r2   r3   s                  r4   r)   zClassAttentionBlock.__init__   s=     /*r*


 
	 2;R(9-R[[]*r*
 
i0	

 
 2;R(9-R[[]?,,sUZZ-Br-B'BCDK,,sUZZ-Br-B'BCDK
 ' (0$DK 'r5   c                 >   | j                  |      }t        j                  | j                  |      |d d dd f   gd      }|| j	                  | j
                  |z        z   }| j                  r| j                  |      }n:t        j                  | j                  |d d ddf         |d d dd f   gd      }|}|d d ddf   }| j                  | j                  |      z  }t        j                  ||d d dd f   gd      }|| j                  |      z   }|S )Nr   r>   r   )r   rA   rL   r   r   r   r   r   r   r   r   )r1   rx   x_norm1x_attnx_res	cls_tokens         r4   rT   zClassAttentionBlock.forward   s   **Q-DIIg.12?QGf 455

1A		4::a1Q3i0!AqrE(;CAa1fI	KK$((9"55	IIy!AqrE(+3DOOA&&r5   )rW   rX   rY   rZ   r*   r{   	LayerNormr[   r\   boolr   r|   r   r)   rT   r]   r^   s   @r4   r   r      s    K  ""!!!)+*,,,#% %.'.' .' 	.'
 .' .' .' .' BII.' RYY.' %.' .'`r5   r   c                        e Zd ZU ej                  j
                  e   ed<   	 	 	 	 	 	 	 d
dededede	de	f
 fdZ
d Zej                  j                  d	        Z xZS )XCA
fused_attnr   r   r   r   r   c                    ||d}t         	|           || _        t        d      | _        t        j                  t        j                  |ddfi |      | _	        t        j                  ||dz  fd|i|| _        t        j                  |      | _        t        j                  ||fi || _        t        j                  |      | _        y )Nr"   T)experimentalr   r?   rb   )r(   r)   r   r   r   r*   r   rA   r   r    LinearqkvDropoutr   rt   r   )
r1   r   r   r   r   r   r#   r$   r2   r3   s
            r4   r)   zXCA.__init__   s     /"(d;<<

9a(Ib(IJ99S#'??B?I.IIc3-"-	I.r5   c                    |j                   \  }}}| j                  |      j                  ||d| j                  || j                  z        j	                  ddddd      }|j                  d      \  }}}| j                  rt        j                  j                  j                  |d      | j                  z  }t        j                  j                  j                  |d      }t        j                  j                  j                  |||d	      }nt        j                  j                  j                  |d      }t        j                  j                  j                  |d      }||j                  d
d      z  | j                  z  }	|	j                  d      }	| j                  |	      }	|	|z  }|j	                  dddd      j                  |||      }| j!                  |      }| j#                  |      }|S )Nr?   r%   r   r=   r   r:   r>   r   )r/   )rv   r   r   r   rM   unbindr   rA   r*   
functional	normalizer    scaled_dot_product_attentionrw   softmaxr   rt   r   )
r1   rx   r6   r   r   r   qkvr   s
             r4   rT   zXCA.forward  s   ''1ahhqk!!!Q4>>1;NOWWXY[\^_abdef**Q-1a??##--aR-84;K;KKA##--aR-8A##@@AqPS@TA ##--aR-8A##--aR-8AB++t/?/??D<<B<'D>>$'DqAIIaAq!))!Q2IIaLNN1r5   c                     dhS )Nr    r   r1   s    r4   no_weight_decayzXCA.no_weight_decay$  s
    r5   )rr   Fr   r   NN)rW   rX   rY   rA   jitFinalr   __annotations__r[   r\   r)   rT   ignorer   r]   r^   s   @r4   r   r      s    		%% "!!// / 	/
 / /(0 YY r5   r   c                        e Zd Zdddddej                  ej
                  dddf
dededed	ed
ededede	ej                     de	ej                     def fdZdedefdZ xZS )XCABlockr   Fr   r   Nr   r   r   r   r   r   r   rp   r   r   c                 0   ||d}t         |            |	|fi || _        t        |f||||d|| _        |dkD  rt        |      nt        j                         | _         |	|fi || _	        t        d||d|| _        |dkD  rt        |      nt        j                         | _         |	|fi || _        t        d|t        ||z        ||d|| _        |dkD  rt        |      nt        j                         | _        t        j$                  |
t'        j(                  |fi |z        | _        t        j$                  |
t'        j(                  |fi |z        | _        t        j$                  |
t'        j(                  |fi |z        | _        y )Nr"   r   r   )r   rp   r   r   )r(   r)   r   r   r   r   r*   r   r   norm3r~   local_mp
drop_path3r   r   r[   r   r   r   rA   r   r   gamma3r   )r1   r   r   r   r   r   r   r   rp   r   r   r#   r$   r2   r3   s                 r4   r)   zXCABlock.__init__*  sx    /*r*


 
	 2;R(9-R[[]*r*
GyGBG1:R(9-R[[]*r*
 
i0	

 
 2;R(9-R[[]ll3C)>2)>#>?ll3C)>2)>#>?ll3C)>2)>#>?r5   r7   r8   c           	         || j                  | j                  | j                  | j                  |            z        z   }|| j	                  | j
                  | j                  | j                  |      ||      z        z   }|| j                  | j                  | j                  | j                  |            z        z   }|S N)r   r   r   r   r   r   r   r   r   r   r   r   )r1   rx   r7   r8   s       r4   rT   zXCABlock.forwardX  s    dii

1.F FGG dmmDJJqM1a.P PQQdhhtzz!}.E EFFr5   )rW   rX   rY   r*   r{   r   r[   r\   r   r   r|   r)   rT   r]   r^   s   @r4   r   r   )  s    
  ""!!!)+*,,,,@,@ ,@ 	,@
 ,@ ,@ ,@ ,@ BII,@ RYY,@ ,@\C C r5   r   c            +           e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d-deeeeef   f   dedededededed	ed
ede	dededededede
eej                        de
eej                        dede	dede	f* fdZd Zej"                  j$                  d        Zej"                  j$                  d.d       Zej"                  j$                  d/d       Zej"                  j$                  dej                  fd       Zd0dede
e   fdZ	 	 	 	 	 d1dej0                  d e
eeee   f      d!e	d"e	d#ed$e	deeej0                     eej0                  eej0                     f   f   fd%Z	 	 	 d2d eeee   f   d&e	d'e	fd(Zd) Zd.d*e	fd+Zd, Z xZS )3r   z
    Based on timm and DeiT code bases
    https://github.com/rwightman/pytorch-image-models/tree/master/timm
    https://github.com/facebookresearch/deit/
    rl   rm   rn   num_classesglobal_poolro   depthr   r   r   	drop_ratepos_drop_rateproj_drop_rateattn_drop_ratedrop_path_raterp   r   cls_attn_layersuse_pos_embedr   r   c                    t         |           ||d}|dv sJ t        |      }|d   |z  dk(  r|d   |z  dk(  sJ d       |xs t        t        j
                  d      }|xs t        j                  }|| _        || _        |x| _	        x| _
        | _        || _        d| _        t        d|||||d|| _        |}t	        j                   t#        j$                  d	d	|fi |      | _        |rt)        dd
|i|| _        nd| _        t	        j,                  |      | _        t	        j0                  t3        |      D cg c]  }t5        d|||	|
||||||d
| c}      | _        t3        |      D cg c]  }t9        ||d|        c}| _        t	        j0                  t3        |      D cg c]  }t=        d|||	|
||||||d
| c}      | _         ||fi || _         t	        j,                  |      | _!        |dkD  r!t	        jD                  | j                  |fi |nt	        jF                         | _$        tK        | j&                  d       | jM                  | jN                         yc c}w c c}w c c}w )a  
        Args:
            img_size (int, tuple): input image size
            patch_size (int): patch size
            in_chans (int): number of input channels
            num_classes (int): number of classes for classification head
            embed_dim (int): embedding dimension
            depth (int): depth of transformer
            num_heads (int): number of attention heads
            mlp_ratio (int): ratio of mlp hidden dim to embedding dim
            qkv_bias (bool): enable bias for qkv if True
            drop_rate (float): dropout rate after positional embedding, and in XCA/CA projection + MLP
            pos_drop_rate: position embedding dropout rate
            proj_drop_rate (float): projection dropout rate
            attn_drop_rate (float): attention dropout rate
            drop_path_rate (float): stochastic depth rate (constant across all layers)
            norm_layer: (nn.Module): normalization layer
            cls_attn_layers: (int) Depth of Class attention layers
            use_pos_embed: (bool) whether to use positional encoding
            eta: (float) layerscale initialization value
            tokens_norm: (bool) Whether to normalize all tokens or just the cls_token in the CA

        Notes:
            - Although `layer_norm` is user specifiable, there are hard-coded `BatchNorm2d`s in the local patch
              interaction (class LPI) and the patch embedding (class ConvPatchEmbed)
        r"    avgtokenr   z2`patch_size` should divide image dimensions evenlyr'   )r0   F)rl   rm   rn   ro   rp   r   r   N)p)
r   r   r   r   r   r   r   rp   r   r   zblocks.)num_chs	reductionmodule)
r   r   r   r   r   r   rp   r   r   r   {Gz?stdr   )(r(   r)   r   r   r*   r   r{   r   rn   num_featureshead_hidden_sizero   r   grad_checkpointingri   patch_embedr   rA   zerosr   r   	pos_embedr   pos_drop
ModuleListranger   blocksdictfeature_infor   cls_attn_blocksnorm	head_dropr   r   headr   apply_init_weights)r1   rl   rm   rn   r   r   ro   r   r   r   r   r   r   r   r   r   rp   r   r   r   r   r   r#   r$   r2   r_ir3   s                               r4   r)   zXcit.__init__h  s   h 	/2222X&j(A-HQK*4LPQ4Q 	A@	ARB72<<T#B
(	& ENNND1DN&"') 
!
 
 ekk!Q	&HR&HI6K9KKDN!DN

]3mm 5\%#   ##!(((#% %# $ bgglamn\]T)q7STRUWn!}} ?+.-    ##!#(#%' .-  ." y/B/	I.GRUVBIId//CC\^\g\g\i	 	dnn#.

4%%&Q%# o.-s   I78I<4Jc                    t        |t        j                        rjt        |j                  d       t        |t        j                        r8|j
                  +t        j                  j                  |j
                  d       y y y y )Nr   r   r   )
isinstancer*   r   r   r@   rb   init	constant_)r1   ms     r4   r   zXcit._init_weights  sZ    a#!((,!RYY'AFF,>!!!&&!, -?' $r5   c                 
    ddhS )Nr   r   r   r   s    r4   r   zXcit.no_weight_decay  s    [))r5   c                 "    t        ddddg      S )Nz ^cls_token|pos_embed|patch_embedz^blocks\.(\d+))z^cls_attn_blocks\.(\d+)N)z^norm)i )stemr   r   )r   )r1   coarses     r4   group_matcherzXcit.group_matcher  s    4$?AUV
 	
r5   c                     || _         y r   )r   )r1   enables     r4   set_grad_checkpointingzXcit.set_grad_checkpointing  s
    "(r5   returnc                     | j                   S r   )r   r   s    r4   get_classifierzXcit.get_classifier  s    yyr5   c                    || _         ||dv sJ || _        t        | j                  d      r | j                  j                  j
                  nd }t        | j                  d      r | j                  j                  j                  nd }|dkD  r)t        j                  | j                  |||      | _        y t        j                         | _        y )Nr   r@   r   r"   )r   r   hasattrr   r@   r#   r$   r*   r   r   r   )r1   r   r   r#   r$   s        r4   reset_classifierzXcit.reset_classifier   s    &""6666*D,3DIIx,H!!((d*1$))X*F		  &&D]hkl]lBIId//VSXY	rtr}r}r	r5   rx   indicesr   
stop_early
output_fmtintermediates_onlyc           	         |dv sJ d       |dk(  }g }t        t        | j                        |      \  }	}
|j                  \  }}}}| j	                  |      \  }\  }}| j
                  G| j                  |||      j                  |d|j                  d         j                  ddd      }||z   }| j                  |      }t        j                  j                         s|s| j                  }n| j                  d|
dz    }t        |      D ]q  \  }}| j                  r-t        j                  j                         st        ||||      }n
 ||||      }||	v sN|j                  |r| j!                  |      n|       s |rA|D cg c]6  }|j                  |||d      j                  dd	dd      j#                         8 }}|r|S t        j$                  | j&                  j)                  |dd      |fd
      }| j*                  D ]A  }| j                  r+t        j                  j                         st        ||      }: ||      }C | j!                  |      }||fS c c}w )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWNLCz)Output format must be one of NCHW or NLC.r  Nr:   r   r   r%   r?   r>   )r   lenr   rv   r   r   r   rM   r   rA   r   is_scripting	enumerater   r   appendr   
contiguousrL   r   expandr   )r1   rx   r  r   r  r  r  r   intermediatestake_indices	max_indexr6   r   heightwidthry   rz   pos_encodingr   r   blkys                         r4   forward_intermediateszXcit.forward_intermediates	  sA   * _,Y.YY,&"6s4;;7G"Qi  gg1fe&&q)8B>>%>>!R4<<QAGGAJOWWXY[\^_`LL AMM!99!!#:[[F[[)a-0F' 	BFAs&&uyy/E/E/GsAr2.2rNL $$TTYYq\qA	B `mn[\QYYq"b"5==aAqITTVnMn   IIt~~,,QB7;C'' 	C&&uyy/E/E/GsA&F		 IIaL- os   ;I
prune_norm
prune_headc                    t        t        | j                        |      \  }}| j                  d|dz    | _        |rt        j                         | _        |r+t        j                         | _        | j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r   )	r   r  r   r*   r   r   r   r   r  )r1   r  r(  r)  r   r!  s         r4   prune_intermediate_layerszXcit.prune_intermediate_layersM  sk     #7s4;;7G"Qikk.9q=1DI#%==?D !!!R(r5   c                    |j                   d   }| j                  |      \  }\  }}| j                  G| j                  |||      j                  |d|j                   d         j	                  ddd      }||z   }| j                  |      }| j                  D ]E  }| j                  r-t        j                  j                         st        ||||      }< ||||      }G t        j                  | j                  j                  |dd      |fd      }| j                  D ]A  }| j                  r+t        j                  j                         st        ||      }: ||      }C | j!                  |      }|S )Nr   r:   r   r%   r>   )rv   r   r   r   rM   r   r   r   rA   r   r  r   rL   r   r  r   r   )r1   rx   r6   ry   rz   r$  r%  s          r4   forward_featureszXcit.forward_features^  sL   GGAJ&&q)8B>>%>>!R4<<QAGGAJOWWXY[\^_`LL AMM!;; 	#C&&uyy/E/E/GsAr2.2rN		# IIt~~,,QB7;C'' 	C&&uyy/E/E/GsA&F		 IIaLr5   
pre_logitsc                     | j                   r3| j                   dk(  r|d d dd f   j                  d      n|d d df   }| j                  |      }|r|S | j                  |      S )Nr   r   r>   r   )r   meanr   r   )r1   rx   r.  s      r4   forward_headzXcit.forward_headz  s_    (,(8(8E(A!QR%!$qAwANN1q0DIIaL0r5   c                 J    | j                  |      }| j                  |      }|S r   )r-  r1  )r1   rx   s     r4   rT   zXcit.forward  s'    !!!$a r5   )rj   rk   r?     r   rV      r4  r   Tr   r   r   r   r   NNr%   Tr   FNNF)Tr   )NFFr  F)r   FT) rW   rX   rY   rZ   r   r[   r   strr\   r   r   r   r*   r|   r)   r   rA   r   r   r   r  r  r  r  Tensorr   r'  r+  r-  r1  rT   r]   r^   s   @r4   r   r   a  s    58 #& !!!#%$&$&$&3748#$"& %1|'CsCx01|' |' 	|'
 |' |' |' |' |' |' |' |' !|' "|' "|'  "!|'"  RYY0#|'$ !bii1%|'& !'|'(  )|'* +|', -|'|- YY* * YY
 
 YY) ) YY		  @C @hsm @ 8<$$',B ||B  eCcN34B  	B 
 B  B  !%B  
tELL!5tELL7I)I#JJ	KB L ./$#	3S	>*  	"81$ 1r5   c           	         d| v r| d   } t        |dd       d u}| D cg c]  }|j                  d      s| }}|D ],  }|r%| j                  |      | |j                  dd      <   *| |= . d| v rd|j	                         v rt        |j                        }t        |      D ]  }| j                  d| d      }|j                  d	d
|j                  d
         }t        d      D ]  \  }}	||   | d| d|	 d<    | j                  d| dd       }
|
s|
j                  d	d
      }
t        d      D ]  \  }}	|
|   | d| d|	 d<     | S c c}w )Nmodelr   zpos_embeder.z
pos_embed.z!cls_attn_blocks.0.attn.qkv.weightzcls_attn_blocks.0.attn.q.weightzcls_attn_blocks.z.attn.qkv.weightr?   r:   r   z.attn.z.weightz.attn.qkv.biasz.bias)getattr
startswithpopreplace
state_dictr  r   r   r   rv   r  )r>  r9  r   r   pos_embed_keysnum_ca_blocksr   
qkv_weightj	subscriptr   s              r4   checkpoint_filter_fnrD    s   *(
 E;5TAM!+IAq||K/HaINI BL..QRBSJqyy>?1	 +j8=^bgbrbrbt=tE112}% 		[A#*:1#=M(NOJ#++Ar:3C3CB3GHJ )% 0 [9MWXY]
-aSykIJ[!~~(8>&JDQH##++Ar2$-e$4 [LAyOWXY{J!1!F9+UKL[		[ ) Js
   EEc                 r    |j                  dd      }t        t        | |ft        t	        |d      d|}|S )Nout_indicesr?   getter)rF  feature_cls)pretrained_filter_fnfeature_cfg)r<  r   r   rD  r   )variant
pretraineddefault_cfgkwargsrF  r9  s         r4   _create_xcitrO    sJ    **]A.K  2[hG E Lr5   c                 4    | ddd dddt         t        dddd	|S )
Nr3  )r?   rj   rj   r   bicubicTzpatch_embed.proj.0.0r   z
apache-2.0)urlr   
input_size	pool_sizecrop_pctinterpolationfixed_input_sizer0  r   
first_conv
classifierlicenser
   )rR  rN  s     r4   _cfgr[    s5    =t)%.B,F $* r5   zxcit_nano_12_p16_224.fb_in1kztimm/z<https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p16_224.pth)	hf_hub_idrR  z!xcit_nano_12_p16_224.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p16_224_dist.pthz!xcit_nano_12_p16_384.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p16_384_dist.pth)r?     r]  )r\  rR  rS  zxcit_tiny_12_p16_224.fb_in1kz<https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p16_224.pthz!xcit_tiny_12_p16_224.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p16_224_dist.pthz!xcit_tiny_12_p16_384.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p16_384_dist.pthzxcit_tiny_24_p16_224.fb_in1kz<https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p16_224.pthz!xcit_tiny_24_p16_224.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p16_224_dist.pthz!xcit_tiny_24_p16_384.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p16_384_dist.pthzxcit_small_12_p16_224.fb_in1kz=https://dl.fbaipublicfiles.com/xcit/xcit_small_12_p16_224.pthz"xcit_small_12_p16_224.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p16_224_dist.pthz"xcit_small_12_p16_384.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p16_384_dist.pthzxcit_small_24_p16_224.fb_in1kz=https://dl.fbaipublicfiles.com/xcit/xcit_small_24_p16_224.pthz"xcit_small_24_p16_224.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p16_224_dist.pthz"xcit_small_24_p16_384.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p16_384_dist.pthzxcit_medium_24_p16_224.fb_in1kz>https://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p16_224.pthz#xcit_medium_24_p16_224.fb_dist_in1kzChttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p16_224_dist.pthz#xcit_medium_24_p16_384.fb_dist_in1kzChttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p16_384_dist.pthzxcit_large_24_p16_224.fb_in1kz=https://dl.fbaipublicfiles.com/xcit/xcit_large_24_p16_224.pthz"xcit_large_24_p16_224.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p16_224_dist.pthz"xcit_large_24_p16_384.fb_dist_in1kzBhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p16_384_dist.pthzxcit_nano_12_p8_224.fb_in1kz;https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p8_224.pthz xcit_nano_12_p8_224.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p8_224_dist.pthz xcit_nano_12_p8_384.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_nano_12_p8_384_dist.pthzxcit_tiny_12_p8_224.fb_in1kz;https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p8_224.pthz xcit_tiny_12_p8_224.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p8_224_dist.pthz xcit_tiny_12_p8_384.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_12_p8_384_dist.pthzxcit_tiny_24_p8_224.fb_in1kz;https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p8_224.pthz xcit_tiny_24_p8_224.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p8_224_dist.pthz xcit_tiny_24_p8_384.fb_dist_in1kz@https://dl.fbaipublicfiles.com/xcit/xcit_tiny_24_p8_384_dist.pthzxcit_small_12_p8_224.fb_in1kz<https://dl.fbaipublicfiles.com/xcit/xcit_small_12_p8_224.pthz!xcit_small_12_p8_224.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p8_224_dist.pthz!xcit_small_12_p8_384.fb_dist_in1kzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_12_p8_384_dist.pthzxcit_small_24_p8_224.fb_in1kz<https://dl.fbaipublicfiles.com/xcit/xcit_small_24_p8_224.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p8_224_dist.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_small_24_p8_384_dist.pthz=https://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p8_224.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p8_224_dist.pthzBhttps://dl.fbaipublicfiles.com/xcit/xcit_medium_24_p8_384_dist.pthz<https://dl.fbaipublicfiles.com/xcit/xcit_large_24_p8_224.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p8_224_dist.pthzAhttps://dl.fbaipublicfiles.com/xcit/xcit_large_24_p8_384_dist.pth)!xcit_small_24_p8_224.fb_dist_in1k!xcit_small_24_p8_384.fb_dist_in1kzxcit_medium_24_p8_224.fb_in1k"xcit_medium_24_p8_224.fb_dist_in1k"xcit_medium_24_p8_384.fb_dist_in1kzxcit_large_24_p8_224.fb_in1k!xcit_large_24_p8_224.fb_dist_in1k!xcit_large_24_p8_384.fb_dist_in1kr  c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrk      r4  r=   r   Frm   ro   r   r   r   r   rL  )xcit_nano_12_p16_224r   rO  rL  rN  
model_argsr9  s       r4   rg  rg  @  s=    B!RWYJeJe$zJd]cJdeELr5   c           	      X    t        ddddddd      }t        d
d	| it        |fi |}|S )Nrk   re  r4  r=   r   Fr]  )rm   ro   r   r   r   r   rl   rL  )xcit_nano_12_p16_384rh  ri  s       r4   rl  rl  H  s@    B!RWbegJeJe$zJd]cJdeELr5   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrk      r4  r=   r   Trf  rL  )xcit_tiny_12_p16_224rh  ri  s       r4   ro  ro  P  =    B!RVXJeJe$zJd]cJdeELr5   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrk   rn  r4  r=   r   Trf  rL  )xcit_tiny_12_p16_384rh  ri  s       r4   rr  rr  X  rp  r5   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrk   r]  r4  rr   r   Trf  rL  )xcit_small_12_p16_224rh  ri  s       r4   rt  rt  `  >    B!RVXJfZf4PZKe^dKefELr5   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrk   r]  r4  rr   r   Trf  rL  )xcit_small_12_p16_384rh  ri  s       r4   rw  rw  h  ru  r5   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrk   rn     r=   h㈵>Trf  rL  )xcit_tiny_24_p16_224rh  ri  s       r4   r{  r{  p  =    B!SWYJeJe$zJd]cJdeELr5   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrk   rn  ry  r=   rz  Trf  rL  )xcit_tiny_24_p16_384rh  ri  s       r4   r~  r~  x  r|  r5   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrk   r]  ry  rr   rz  Trf  rL  )xcit_small_24_p16_224rh  ri  s       r4   r  r    >    B!SWYJfZf4PZKe^dKefELr5   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrk   r]  ry  rr   rz  Trf  rL  )xcit_small_24_p16_384rh  ri  s       r4   r  r    r  r5   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrk      ry  rr   rz  Trf  rL  )xcit_medium_24_p16_224rh  ri  s       r4   r  r    >    B!SWYJgjgDQ[Lf_eLfgELr5   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrk   r  ry  rr   rz  Trf  rL  )xcit_medium_24_p16_384rh  ri  s       r4   r  r    r  r5   c           	      V    t        dddddd      }t        dd| it        |fi |}|S )	Nrk   rV   ry  rz  Trf  rL  )xcit_large_24_p16_224rh  ri  s       r4   r  r    >    B"$TXZJfZf4PZKe^dKefELr5   c           	      V    t        dddddd      }t        dd| it        |fi |}|S )	Nrk   rV   ry  rz  Trf  rL  )xcit_large_24_p16_384rh  ri  s       r4   r  r    r  r5   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrr   re  r4  r=   r   Frf  rL  )xcit_nano_12_p8_224rh  ri  s       r4   r  r    =    2QVXJd:djIc\bIcdELr5   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrr   re  r4  r=   r   Frf  rL  )xcit_nano_12_p8_384rh  ri  s       r4   r  r    r  r5   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrr   rn  r4  r=   r   Trf  rL  )xcit_tiny_12_p8_224rh  ri  s       r4   r  r    =    2QUWJd:djIc\bIcdELr5   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrr   rn  r4  r=   r   Trf  rL  )xcit_tiny_12_p8_384rh  ri  s       r4   r  r    r  r5   c           	      V    t        dddddd      }t        dd| it        |fi |}|S )	Nrr   r]  r4  r   Trf  rL  )xcit_small_12_p8_224rh  ri  s       r4   r  r    =    2QUWJeJe$zJd]cJdeELr5   c           	      V    t        dddddd      }t        dd| it        |fi |}|S )	Nrr   r]  r4  r   Trf  rL  )xcit_small_12_p8_384rh  ri  s       r4   r  r    r  r5   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrr   rn  ry  r=   rz  Trf  rL  )xcit_tiny_24_p8_224rh  ri  s       r4   r  r    =    2RVXJd:djIc\bIcdELr5   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrr   rn  ry  r=   rz  Trf  rL  )xcit_tiny_24_p8_384rh  ri  s       r4   r  r    r  r5   c           	      V    t        dddddd      }t        dd| it        |fi |}|S )	Nrr   r]  ry  rz  Trf  rL  )xcit_small_24_p8_224rh  ri  s       r4   r  r    =    2RVXJeJe$zJd]cJdeELr5   c           	      V    t        dddddd      }t        dd| it        |fi |}|S )	Nrr   r]  ry  rz  Trf  rL  )xcit_small_24_p8_384rh  ri  s       r4   r  r    r  r5   c           	      V    t        dddddd      }t        dd| it        |fi |}|S )	Nrr   r  ry  rz  Trf  rL  )xcit_medium_24_p8_224rh  ri  s       r4   r  r    >    2RVXJfZf4PZKe^dKefELr5   c           	      V    t        dddddd      }t        dd| it        |fi |}|S )	Nrr   r  ry  rz  Trf  rL  )xcit_medium_24_p8_384rh  ri  s       r4   r  r  	  r  r5   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrr   rV   ry  rk   rz  Trf  rL  )xcit_large_24_p8_224rh  ri  s       r4   r  r    =    2SWYJeJe$zJd]cJdeELr5   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
Nrr   rV   ry  rk   rz  Trf  rL  )xcit_large_24_p8_384rh  ri  s       r4   r  r    r  r5   xcit_nano_12_p16_224_distxcit_nano_12_p16_384_distxcit_tiny_12_p16_224_distxcit_tiny_12_p16_384_distxcit_tiny_24_p16_224_distxcit_tiny_24_p16_384_distxcit_small_12_p16_224_distxcit_small_12_p16_384_distxcit_small_24_p16_224_distxcit_small_24_p16_384_distxcit_medium_24_p16_224_distxcit_medium_24_p16_384_distxcit_large_24_p16_224_distxcit_large_24_p16_384_distxcit_nano_12_p8_224_distxcit_nano_12_p8_384_distxcit_tiny_12_p8_224_distr^  r_  r`  ra  rb  rc  )xcit_tiny_12_p8_384_distxcit_tiny_24_p8_224_distxcit_tiny_24_p8_384_distxcit_small_12_p8_224_distxcit_small_12_p8_384_distxcit_small_24_p8_224_distxcit_small_24_p8_384_distxcit_medium_24_p8_224_distxcit_medium_24_p8_384_distxcit_large_24_p8_224_distxcit_large_24_p8_384_dist)r   NN)FN)r   r5  )PrZ   r-   	functoolsr   typingr   r   r   r   r   r	   rA   torch.nnr*   	timm.datar   r   timm.layersr   r   r   r   r   _builderr   	_featuresr   _features_fxr   _manipulater   	_registryr   r   r   caitr   __all__r|   r   rg   ri   r~   r   r   r   r   rD  rO  r[  default_cfgsrg  rl  ro  rr  rt  rw  r{  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rW   r   r5   r4   <module>r     s8	  	   : :   A O O * + 1 # Y Y ( '&		 '& '&T.RYY .b$")) $NA")) AH5")) 5p5ryy 5pb299 bJ	:
 % B&"DJ%LB&
 (O*QB& (O\i*kB& #DJ%LB& (O*QB&" (O\i*k#B&( #DJ%L)B&. (O*Q/B&4 (O\i*k5B&: $TK&M;B&@ )$P+RAB&F )$P]j+lGB&L $TK&MMB&R )$P+RSB&X )$P]j+lYB&^ %dL'N_B&d *4Q,SeB&j *4Q^k,mkB&p $TK&MqB&v )$P+RwB&| )$P]j+l}B&F "4I$KGB&L 'N)PMB&R 'N[h)jSB&X "4I$KYB&^ 'N)P_B&d 'N[h)jeB&j "4I$KkB&p 'N)PqB&v 'N[h)jwB&| #DJ%L}B&B (O*QCB&H (O\i*kIB&N #DJ%LOB&T *.O*Q *.O\i*k &*K&M +/P+R +/P]j+l %)J%L *.O*Q *.O\i*kB& BJ                               $   $         t   t   t   t         t   t                     H  '!D '  !D '  !D	 '
  !D '  !D '  !D ' !"F ' !"F ' !"F ' !"F ' "#H ' "#H ' !"F ' !"F '$  B% '&  B' '(  B) '* !C B B!D!D!D!D"F"F!D!D? '  r5   