
    ^je                        d Z ddlmZ ddlmZmZmZmZmZ ddl	Z	ddl
mZ ddlmZmZ ddlmZmZmZmZmZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ dgZ G d dej>                        Z  G d dej>                        Z! G d dej>                        Z" G d dej>                        Z#d Z$ed0dee%e%f   de&fd       Z' G d dej>                        Z(d0dZ)d1dZ* e e*d       e*dd       e*dd dd!"       e*d       e*dd       e*dd dd!"       e*d       e*dd       e*d       e*d       e*d      d#      Z+ed0d$e(fd%       Z,ed0d$e(fd&       Z-ed0d$e(fd'       Z.ed0d$e(fd(       Z/ed0d$e(fd)       Z0ed0d$e(fd*       Z1ed0d$e(fd+       Z2ed0d$e(fd,       Z3ed0d$e(fd-       Z4ed0d$e(fd.       Z5ed0d$e(fd/       Z6y)2a   CrossViT Model

@inproceedings{
    chen2021crossvit,
    title={{CrossViT: Cross-Attention Multi-Scale Vision Transformer for Image Classification}},
    author={Chun-Fu (Richard) Chen and Quanfu Fan and Rameswar Panda},
    booktitle={International Conference on Computer Vision (ICCV)},
    year={2021}
}

Paper link: https://arxiv.org/abs/2103.14899
Original code: https://github.com/IBM/CrossViT/blob/main/models/crossvit.py

NOTE: model names have been renamed from originals to represent actual input res all *_224 -> *_240 and *_384 -> *_408

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
Modified from Timm. https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py
    )partial)ListOptionalTupleTypeUnionNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPathcalculate_drop_path_rates	to_2tupletrunc_normal__assert   )build_model_with_cfg)register_notrace_function)register_modelgenerate_default_cfgs)BlockCrossVitc                   b     e Zd ZdZ	 	 	 	 	 	 	 d	deeeeef   f   dedededef
 fdZd Z	 xZ
S )

PatchEmbedz Image to Patch Embedding
    img_size
patch_sizein_chans	embed_dim
multi_convc                    ||d}t         
|           t        |      }t        |      }|d   |d   z  |d   |d   z  z  }	|| _        || _        |	| _        |rM|d   dk(  rt        j                  t        j                  ||dz  fdddd|t        j                  d	
      t        j                  |dz  |dz  fdddd|t        j                  d	
      t        j                  |dz  |fdddd|      | _
        y |d   dk(  rt        j                  t        j                  ||dz  fdddd|t        j                  d	
      t        j                  |dz  |dz  fdddd|t        j                  d	
      t        j                  |dz  |fdddd|      | _
        y y t        j                  ||f||d|| _
        y )Ndevicedtyper   r               )kernel_sizestridepaddingT)inplace      )r'   r(   )super__init__r   r   r   num_patchesnn
SequentialConv2dReLUproj)selfr   r   r   r   r   r!   r"   ddr/   	__class__s             _/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/crossvit.pyr.   zPatchEmbed.__init__+   s    /X&z*
{jm3zRS}8TU $&!}"MMIIh	QaAaYZa^`aGGD)IIi1ni1ng!TU_`gdfgGGD)IIi1nibQqZ[b_ab	 A"$MMIIh	QaAaYZa^`aGGD)IIi1ni1ng!TU_`gdfgGGD)IIi1nibQqZ[b_ab	 % 		(Ig:V`gdfgDI    c                    |j                   \  }}}}t        || j                  d   k(  d| d| d| j                  d    d| j                  d    d	       t        || j                  d   k(  d| d| d| j                  d    d| j                  d    d	       | j                  |      j	                  d      j                  dd      }|S )Nr   zInput image size (*z) doesn't match model (r   z).r+   )shaper   r   r4   flatten	transpose)r5   xBCHWs         r8   forwardzPatchEmbed.forwardQ   s    WW
1aT]]1%%$QCq+B4==QRCSBTTUVZVcVcdeVfUggij	lT]]1%%$QCq+B4==QRCSBTTUVZVcVcdeVfUggij	lIIaL  #--a3r9   )   r,   r&      FNN)__name__
__module____qualname____doc__r   intr   boolr.   rD   __classcell__r7   s   @r8   r   r   '   sr    
 58  $$hCsCx01$h $h 	$h
 $h $hLr9   r   c                   H     e Zd Z	 	 	 	 	 	 ddededededef
 fdZd Z xZS )	CrossAttentiondim	num_headsqkv_bias	attn_drop	proj_dropc                    ||d}t         
|           || _        ||z  }	|	dz  | _        t	        j
                  ||fd|i|| _        t	        j
                  ||fd|i|| _        t	        j
                  ||fd|i|| _        t	        j                  |      | _
        t	        j
                  ||fi || _        t	        j                  |      | _        y )Nr    g      bias)r-   r.   rR   scaler0   LinearwqwkwvDropoutrT   r4   rU   )r5   rQ   rR   rS   rT   rU   r!   r"   r6   head_dimr7   s             r8   r.   zCrossAttention.__init__]   s     /")#%
))C:8:r:))C:8:r:))C:8:r:I.IIc3-"-	I.r9   c                    |j                   \  }}}| j                  |d d dddf         j                  |d| j                  || j                  z        j	                  dddd      }| j                  |      j                  ||| j                  || j                  z        j	                  dddd      }| j                  |      j                  ||| j                  || j                  z        j	                  dddd      }||j                  dd      z  | j                  z  }|j                  d      }| j                  |      }||z  j                  dd      j                  |d|      }| j                  |      }| j                  |      }|S )	Nr   r   .r+   r&   rQ   )r<   rZ   reshaperR   permuter[   r\   r>   rX   softmaxrT   r4   rU   )	r5   r?   r@   NrA   qkvattns	            r8   rD   zCrossAttention.forwardu   sa   ''1aGGAa1ckN#++Aq$..!t~~BUV^^_`bcefhijGGAJq!T^^Q$..5HIQQRSUVXY[\]GGAJq!T^^Q$..5HIQQRSUVXY[\]AKKB''4::5|||#~~d#AX  A&..q!Q7IIaLNN1r9   )   F        rl   NN)	rG   rH   rI   rK   rL   floatr.   rD   rM   rN   s   @r8   rP   rP   \   sQ     "!!// / 	/
 / /0r9   rP   c                        e Zd Zdddddej                  ej
                  ddf	dedededed	ed
edede	ej                     de	ej                     f fdZd Z xZS )CrossAttentionBlock      @Frl   NrQ   rR   	mlp_ratiorS   rU   rT   	drop_path	act_layer
norm_layerc                     |
|d}t         |            |	|fi || _        t        |f||||d|| _        |dkD  rt        |      | _        y t        j                         | _        y )Nr    )rR   rS   rT   rU   rl   )	r-   r.   norm1rP   rj   r   r0   Identityrr   )r5   rQ   rR   rq   rS   rU   rT   rr   rs   rt   r!   r"   r6   r7   s                r8   r.   zCrossAttentionBlock.__init__   su     /*r*
"

 
	 1:B),BKKMr9   c                 ~    |d d dddf   | j                  | j                  | j                  |                  z   }|S )Nr   r   .)rr   rj   rv   )r5   r?   s     r8   rD   zCrossAttentionBlock.forward   s9    a1ckNT^^DIIdjjm,DEEr9   )rG   rH   rI   r0   GELU	LayerNormrK   rm   rL   r   Moduler.   rD   rM   rN   s   @r8   ro   ro      s      ""!!!)+*,,,RR R 	R
 R R R R BIIR RYYR8r9   ro   c                   @    e Zd Zddddej                  ej
                  ddfdeedf   deedf   deedf   deedf   d	eedf   d
e	dedede
ee   ef   deej                     deej                     f fdZdeej                      deej                      fdZ xZS )MultiScaleBlockFrl   NrQ   .patchesdepthrR   rq   rS   rU   rT   rr   rs   rt   c                    ||d}t         |           t        |      }|| _        t	        j
                         | _        t        |      D ]  }g }t        ||         D ]1  }|j                  t        d||   ||   ||   ||||	|   |d|       3 t        |      dk7  sV| j                  j                  t	        j                  |         t        | j                        dk(  rd | _        t	        j
                         | _        t        |      D ]t  }||   ||dz   |z     k(  r	  |||   fi | |
       t	        j                  ||   ||dz   |z     fi |g}| j                  j                  t	        j                  |        v t	        j
                         | _        t        |      D ]  }|dz   |z  }||   }|d   dk(  r7| j                  j                  t        d||   |||   ||||	d   |d|       Og }t        |d         D ].  }|j                  t        d||   |||   ||||	d   |d|       0 | j                  j                  t	        j                  |         t	        j
                         | _        t        |      D ]z  }||dz   |z     ||   k(  r	  |||dz   |z     fi | |
       t	        j                  ||dz   |z     ||   fi |g}| j                   j                  t	        j                  |        | y )Nr    )rQ   rR   rq   rS   rU   rT   rr   rt   r   r   ra    )r-   r.   lennum_branchesr0   
ModuleListblocksrangeappendr   r1   projsrw   rY   fusionro   revert_projs)r5   rQ   r~   r   rR   rq   rS   rU   rT   rr   rs   rt   r!   r"   r6   r   dtmpid_nh_r7   s                         r8   r.   zMultiScaleBlock.__init__   sS     /3x(mmo|$ 	8AC58_ 

5 
A'l'l%'''l)
 
 
 3x1}""2==##67	8" t{{q DK]]_
|$ 	3A1va!e|344 "#a&/B/biiAPSUVYZUZ^jTjPk>roq>rsJJbmmS12	3 mmo|$ 	8Aa%<'B2BRyA~""' 
G"$"+A,!)"+"+"+B-#-
 
 uRy) AJJ2 
 G"$"+A,!)"+"+"+B-#-
  
  
 ""2==##67=	8@ MMO|$ 	:AAE\)*c!f4 "#q1u&<"=DDikyya!e|%;!<c!fKKM$$R]]C%89	:r9   r?   returnc           	         g }t        | j                        D ]  \  }}|j                   |||                ! t        j                  j                  t        t        j                     g       }t        | j                        D ])  \  }}|j                   |||   d d dddf                + g }t        t        | j                  | j                              D ]  \  }\  }}	t        j                  ||   ||dz   | j                  z     d d dd df   fd      }
 ||
      }
 |	|
d d dddf         }t        j                  |||   d d dd df   fd      }
|j                  |
        |S )Nr   r   .rb   )	enumerater   r   torchjitannotater   Tensorr   zipr   r   catr   )r5   r?   outs_br   blockproj_cls_tokenr4   outsr   revert_projr   reverted_proj_cls_tokens               r8   rD   zMultiScaleBlock.forward	  sq   !$++. 	'HAuMM%!+&	' ++D,>C , 	@GAt!!$vayAaC'=">?	@ (1#dkk4CTCT2U(V 	$A$))^A.AARAR7R0STUWXWY[^T^0_`fghC+C&1#a1ck2B&C#))4fQi12s
6KLRSTCKK	 r9   )rG   rH   rI   r0   ry   rz   r   rK   rm   rL   r   r   r   r{   r.   r   r   rD   rM   rN   s   @r8   r}   r}      s    #!!35)+*,,,Z:sCxZ: 38_Z: c?	Z:
 S#XZ: UCZ(Z: Z: Z: Z: T%[%/0Z: BIIZ: RYYZ:xell+ U\\0B r9   r}   c                 j    t        | |      D cg c]  \  }}|d   |z  |d   z  |z   c}}S c c}}w )Nr   r   )r   )r   r~   r   ps       r8   _compute_num_patchesr     s7    .1(G.DEdaAaDAI!!EEEs   /ss
crop_scalec                 z   | j                   dd \  }}||d   k7  s||d   k7  r|ri|d   |k  ra|d   |k  rYt        t        ||d   z
  dz              t        t        ||d   z
  dz              }}| dddd|||d   z   |||d   z   f   } | S t        j                  j
                  j                  | |dd      } | S )	a~  
    Pulled out of CrossViT.forward_features to bury conditional logic in a leaf node for FX tracing.
    Args:
        x (Tensor): input image
        ss (tuple[int, int]): height and width to scale to
        crop_scale (bool): whether to crop instead of interpolate to achieve the desired scale. Defaults to False
    Returns:
        Tensor: the "scaled" image batch tensor
    r`   Nr   r          @bicubicF)sizemodealign_corners)r<   rK   roundr   r0   
functionalinterpolate)r?   r   r   rB   rC   cucls          r8   scale_imager   #  s     7723<DAqBqEzQ"Q%Z"Q%1*A!BqE	R/013ua"Q%i2=M7N3OB!Q21:r"r!u*}45A H ##//Z_/`AHr9   c            '           e Zd ZdZdddddddd	d
dddddddd eej                  d      dddfdedee	df   deedf   dededeedf   deeedf   df   deedf   dee	df   de
de
de
de	d e	d!e	d"e	d#e	d$eej                     d%ef& fd&Zd' Zej"                  j$                  d(        Zej"                  j$                  d3d)       Zej"                  j$                  d4d*       Zej"                  j$                  d+ej                  fd,       Zd5ded%ee   fd-Zd+eej4                     fd.Zd3d/eej4                     d0e
d+ej4                  fd1Zd2 Z xZS )6r   zI Vision Transformer with support for patch or hybrid CNN input stage
    rE   )      ?r   )rk   r,   r&     )     )r   r&   r   r   r   )   r#   )r   r   rp   FTrl   gư>)epstokenNr   	img_scale.r   r   num_classesr   r   rR   rq   r   r   rS   	drop_ratepos_drop_rateproj_drop_rateattn_drop_ratedrop_path_ratert   global_poolc                    t         '|           ||d}|dv sJ || _        || _        || _        t        |      | _        t        |      }|D cg c]1  }t        | j                  D cg c]  }t        ||z         c}      3 c}}| _	        || _
        t        | j                  |      }t        |      | _        || _        t        |      x| _        | _        t%        j&                         | _        t+        | j                        D ]  }t-        | d| t%        j.                  t1        j2                  dd||   z   ||   fi |             t-        | d| t%        j.                  t1        j2                  dd||   fi |              t5        | j                  ||      D ]0  \  }}}| j(                  j7                  t9        d|||||
d|       2 t%        j:                  |      | _        t        |D cg c]  }t        |dd         c}      }t?        ||      } d	}!t%        j&                         | _         tC        |      D ]W  \  }"}#tE        |#d d
       |#d
   z   }$| |!|!|$z    }%tG        |||#f||	||||%|d|}&|!|$z  }!| j@                  j7                  |&       Y t%        j&                  t+        | j                        D cg c]  } |||   fi | c}      | _$        t%        j:                  |      | _%        t%        j&                  t+        | j                        D cg c]5  }|d	kD  rt%        jL                  ||   |fi |nt%        jN                         7 c}      | _(        t+        | j                        D ]6  }tS        tU        | d|       d       tS        tU        | d|       d       8 | jW                  | jX                         y c c}w c c}}w c c}w c c}w c c}w )Nr    r   avg
pos_embed_r   
cls_token_)r   r   r   r   r   )r   r`   r   ra   )rR   rq   rS   rU   rT   rr   rt   {Gz?stdr   )-r-   r.   r   r   r   r   r   tuplerK   img_size_scaledr   r   r   r   r   sumnum_featureshead_hidden_sizer0   r   patch_embedr   setattr	Parameterr   zerosr   r   r   r]   pos_dropr   r   r   maxr}   norm	head_droprY   rw   headr   getattrapply_init_weights)(r5   r   r   r   r   r   r   r   rR   rq   r   r   rS   r   r   r   r   r   rt   r   r!   r"   r6   sisjr/   r   im_sr   r   r?   total_depthdprdpr_ptridx	block_cfg
curr_depthdpr_blkr7   s(                                          r8   r.   zCrossVit.__init__<  s   0 	/....& &!(+i(	XabRTdmm&Ls27|&L Mb$*4+?+?L
O"47	NBD1==? t(() 	aADJqc*BLLQKXYNHZ\efg\h9oln9o,pqDJqc*BLLQ9UV<9^[]9^,_`	a d22J	J 		JD!Q## ! %) 		 

]3613qv;67'Dmmo'. 	$NCYs^,y}<Jww34D! $#!((% C z!GKKs##	$& MM5QUQbQbKc"da:il#Ab#A"de	I.MM4,,-#/ ;F/BIIilK626r{{}\#/ 0	 t(() 	DA'$*QC(89sC'$*QC(89sC	D 	

4%%&w 'Mb4 7. #e#/s*   O(O=	O9O!O&&:O+Oc                    t        |t        j                        rjt        |j                  d       t        |t        j                        r8|j
                  +t        j                  j                  |j
                  d       y y y t        |t        j                        rUt        j                  j                  |j
                  d       t        j                  j                  |j                  d       y y )Nr   r   r   r   )	
isinstancer0   rY   r   weightrW   init	constant_rz   )r5   ms     r8   r   zCrossVit._init_weights  s    a#!((,!RYY'AFF,>!!!&&!, -?'2<<(GGaffa(GGahh, )r9   c                     t               }t        | j                        D ]J  }|j                  d|        t	        | d| d       }|*|j
                  s7|j                  d|        L |S )Nr   r   )setr   r   addr   requires_grad)r5   outr   pes       r8   no_weight_decayzCrossVit.no_weight_decay  sr    et(() 	*AGGj$%A3/6B~""2"2*QC()		*
 
r9   c                      t        dddg      S )Nz ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)i )stemr   )dict)r5   coarses     r8   group_matcherzCrossVit.group_matcher  s    4-/CD
 	
r9   c                     |rJ d       y )Nz$gradient checkpointing not supportedr   )r5   enables     r8   set_grad_checkpointingzCrossVit.set_grad_checkpointing  s    AAAz6r9   r   c                     | j                   S N)r   )r5   s    r8   get_classifierzCrossVit.get_classifier  s    yyr9   c           
      (   || _         ||dv sJ || _        t        | j                  d   d      r#| j                  d   j                  j
                  nd }t        | j                  d   d      r#| j                  d   j                  j                  nd }||d}t        j                  t        | j                        D cg c]?  }|dkD  r$t        j                  | j                  |   |fi |nt        j                         A c}      | _        y c c}w )Nr   r   r   r    )r   r   hasattrr   r   r!   r"   r0   r   r   r   rY   r   rw   )r5   r   r   r!   r"   r6   r   s          r8   reset_classifierzCrossVit.reset_classifier  s    &""2222*D/6tyy|X/N1$$++TX-4TYYq\8-L		!##))RV/MM4,,-#
 @KQBIIdnnQ';;TVT_T_Taa#
 	 #
s   >ADc                    |j                   d   }g }t        | j                        D ]  \  }}|}| j                  |   }t	        ||| j
                        } ||      }|dk(  r| j                  n| j                  }|j                  |dd      }t        j                  ||fd      }|dk(  r| j                  n| j                  }	||	z   }| j                  |      }|j                  |        t        | j                        D ]  \  }}
 |
|      } t        | j                         D cg c]  \  }} |||          }}}|S c c}}w )Nr   ra   r   rb   )r<   r   r   r   r   r   cls_token_0cls_token_1expandr   r   pos_embed_0pos_embed_1r   r   r   r   )r5   r?   r@   xsr   r   x_r   
cls_tokens	pos_embedr   r   s               r8   forward_featureszCrossVit.forward_features  s:   GGAJ'(8(89 	NA{B%%a(BRT__5BRB-.!V))9I9IJ#**1b"5JJ+3B,-F((8H8HIiBr"BIIbM	  , 	FAsRB	 *3499)=>gad2a5k>>	 ?s   )Er  
pre_logitsc                 b   | j                   dk(  r)|D cg c]  }|d d dd f   j                  d       c}n|D cg c]  }|d d df    c}}|D cg c]  }| j                  |       }}|s't        | j                  d   t
        j                        r%t        j                  |D cg c]  }| c}d      S t        j                  t        j                  t        | j                        D cg c]  \  }} |||          c}}d      d      S c c}w c c}w c c}w c c}w c c}}w )Nr   r   rb   r   )r   meanr   r   r   r0   rw   r   r   stackr   )r5   r  r  r?   r   r   s         r8   forward_headzCrossVit.forward_head  s    373C3Cu3LB/qa12hmmm"/dfRg_`STUVXYUYSZRg)+,AdnnQ,,DIIaL"++>99_Aa_!44zz%++)DIIBV&Wwq$tBqE{&W]^_efgg	 0Rg,,&Ws   "DDD!*	D&2D+c                 J    | j                  |      }| j                  |      }|S r   )r  r  )r5   r?   r  s      r8   rD   zCrossVit.forward  s'    ""1%b!r9   F)Tr   )rG   rH   rI   rJ   r   r0   rz   rK   r   rm   rL   r   r{   strr.   r   r   r   ignorer   r   r   r   r   r   r   r   r  r  rD   rM   rN   s   @r8   r   r   8  sW   
  +5*1#)31R)0+7$$!!#%$&$&$&*1",,D*I&-\'\' UCZ(\' c3h	\'
 \' \' S#X\' sCx#-.\' S#X\' UCZ(\' \' \' \' \' !\'  "!\'" "#\'$ "%\'& RYY'\'( )\'|- YY  YY
 
 YYB B YY		  C hsm T%,,%7 .htELL1 ht hPUP\P\ hr9   c                 j    |j                  dd       rt        d      d }t        t        | |fd|i|S )Nfeatures_onlyz<features_only not implemented for Vision Transformer models.c                 ~    i }| j                         D ]'  }d|v sd|v r|j                  dd      }n|}| |   ||<   ) |S )Nr
  	cls_token.r   )keysreplace)
state_dictnew_state_dictkeynew_keys       r8   pretrained_filter_fnz._create_crossvit.<locals>.pretrained_filter_fn  sW    ??$ 	6Cc![C%7++c3/&0oN7#	6 r9   r   )getRuntimeErrorr   r   )variant
pretrainedkwargsr   s       r8   _create_crossvitr&    sM    zz/4(YZZ   2	
  r9   c                 2    | ddd dt         t        ddddd|S )	Nr   )r&      r(  g      ?T)zpatch_embed.0.projzpatch_embed.1.proj)zhead.0zhead.1z
apache-2.0)urlr   
input_size	pool_sizecrop_pctr  r   fixed_input_size
first_conv
classifierlicenser	   )r)  r%  s     r8   _cfgr1    s5    =tY^%.BX\B*  r9   ztimm/)	hf_hub_id)zpatch_embed.0.proj.0zpatch_embed.1.proj.0)r2  r.  )r&     r3  r   )r2  r*  r.  r,  )zcrossvit_15_240.in1kzcrossvit_15_dagger_240.in1kzcrossvit_15_dagger_408.in1kzcrossvit_18_240.in1kzcrossvit_18_dagger_240.in1kzcrossvit_18_dagger_408.in1kzcrossvit_9_240.in1kzcrossvit_9_dagger_240.in1kzcrossvit_base_240.in1kzcrossvit_small_240.in1kzcrossvit_tiny_240.in1kr   c           	      z    t        dddgddgg dg dg dgddgg d	      }t        dd
| dt        |fi |}|S )Nr   g?r#   r,   `   r   r   r$   r   r&   r$   r$   r   r   r   r   r   rR   rq   crossvit_tiny_240r#  r$  r   r   r&  r$  r%  
model_argsmodels       r8   r:  r:  .  sY     b"X"cS\^girRsa&I/J n%8ZnSWXbSmflSmnELr9   c           	      z    t        dddgddgg dg dg dgddgg d	      }t        dd
| dt        |fi |}|S )Nr5  r#   r,   r   r   r7  r   r8  r9  crossvit_small_240r;  r   r<  r=  s       r8   rA  rA  7  sY     b"X#sT]_hjsSta&I/J o%9joTXYcTngmTnoELr9   c           	      z    t        dddgddgg dg dg dgddgg d      }t        dd	| d
t        |fi |}|S )Nr5  r#   r,   r   rF   r7  r8  r9  crossvit_base_240r;  r   r<  r=  s       r8   rC  rC  @  sY     b"X#sT]_hjsStr(i1J n%8ZnSWXbSmflSmnELr9   c           	      z    t        dddgddgg dg dg dgddgg d	      }t        dd
| dt        |fi |}|S )Nr5  r#   r,         r   r&   r   r$   r&   r&   r   r9  crossvit_9_240r;  r   r<  r=  s       r8   rI  rI  I  sY     b"X#sT]_hjsSta&I/J k%5*kPTU_PjciPjkELr9   c           	      z    t        dddgddgg dg dg dgddgg d	      }t        dd
| dt        |fi |}|S )Nr5  r#   r,   r   r   r      r   r   rH  r9  crossvit_15_240r;  r   r<  r=  s       r8   rM  rM  R  sY     b"X#sT]_hjsSta&I/J l%6:lQUV`QkdjQklELr9   c           
      |    t        ddddgddgg dg dg dgddgg dd	|}t        dd
| dt        |fi |}|S )Nr5  r#   r,   rE     r   r   r   r%   rH  r9  crossvit_18_240r;  r   r<  r=  s       r8   rQ  rQ  [  sf     9"BxC:V_ajluUva&I9179J l%6:lQUV`QkdjQklELr9   c           	      |    t        dddgddgg dg dg dgddgg dd	
      }t        dd| dt        |fi |}|S )Nr5  r#   r,   rE  rF  rG  r$   rH  Tr   r   r   r   rR   rq   r   crossvit_9_dagger_240r;  r   r<  r=  s       r8   rT  rT  d  s\    "BxC:V_ajluUva&I$@J r%<rW[\fWqjpWqrELr9   c           	      |    t        dddgddgg dg dg dgddgg dd	
      }t        dd| dt        |fi |}|S )Nr5  r#   r,   r   r   rK  r   rH  TrS  crossvit_15_dagger_240r;  r   r<  r=  s       r8   rV  rV  m  \     b"X#sT]_hjsSta&I$@J s%=*sX\]gXrkqXrsELr9   c           	      |    t        dddgddgg dg dg dgddgg dd	
      }t        dd| dt        |fi |}|S )Nr   g?r#   r,   r   r   rK  r   rH  TrS  crossvit_15_dagger_408r;  r   r<  r=  s       r8   rZ  rZ  v  rW  r9   c           	      |    t        dddgddgg dg dg dgddgg dd	
      }t        dd| dt        |fi |}|S )Nr5  r#   r,   rE   rO  rP  r%   rH  TrS  crossvit_18_dagger_240r;  r   r<  r=  s       r8   r\  r\    rW  r9   c           	      |    t        dddgddgg dg dg dgddgg dd	
      }t        dd| dt        |fi |}|S )NrY  r#   r,   rE   rO  rP  r%   rH  TrS  crossvit_18_dagger_408r;  r   r<  r=  s       r8   r^  r^    rW  r9   r  ) )7rJ   	functoolsr   typingr   r   r   r   r   r   torch.nnr0   	timm.datar
   r   timm.layersr   r   r   r   r   _builderr   _features_fxr   	_registryr   r   vision_transformerr   __all__r{   r   rP   ro   r}   r   rK   rL   r   r   r&  r1  default_cfgsr:  rA  rC  rI  rM  rQ  rT  rV  rZ  r\  r^  r   r9   r8   <module>rk     s  ,  5 5   A ^ ^ * 3 < %,2 2j)RYY )X "))  Fqbii qhF uS#X D  (rryy rj.	 % 73#'C$ $( -]hk$ !73#'C$ $( -]hk$  '2"&C# #W5#g6"W55& < X   h   X   (   8   8      (   (   (   (  r9   