
    ^j$a              	       d   d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZmZmZmZmZ ddlm Z  dd	l!m"Z" dd
l#m$Z$ ddl%m&Z&m'Z' ddl(m)Z)m*Z* dgZ+ G d dejX                        Z- G d dejX                        Z. G d dejX                        Z/ G d dejX                        Z0 G d dejX                        Z1dee2e2f   fdZ3e$dee2e2f   dee2e2f   fd       Z4 G d dejX                        Z5 G d dejX                        Z6 G d  dejX                        Z7d0d!Z8d1d"Z9 e* e9d#$       e9d%$       e9d&$       e9d'$       e9d($      d)      Z:e)d0d*e7fd+       Z;e)d0d*e7fd,       Z<e)d0d*e7fd-       Z=e)d0d*e7fd.       Z>e)d0d*e7fd/       Z?y)2a)   Global Context ViT

From scratch implementation of GCViT in the style of timm swin_transformer_v2_cr.py

Global Context Vision Transformers -https://arxiv.org/abs/2206.09959

@article{hatamizadeh2022global,
  title={Global Context Vision Transformers},
  author={Hatamizadeh, Ali and Yin, Hongxu and Kautz, Jan and Molchanov, Pavlo},
  journal={arXiv preprint arXiv:2206.09959},
  year={2022}
}

Free of any code related to NVIDIA GCVit impl at https://github.com/NVlabs/GCVit.
The license for this code release is Apache 2.0 with no commercial restrictions.

However, weight files adapted from NVIDIA GCVit impl ARE under a non-commercial share-alike license
(https://creativecommons.org/licenses/by-nc-sa/4.0/) until I have a chance to train new ones...

Hacked together by / Copyright 2022, Ross Wightman
    N)partial)CallableListOptionalTupleTypeUnionIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPathcalculate_drop_path_rates	to_2tuple	to_ntupleMlpClassifierHeadLayerNorm2d
LayerScaleget_attnget_act_layerget_norm_layer
RelPosBias_assert   )build_model_with_cfg)feature_take_indices)register_notrace_function)named_apply
checkpoint)register_modelgenerate_default_cfgsGlobalContextVitc                        e Zd ZdZddddej
                  ddfdedee   ded	e	d
e
deej                     f fdZd Z xZS )MbConvBlockzR A depthwise separable / fused mbconv style residual block with SE, `no norm.
    Ng      ?seFin_chsout_chsexpand_ratio
attn_layerbias	act_layerc	                    ||d}	t         |           t        dd|i|	}
t        |t              r|dk(  s|dk(  r
d|
d<   d|
d<   t        |      }|xs |}t        ||z        }t        j                  ||d	d
d
f||d|	| _	         |       | _
         ||fi |
| _        t        j                  ||d
d
dfd|i|	| _        y )Ndevicedtyper+   r%   ecag      ?rd_ratioFr*      r   )groupsr*   r    )super__init__dict
isinstancestrr   intnnConv2dconv_dwactr%   conv_pw)selfr&   r'   r(   r)   r*   r+   r.   r/   ddattn_kwargsmid_chs	__class__s               \/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/gcvit.pyr6   zMbConvBlock.__init__9   s     /5Y5"5j#&:+=uAT&*K
#"'Kj)
#VlV+,yy!QZ&tZWYZ;W44yy'1aLLL    c                     |}| j                  |      }| j                  |      }| j                  |      }| j                  |      }||z   }|S N)r=   r>   r%   r?   )r@   xshortcuts      rE   forwardzMbConvBlock.forwardS   sI    LLOHHQKGGAJLLOLrF   )__name__
__module____qualname____doc__r;   GELUr:   r   floatr9   boolr   Moduler6   rK   __classcell__rD   s   @rE   r$   r$   6   sz    
 &*"%")+MM c]M  	M
 M M BIIM4rF   r$   c                        e Zd Zddej                  eddfdedee   dede	ej                     de	ej                     f
 fdZd	 Z xZS )
Downsample2dNconvdimdim_out	reductionr+   
norm_layerc                    ||d}t         	|           |xs |}|	 ||fi |nt        j                         | _        t        |fd|i|| _        |dv sJ |dk(  r"t        j                  ||dddfdd	i|| _        nL|d
k(  r%||k(  sJ t        j                  ddd      | _        n"||k(  sJ t        j                  d      | _        | ||fi || _        y t        j                         | _        y )Nr-   r+   )rX   maxavgrX   r2      r   r*   Fr^   kernel_sizestridepaddingrb   )r5   r6   r;   Identitynorm1r$   
conv_blockr<   r[   	MaxPool2d	AvgPool2dnorm2)
r@   rY   rZ   r[   r+   r\   r.   r/   rA   rD   s
            rE   r6   zDownsample2d.__init__^   s     /.S.8.DZ*r*"++-
%cEYE"E2222YYsGQ1O5OBODN%'>!>\\a1MDN'>!>\\a8DN2<2HZ.2.
bkkm
rF   c                     | j                  |      }| j                  |      }| j                  |      }| j                  |      }|S rH   )rg   rh   r[   rk   r@   rI   s     rE   rK   zDownsample2d.forwardy   s=    JJqMOOANN1JJqMrF   )rL   rM   rN   r;   rP   r   r:   r   r9   r   rS   r6   rK   rT   rU   s   @rE   rW   rW   ]   so     &*#)+*5\\ c]\ 	\
 BII\ RYY\6rF   rW   c            
       p     e Zd Zddej                  ddfdedededeej                     f fdZ	d	 Z
 xZS )
FeatureBlockr   r^   NrY   levelsr[   r+   c           	         ||d}t         |           |}t        d|      }|dk(  rt        t        j
                  d      }	nt        t        j                  ddd      }	t	        j                         | _        t        |      D ]_  }
| j                  j                  d|
dz    t        |fd	|i|       |s4| j                  j                  d
|
dz     |	              |dz  }a y )Nr-   r   r_   r`   re   r2   ra   rX   r+   pool)r5   r6   r^   r   r;   rj   ri   
Sequentialblocksrange
add_moduler$   )r@   rY   rp   r[   r+   r.   r/   rA   
reductionspool_fnirD   s              rE   r6   zFeatureBlock.__init__   s     /
Qbll:Gbll!QOGmmov 	 AKK""T!A#<S1\I1\Y[1\]&&acU|WY?a
		 rF   c                 $    | j                  |      S rH   )rt   rm   s     rE   rK   zFeatureBlock.forward   s    {{1~rF   )rL   rM   rN   r;   rP   r:   r9   r   rS   r6   rK   rT   rU   s   @rE   ro   ro      sP     ")+     	 
 BII 0rF   ro   c            
            e Zd Zddej                  eddfdededeej                     deej                     f fdZ	d	 Z
 xZS )
Stemr2   `   Nr&   r'   r+   r\   c                     t         |           ||d}t        j                  ||fdddd|| _        t        |f||d|| _        y )Nr-   r2   r`   r   ra   )r+   r\   )r5   r6   r;   r<   conv1rW   down)	r@   r&   r'   r+   r\   r.   r/   rA   rD   s	           rE   r6   zStem.__init__   sU     	/YYvwYAaQRYVXY
 [I*[XZ[	rF   c                 J    | j                  |      }| j                  |      }|S rH   )r   r   rm   s     rE   rK   zStem.forward   s!    JJqMIIaLrF   )rL   rM   rN   r;   rP   r   r:   r   rS   r6   rK   rT   rU   s   @rE   r|   r|      s`     )+*5\\ \ BII	\
 RYY\rF   r|   c                   |     e Zd Z	 	 	 	 	 	 ddededeeef   dedededef fdZdd	ee	j                     fd
Z xZS )WindowAttentionGlobalrY   	num_headswindow_size
use_globalqkv_bias	attn_drop	proj_dropc
                    ||	d}
t         |           t        |      }|| _        || _        ||z  | _        | j
                  dz  | _        || _        t        d||d|
| _	        | j                  r"t        j                  ||dz  fd|i|
| _        n!t        j                  ||dz  fd|i|
| _        t        j                  |      | _        t        j                  ||fi |
| _        t        j                  |      | _        y )Nr-   g      )r   r   r`   r*   r2   r4   )r5   r6   r   r   r   head_dimscaler   r   rel_posr;   LinearqkvDropoutr   projr   )r@   rY   r   r   r   r   r   r   r.   r/   rA   rD   s              rE   r6   zWindowAttentionGlobal.__init__   s     /,&"y(]]d*
$!UkYURTU??yycAgCHCCDHyycAgCHCCDHI.IIc3-"-	I.rF   q_globalc                 0   |j                   \  }}}| j                  r|t        |j                   d   |j                   d   k(  d       | j                  |      }|j	                  ||d| j
                  | j                        j                  ddddd      }|j                  d      \  }}|j                  ||j                   d   z  ddd      }	|	j	                  ||| j
                  | j                        j                  dddd      }	n`| j                  |      j	                  ||d| j
                  | j                        j                  ddddd      }
|
j                  d      \  }	}}|	| j                  z  }	|	|j                  dd      j                         z  }| j                  |      }|j                  d	      }| j                  |      }||z  j                  dd      j	                  |||      }| j!                  |      }| j#                  |      }|S )
Nz*x and q_global seq lengths should be equalr`   r   r2   r      )rY   )shaper   r   r   reshaper   r   permuteunbindrepeatr   	transpose
contiguousr   softmaxr   r   r   )r@   rI   r   BNCkvkvqr   attns               rE   rK   zWindowAttentionGlobal.forward   s   ''1a??x3AGGBK8>>"#557cd!BAq!T^^T]]CKKAqRSUVXYZB99Q<DAqX^^A%6 61a@A		!Q>FFq!QPQRA((1+%%aAt~~t}}MUUVWYZ\]_`bcdCjjmGAq!

N1;;r2&1133||D!|||#~~d#AX  A&..q!Q7IIaLNN1rF   )TT        r   NNrH   )rL   rM   rN   r:   r   rR   rQ   r6   r   torchTensorrK   rT   rU   s   @rE   r   r      s}      $!!!// / sCx	/
 / / / /<8ELL#9 rF   r   r   c                     | j                   \  }}}}| j                  |||d   z  |d   ||d   z  |d   |      } | j                  dddddd      j                         j                  d|d   |d   |      }|S )Nr   r   r2   r`   r      r   r   viewr   r   )rI   r   r   HWr   windowss          rE   window_partitionr      s    JAq!Q	q!{1~%{1~qKN7JKXYN\]^Aii1aAq)446;;BAP[\]P^`abGNrF   img_sizec                     |\  }}| j                   d   }| j                  d||d   z  ||d   z  |d   |d   |      }|j                  dddddd      j                         j                  d|||      }|S )Nr   r   r   r2   r`   r   r   r   )r   r   r   r   r   r   rI   s          rE   window_reverser      s    DAqbARk!n,a;q>.A;q>S^_`SacdeA			!Q1a#..055b!QBAHrF   c                   0    e Zd Zddddddddeej
                  ej                  ddfdedeeef   ded	ed
e	de
de
dee	   de	de	de	dedeej                     deej                     f fdZddeej"                     fdZddeej"                     fdZ xZS )GlobalContextVitBlock         @TNr   rY   	feat_sizer   r   	mlp_ratior   r   layer_scaler   r   	drop_pathr)   r+   r\   c           
         ||d}t         |           t        |      }t        |      }|| _        t	        |d   |d   z  |d   |d   z  z        | _         ||fi || _         ||f|||||
|	d|| _        |t        ||fi |nt        j                         | _        |dkD  rt        |      nt        j                         | _         ||fi || _        t        d|t	        ||z        ||	d|| _        |t        ||fi |nt        j                         | _        |dkD  rt        |      | _        y t        j                         | _        y )Nr-   r   r   )r   r   r   r   r   r   r   )in_featureshidden_featuresr+   dropr4   )r5   r6   r   r   r:   num_windowsrg   r   r   r;   rf   ls1r   
drop_path1rk   r   mlpls2
drop_path2)r@   rY   r   r   r   r   r   r   r   r   r   r   r)   r+   r\   r.   r/   rA   rD   s                     rE   r6   zGlobalContextVitBlock.__init__   s`   & /i(	,&	!A >9Q<S^_`SaCabc*r*
	
#!	
 	
	 :E9P:c;5"5VXVaVaVc1:R(9-R[[]*r*
x3C)O8LXahqxuwx9D9P:c;5"5VXVaVaVc1:R(9-R[[]rF   r   c                 
   |j                   \  }}}}t        || j                        }|j                  d| j                  d   | j                  d   z  |      }| j	                  ||      }t        || j                  ||f      }|S )Nr   r   r   )r   r   r   r   r   r   )	r@   rI   r   r   r   r   r   x_winattn_wins	            rE   _window_attnz"GlobalContextVitBlock._window_attn,  s    WW
1a D$4$45

2t//2T5E5Ea5HH!L99UH-8T%5%51v>rF   c           
         || j                  | j                  | j                  | j                  |      |                  z   }|| j	                  | j                  | j                  | j                  |                        z   }|S rH   )r   r   r   rg   r   r   r   rk   )r@   rI   r   s      rE   rK   zGlobalContextVitBlock.forward4  se    ):):4::a=()S TUU$**Q-)@ ABBrF   rH   )rL   rM   rN   r   r;   rP   	LayerNormr:   r   rQ   rR   r   r   r   rS   r6   r   r   r   rK   rT   rU   s   @rE   r   r      s     !!#!+/!!!#8)+*,,,#+S+S S#X+S 	+S
 +S +S +S +S "%+S +S +S +S !+S BII+S RYY+SZ(> 8ELL#9 rF   r   c            $       (    e Zd Zdddddddddej                  ej
                  eddfdededed	eeef   d
eeef   de	de	de	de
de	dee
   de
de
deee
   e
f   deej                     deej                     deej                     f" fdZd Z xZS )GlobalContextVitStageTFr   Nr   rY   depthr   r   r   
downsampleglobal_norm
stage_normr   r   r   r   r   r   r+   r\   norm_layer_clc                    ||d}t         |           |r-t        d||dz  |d|| _        |dz  }|d   dz  |d   dz  f}nt	        j
                         | _        || _        t        |      }t        t        j                  t        |      t        |      z              }t        ||fi || _        |r	 ||fi |nt	        j
                         | _        t	        j                  t!        |      D cg c]6  }t#        d|||||	|
|dz  dk7  |||t%        |t&              r||   n|||d|8 c}      | _        |r	 ||fi |nt	        j
                         | _        || _        || _        d| _        y c c}w )	Nr-   r`   )rY   rZ   r\   r   r   )rY   r   r   r   r   r   r   r   r   r   r   r+   r\   Fr4   )r5   r6   rW   r   r;   rf   r   r   r:   mathlog2minro   global_blockr   
ModuleListru   r   r8   listrt   normrY   grad_checkpointing)r@   rY   r   r   r   r   r   r   r   r   r   r   r   r   r   r+   r\   r   r.   r/   rA   feat_levelsry   rD   s                          rE   r6   zGlobalContextVitStage.__init__;  s   , /* a% 	DO 'C"1*IaLA,=>I kkmDO",$))C	NS5E$EFG(k@R@7B=33mm" 5\#%
" ! " ##'#!EQJ'##*4Y*E)A,9#( %
 & 1;M#,,	""'-%
s   4;E5c                    | j                  |      }| j                  |      }|j                  dddd      }| j                  |j                  dddd            }| j                  D ]C  }| j
                  r,t        j                  j                         st        |||      }; |||      }E | j                  |      }|j                  dddd      j                         }|S )Nr   r`   r2   r   )r   r   r   r   rt   r   r   jitis_scriptingr   r   r   )r@   rI   global_queryblks       rE   rK   zGlobalContextVitStage.forward}  s    OOA((+ IIaAq!''(<(<Q1a(HI;; 	)C&&uyy/E/E/GsA|4<(		)
 IIaLIIaAq!,,.rF   )rL   rM   rN   r;   rP   r   r   r:   r   rR   rQ   r   r	   r   r   rS   r6   rK   rT   rU   s   @rE   r   r   :  s6     $ %$!!+/!!36)+*,,,-8)@(@( @( 	@(
 S#X@( sCx@( @( @( @( @( @( "%@( @( @( T%[%/0@(  BII!@(" RYY#@($  		?%@(DrF   r   c            +       t    e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d-dedededeeeeef   f   deedf   deeeeedf   f      ded	eedf   d
eedf   dede	dee   dededededededededef* fdZ
d.dZej                  j                  d        Zej                  j                  d/d       Zej                  j                  d0d       Zej                  j                  dej&                  fd       Zd1dedee   fdZ	 	 	 	 	 d2dej,                  d eeeee   f      d!e	d"e	d#ed$e	deeej,                     eej,                  eej,                     f   f   fd%Z	 	 	 d3d eeee   f   d&e	d'e	fd(Zdej,                  dej,                  fd)Zd/d*e	fd+Zdej,                  dej,                  fd,Z xZS )4r"   in_chansnum_classesglobal_poolr   window_ratio.r   	embed_dimdepthsr   r   r   r   	drop_rateproj_drop_rateattn_drop_ratedrop_path_rateweight_initr+   r\   r   norm_epsc                    t         !|           ||d}t        |      }t        t	        |      |      }t        t	        |      |      }g | _        t        |      }t        d |D              }|| _        || _	        || _
        || _        t        |      }t        |d|dz
  z  z        x| _        | _        | t!        |      |      }n:|J t         t!        |      |      D cg c]  }|d   |z  |d   |z  f c}      }t#        d||||d|| _        t'        ||d	      }g }t)        |      D ]  }||dz
  k(  }dt+        |dz
  d      z  } |j-                  t/        di d
|| z  d||   d|	|   d|d   | z  |d   | z  fd||   d|dk7  d|d|
d|d|d|d|d||   d|d|d||       | xj
                  t1        |d   j2                  d|dz   z  d|       gz  c_         t5        j6                  | | _        t;        | j                  |f||d|| _        |r"t?        t        | j@                  |      |        y y c c}w ) Nr-   )epsc              3   &   K   | ]	  }|d z    yw)r   Nr4   ).0ds     rE   	<genexpr>z,GlobalContextVit.__init__.<locals>.<genexpr>  s     3Q!q&3s   r`   r   r   )r&   r'   r+   r\   T)	stagewiserY   r   r   r   r   r   r   r   r   r   r   r   r   r+   r\   r   r   zstages.)num_chsr[   module	pool_typer   )schemer4   )!r5   r6   r   r   r   feature_infor   tupler   r   r   r   lenr:   num_featureshead_hidden_sizer   r|   stemr   ru   r^   appendr   r7   rY   r;   rs   stagesr   headr   _init_weights)"r@   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r+   r\   r   r   r.   r/   rA   r   
num_stagesrdprr  ry   
last_stagestage_scalerD   s"                                    rE   r6   zGlobalContextVit.__init__  s   4 	/!),	^J7XF
} =8LX&3(33	&& "[
47	A*WX.DY8Y4ZZD1"/)J/<K+++OdyYcOdeqOr s!(1+"2HQK14D!E stK 
!	

 
	 ($Oz" 	jAj1n,Js1q5!},KMM/ +Qi $A, %Q<;6	!8ST	
 (N 6 & $ " ( ) ) a& $ &  ,# & $vbz~~QqS\cdecfZg"h!ii-	j. mmV, #4#4#4kt[dmtqst	 2 2;GN Q !ts   'Ic                 \   |dk(  rt        |t        j                        rt        j                  j	                  |j
                         |j                  Zd|v r,t        j                  j                  |j                  d       y t        j                  j                  |j                         y y y t        |t        j                        rbt        j                  j                  |j
                  d       |j                  *t        j                  j                  |j                         y y y )Nvitr   gư>)stdg{Gz?)	r8   r;   r   initxavier_uniform_weightr*   normal_zeros_)r@   r   namer   s       rE   r  zGlobalContextVit._init_weights  s    U?&")),''6;;*}>v{{3	 + - &")),37;;*GGNN6;;/ + -rF   c                     | j                         D ch c]  \  }t        fddD              r c}}S c c}}w )Nc              3   &   K   | ]  }|v  
 y wrH   r4   )r   nr   s     rE   r   z3GlobalContextVit.no_weight_decay.<locals>.<genexpr>  s     Sa16Ss   )relative_position_bias_tablezrel_pos.mlp)named_parametersany)r@   r   _s    ` rE   no_weight_decayz GlobalContextVit.no_weight_decay  sK     //1U U!QS#RSS U 	U Us    :c                      t        dd      }|S )Nz^stemz^stages\.(\d+))r  rt   )r7   )r@   coarsematchers      rE   group_matcherzGlobalContextVit.group_matcher  s    $
 rF   c                 4    | j                   D ]	  }||_         y rH   )r  r   )r@   enabless      rE   set_grad_checkpointingz'GlobalContextVit.set_grad_checkpointing  s     	*A#)A 	*rF   returnc                 .    | j                   j                  S rH   )r  fc)r@   s    rE   get_classifierzGlobalContextVit.get_classifier
  s    yy||rF   c                     ||d}|| _         | | j                  j                  j                  }t	        | j
                  |f|| j                  d|| _        y )Nr-   r   )r   r  r   r   r   r   r   )r@   r   r   r.   r/   rA   s         rE   reset_classifierz!GlobalContextVit.reset_classifier  sW    /&))//99K"4#4#4ky[dhdrdryvxy	rF   rI   indicesr   
stop_early
output_fmtintermediates_onlyc                 r   |dv sJ d       g }t        t        | j                        |      \  }}	| j                  |      }t        j
                  j                         s|s| j                  }
n| j                  d|	dz    }
t        |
      D ]#  \  }} ||      }||v s|j                  |       % |r|S ||fS )a   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWzOutput shape must be NCHW.Nr   )	r   r   r  r  r   r   r   	enumerater  )r@   rI   r*  r   r+  r,  r-  intermediatestake_indices	max_indexr  feat_idxstages                rE   forward_intermediatesz&GlobalContextVit.forward_intermediates  s    * Y&D(DD&"6s4;;7G"Qi IIaL99!!#:[[F[[)a-0F(0 	(OHeaA<'$$Q'	(
   -rF   
prune_norm
prune_headc                     t        t        | j                        |      \  }}| j                  d|dz    | _        |r| j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r    )r   r   r  r)  )r@   r*  r7  r8  r2  r3  s         rE   prune_intermediate_layersz*GlobalContextVit.prune_intermediate_layers?  sM     #7s4;;7G"Qikk.9q=1!!!R(rF   c                 J    | j                  |      }| j                  |      }|S rH   )r  r  rm   s     rE   forward_featuresz!GlobalContextVit.forward_featuresM  s!    IIaLKKNrF   
pre_logitsc                 N    |r| j                  ||      S | j                  |      S )N)r>  )r  )r@   rI   r>  s      rE   forward_headzGlobalContextVit.forward_headR  s%    6@tyyzy2RdiiPQlRrF   c                 J    | j                  |      }| j                  |      }|S rH   )r=  r@  rm   s     rE   rK   zGlobalContextVit.forwardU  s'    !!!$a rF   )r2     r_      )    rD     rD  N@   r2   r      r   r`   r      rE  g      @TNr   r   r   r   r:  gelulayernorm2d	layernormh㈵>NN)r  F)T)NNN)NFFr/  F)r   FT)rL   rM   rN   r:   r9   r	   r   r   rQ   rR   r6   r  r   r   ignorer  r  r#  r;   rS   r'  r)  r   r   r6  r;  r=  r@  rK   rT   rU   s   @rE   r"   r"     sM    #$47,<AE&3)6"!+/!$&$&$&!#+!,"1UOUO UO 	UO
 CsCx01UO  S/UO "%U38_(<"=>UO UO #s(OUO S#XUO UO UO "%UO UO "UO  "!UO" "#UO$ %UO& 'UO( )UO* +UO, -UOn0  YYU U
 YY  YY* * YY		  zC zhsm z 8<$$',( ||(  eCcN34(  	( 
 (  (  !%(  
tELL!5tELL7I)I#JJ	K( X ./$#	3S	>*  	%,, 5<< 
S$ S %,, rF   c                 D    t        t        | |fdt        dd      i|}|S )Nfeature_cfg)r   r   r`   r2   T)out_indicesflatten_sequential)r   r"   r7   )variant
pretrainedkwargsmodels       rE   _create_gcvitrY  [  s3     ':\dK E
 LrF   c                 4    | dddddt         t        dddd	d
|S )NrB  )r2   rC  rC  )r   r   g      ?bicubicz
stem.conv1zhead.fcTz
apache-2.0)urlr   
input_size	pool_sizecrop_pctinterpolationmeanr  
first_conv
classifierfixed_input_sizelicenser
   )r\  rW  s     rE   _cfgrf  d  s6    4}SYI%.B")   rF   z}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_xxtiny_224_nvidia-d1d86009.pth)r\  z|https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_xtiny_224_nvidia-274b92b7.pthz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_tiny_224_nvidia-ac783954.pthz|https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_small_224_nvidia-4e98afa2.pthz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_base_224_nvidia-f009139b.pth)zgcvit_xxtiny.in1kzgcvit_xtiny.in1kzgcvit_tiny.in1kzgcvit_small.in1kzgcvit_base.in1kr$  c                 8    t        dddd|}t        dd| i|S )N)r`   r`      r`   rI  r   r   rV  r4   )gcvit_xxtinyr7   rY  rV  rW  model_kwargss      rE   rj  rj  ~  s6      L OJO,OOrF   c                 8    t        dddd|}t        dd| i|S )N)r2   r   rh  r   rI  ri  rV  r4   )gcvit_xtinyrk  rl  s      rE   ro  ro    s6      L N:NNNrF   c                 8    t        dddd|}t        dd| i|S )NrG  rI  ri  rV  r4   )
gcvit_tinyrk  rl  s      rE   rq  rq    s6      L M*MMMrF   c           	      >    t        ddddddd|}t        d	d| i|S )
NrG  )r2   rh        r}   r`   rN  r   r   r   r   r   rV  r4   )gcvit_smallrk  rl  s      rE   rv  rv    s?       L N:NNNrF   c           	      >    t        ddddddd|}t        d	d| i|S )
NrG  )r   rJ  rE  rD     r`   rN  ru  rV  r4   )
gcvit_baserk  rl  s      rE   ry  ry    s?       L M*MMMrF   rO  )r:  )@rO   r   	functoolsr   typingr   r   r   r   r   r	   r   torch.nnr;   	timm.datar   r   timm.layersr   r   r   r   r   r   r   r   r   r   r   r   r   _builderr   	_featuresr   _features_fxr   _manipulater   r   	_registryr    r!   __all__rS   r$   rW   ro   r|   r   r:   r   r   r   r   r"   rY  rf  default_cfgsrj  ro  rq  rv  ry  r4   rF   rE   <module>r     s  *   ? ?   A    + + 3 0 <
$")) $N!299 !H299 :299 *8BII 8vU38_  sCx E#s(O  9BII 9xRBII RjIryy IX	 % LM KL JK KL JK&  P0@ P P O/? O O N.> N N O/? O O N.> N NrF   