
    ^j{                        d Z ddlmZmZmZmZmZmZ ddlZddl	m
Z
 ddlm
c mZ ddlmZmZ ddlmZmZmZmZmZmZmZ ddlmZ ddlmZmZ d	gZ G d
 de
j>                        Z  G d de
j>                        Z! G d de
j>                        Z" G d de
j>                        Z# G d de
j>                        Z$ G d d	e
j>                        Z%d Z&d Z'd Z(d*dZ)d+dZ* e e*d       e*d       e*d       e*d       e*d       e*d       e*d       e*dddd      d       Z+ed,d!e%fd"       Z,ed,d!e%fd#       Z-ed,d!e%fd$       Z.ed,d!e%fd%       Z/ed,d!e%fd&       Z0ed,d!e%fd'       Z1ed,d!e%fd(       Z2ed,d!e%fd)       Z3y)-z
CoaT architecture.

Paper: Co-Scale Conv-Attentional Image Transformers - https://arxiv.org/abs/2104.06399

Official CoaT code at: https://github.com/mlpc-ucsd/CoaT

Modified from timm/models/vision_transformer.py
    )ListOptionalTupleUnionTypeAnyNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)
PatchEmbedMlpDropPath	to_2tupletrunc_normal__assert	LayerNorm   )build_model_with_cfg)register_modelgenerate_default_cfgsCoaTc            	       V     e Zd ZdZ	 	 ddededeeef   f fdZdeeef   fdZ	 xZ
S )	ConvRelPosEncz+ Convolutional relative position encoding. head_chs	num_headswindowc           	      N   ||d}t         |           t        |t              r||i}|| _        n"t        |t
              r|| _        n
t               t        j                         | _	        g | _
        |j                         D ]y  \  }}d}	||dz
  |	dz
  z  z   dz  }
t        j                  ||z  ||z  f||f|
|
f|	|	f||z  d|}| j                  j                  |       | j                  j                  |       { | j                  D cg c]  }||z  	 c}| _        yc c}w )aj  
        Initialization.
            Ch: Channels per head.
            h: Number of heads.
            window: Window size(s) in convolutional relative positional encoding. It can have two forms:
                1. An integer of window size, which assigns all attention heads with the same window s
                    size in ConvRelPosEnc.
                2. A dict mapping window size to #attention head splits (
                    e.g. {window size 1: #attention head split 1, window size 2: #attention head split 2})
                    It will apply different window size to the attention head splits.
        devicedtyper      )kernel_sizepaddingdilationgroupsN)super__init__
isinstanceintr   dict
ValueErrornn
ModuleList	conv_listhead_splitsitemsConv2dappendchannel_splits)selfr   r   r   r   r    dd
cur_windowcur_head_splitr$   padding_sizecur_convx	__class__s                [/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/coat.pyr'   zConvRelPosEnc.__init__   s9   & /fc"i(F DK% DK,*0,,. 	4&JH '*q.X\)JJqPLyy)) (4%|4"H-%0 H NN!!(+##N3	4  6:5E5EFq8|FFs   D"sizec                 V   |j                   \  }}}}|\  }}	t        |d||	z  z   k(  d       |d d d d dd d d f   }
|d d d d dd d d f   }|j                  dd      j                  |||z  ||	      }t	        j
                  || j                  d      }g }t        | j                        D ]  \  }}|j                   |||                ! t	        j                  |d      }|j                  |||||	z        j                  dd      }|
|z  }t        j                  |d      }|S )Nr    dim)r   r   r   r   r   r   )shaper   	transposereshapetorchsplitr3   	enumerater.   r2   catFpad)r4   qvr=   Br   NCHWq_imgv_img
v_img_listconv_v_img_listiconv
conv_v_imgEV_hats                     r<   forwardzConvRelPosEnc.forwardM   s'   WW9a1QQY# !QA+!QA+B'//9q=!QG[[(;(;C
 0 	8GAt""4
1#67	8YYA6
''9aQ?II"bQ
#v12    )NN)__name__
__module____qualname____doc__r)   r   r*   r'   r   r\   __classcell__r;   s   @r<   r   r      sQ    5 1G1G 1G #t)$	1Gf%S/ r]   r   c                   h     e Zd ZdZ	 	 	 	 	 	 	 ddedededededee   f fdZ	d	e
eef   fd
Z xZS )FactorAttnConvRelPosEnczK Factorized attention with convolutional relative position encoding class. rC   r   qkv_bias	attn_drop	proj_dropshared_crpec	                 H   ||d}	t         |           || _        ||z  }
|
dz  | _        t	        j
                  ||dz  fd|i|	| _        t	        j                  |      | _        t	        j
                  ||fi |	| _	        t	        j                  |      | _
        || _        y )Nr   g         bias)r&   r'   r   scaler,   LinearqkvDropoutrg   projrh   crpe)r4   rC   r   rf   rg   rh   ri   r   r    r5   head_dimr;   s              r<   r'   z FactorAttnConvRelPosEnc.__init__e   s     /")#%
99S#'??B?I.IIc3-"-	I.  	r]   r=   c                    |j                   \  }}}| j                  |      j                  ||d| j                  || j                  z        j	                  ddddd      }|j                  d      \  }}}	|j                  d      }
|
j                  dd      |	z  }||z  }| j                  ||	|	      }| j                  |z  |z   }|j                  dd      j                  |||      }| j                  |      }| j                  |      }|S )
Nrk   r!   r   r      rB   r@   rA   r=   )rD   ro   rF   r   permuteunbindsoftmaxrE   rr   rm   rq   rh   )r4   r:   r=   rO   rP   rQ   ro   rM   krN   	k_softmax
factor_attrr   s                r<   r\   zFactorAttnConvRelPosEnc.forward~   s	   ''1a hhqk!!!Q4>>1;NOWWXY[\^_abdef**Q-1a II!I$	((R014
^
 yyADy) JJ#d*KK1%%aA. IIaLNN1r]   )   F        r~   NNN)r^   r_   r`   ra   r)   boolfloatr   r   r'   r   r\   rb   rc   s   @r<   re   re   c   st    U "!!)-     	 
     "# 2uS#X r]   re   c                   J     e Zd ZdZ	 	 	 ddedef fdZdeeef   fdZ xZS )
ConvPosEnczy Convolutional Position Encoding.
        Note: This module is similar to the conditional position encoding in CPVT.
    rC   rz   c                 t    ||d}t         |           t        j                  |||d|dz  fd|i|| _        y )Nr   r   r!   r%   )r&   r'   r,   r1   rq   )r4   rC   rz   r   r    r5   r;   s         r<   r'   zConvPosEnc.__init__   s@     /IIc31adE3E"E	r]   r=   c                 j   |j                   \  }}}|\  }}t        |d||z  z   k(  d       |d d d df   |d d dd f   }	}|	j                  dd      j                  ||||      }
| j	                  |
      |
z   }|j                  d      j                  dd      }t        j                  ||fd      }|S )Nr   r?   r!   rB   )rD   r   rE   viewrq   flattenrG   rJ   )r4   r:   r=   rO   rP   rQ   rR   rS   	cls_token
img_tokensfeats              r<   r\   zConvPosEnc.forward   s    ''1a1QQY# !"!RaR%!AqrE(:	 ##Aq)..q!Q:IIdOd"IIaL""1a( IIy!n!,r]   )rk   NN)	r^   r_   r`   ra   r)   r'   r   r\   rb   rc   s   @r<   r   r      sA     	F	F 	FuS#X r]   r   c                        e Zd ZdZdddddej
                  ej                  ddddfdededed	e	d
ededede
ej                     de
ej                     dee   dee   f fdZdeeef   fdZ xZS )SerialBlockz Serial block class.
        Note: In this implementation, each serial block only contains a conv-attention and a FFN (MLP) module.       @Fr~   NrC   r   	mlp_ratiorf   rh   rg   	drop_path	act_layer
norm_layer
shared_cperi   c           	      8   ||d}t         |           |
| _         |	|fi || _        t	        |f|||||d|| _        |dkD  rt        |      nt        j                         | _	         |	|fi || _
        t        ||z        }t        d||||d|| _        y )Nr   r   rf   rg   rh   ri   r~   in_featureshidden_featuresr   drop )r&   r'   cpenorm1re   factoratt_crper   r,   Identityr   norm2r)   r   mlp)r4   rC   r   r   rf   rh   rg   r   r   r   r   ri   r   r    r5   mlp_hidden_dimr;   s                   r<   r'   zSerialBlock.__init__   s      / *r*
5
#
 
 1:B),BKKM  *r*
S9_- 
*	

 
r]   r=   c                    | j                  ||      }| j                  |      }| j                  ||      }|| j                  |      z   }| j	                  |      }| j                  |      }|| j                  |      z   }|S N)r   r   r   r   r   r   )r4   r:   r=   curs       r<   r\   zSerialBlock.forward   sx    HHQjjm!!#t,s## jjmhhsms##r]   )r^   r_   r`   ra   r,   GELUr   r)   r   r   r   Moduler   r   r'   r   r\   rb   rc   s   @r<   r   r      s    s  ""!!!)+*,,,(,)-+
+
 +
 	+

 +
 +
 +
 +
 BII+
 RYY+
 !+
 "#+
ZuS#X r]   r   c                   @    e Zd ZdZdddddej
                  ej                  dddf
dee   dedee	   de
d	e	d
e	de	deej                     deej                     deee      f fdZde	deeef   fdZde	deeef   fdZde	deeef   fdZdeeeef      fdZ xZS )ParallelBlockz Parallel block class. NFr~   dimsr   
mlp_ratiosrf   rh   rg   r   r   r   shared_crpesc           	         ||d}t         |           |g } |	|d   fi || _         |	|d   fi || _         |	|d   fi || _        t        |d   f|||||
d   d|| _        t        |d   f|||||
d   d|| _        t        |d   f|||||
d   d|| _        |dkD  rt        |      nt        j                         | _         |	|d   fi || _         |	|d   fi || _         |	|d   fi || _        |d   |d   cxk(  r	|d   k(  sJ  J |d   |d   cxk(  r	|d   k(  sJ  J t!        |d   |d   z        }t#        d|d   |||d|x| _        x| _        | _        y )	Nr   r   r!   rk   r   r~   r   r   )r&   r'   norm12norm13norm14re   factoratt_crpe2factoratt_crpe3factoratt_crpe4r   r,   r   r   norm22norm23norm24r)   r   mlp2mlp3mlp4)r4   r   r   r   rf   rh   rg   r   r   r   r   r   r    r5   r   r;   s                  r<   r'   zParallelBlock.__init__   s    /J !a/B/ a/B/ a/B/6G 
$Q 
  
  7G 
$Q 
  
  7G 
$Q 
  
 1:B),BKKM !a/B/ a/B/ a/B/Aw$q',T!W,,,,,!}
1>A>>>>>T!Wz!}45,/ -
Q*	-

 -
 	
	 	
DI	r]   factorr=   c                 *    | j                  |||      S )z Feature map up-sampling. scale_factorr=   interpolater4   r:   r   r=   s       r<   upsamplezParallelBlock.upsample@  s    TBBr]   c                 0    | j                  |d|z  |      S )z Feature map down-sampling.       ?r   r   r   s       r<   
downsamplezParallelBlock.downsampleD  s    F
FFr]   r   c                    |j                   \  }}}|\  }}t        |d||z  z   k(  d       |ddddddf   }	|ddddddf   }
|
j                  dd      j                  ||||      }
t	        j
                  |
|ddd      }
|
j                  ||d      j                  dd      }
t        j                  |	|
fd	      }|S )
z Feature map interpolation. r   r?   Nr!   Fbilinear)r   recompute_scale_factormodealign_cornersr@   rB   )rD   r   rE   rF   rK   r   rG   rJ   )r4   r:   r   r=   rO   rP   rQ   rR   rS   r   r   outs               r<   r   zParallelBlock.interpolateH  s    ''1a1QQY#a!QhK	q!"ax[
))!Q/771aC
]]%#(

  ''1b1;;AqA
iiJ/Q7
r]   sizesc                    |\  }}}}	| j                  |      }
| j                  |      }| j                  |      }| j                  |
|      }
| j	                  ||      }| j                  ||	      }| j                  |d|      }| j                  |d|	      }| j                  |d|	      }| j                  |
d|      }| j                  |d|      }| j                  |
d|      }|
|z   |z   }
||z   |z   }||z   |z   }|| j                  |
      z   }|| j                  |      z   }|| j                  |      z   }| j                  |      }
| j                  |      }| j                  |      }| j                  |
      }
| j                  |      }| j                  |      }|| j                  |
      z   }|| j                  |      z   }|| j                  |      z   }||||fS )Nrv   g       @)r   r=   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r4   x1x2x3x4r   _S2S3S4cur2cur3cur4upsample3_2upsample4_3upsample4_2downsample2_3downsample3_4downsample2_4s                      r<   r\   zParallelBlock.forward_  s   2r2{{2{{2{{2##Dr#2##Dr#2##Dr#2mmD"m=mmD"m=mmD"m=RbARbARbAk!K/k!M1m#m3$..&&$..&&$..&& {{2{{2{{2yyyyyy$..&&$..&&$..&&2r2~r]   )r^   r_   r`   ra   r,   r   r   r   r)   r   r   r   r   r   r   r'   r   r   r   r   r\   rb   rc   s   @r<   r   r      s2   !
 '+"!!!)+*,,,04C
s)C
 C
 U	C

 C
 C
 C
 C
 BIIC
 RYYC
 #49-C
JC% CuS#X CGE GsCx G5 c3h . T%S/-B  r]   r   c            '       ~    e Zd ZdZdddddddd	d
dddddeddddddfdededededeeeeef   deeeeef   dededeeeeef   dededededede	e
j                     ded eee      d!ee   d"ef& fd#Zd$ Zej&                  j(                  d%        Zej&                  j(                  d0d&       Zej&                  j(                  d1d'       Zej&                  j(                  d(e
j                  fd)       Zd2ded"ee   fd*Zd+ Zd1d,eej8                  eej8                     f   d-efd.Zd(ej8                  fd/Z xZS )3r   z CoaT class.       rk     @      @     rk   ru      rk   r   r}   )ru   ru   ru   ru   Tr~   FNtokenimg_size
patch_sizein_chansnum_classes
embed_dimsserial_depthsparallel_depthr   r   rf   	drop_rateproj_drop_rateattn_drop_ratedrop_path_rater   return_interm_layersout_featurescrpe_windowglobal_poolc                    t         |           ||d}|dv sJ |xs dddd}|| _        || _        || _        |d   x| _        | _        || _        || _        || _	        t        |      }t        d||||d   t        j                  d|| _        t        d|D cg c]  }|d	z  	 c}d|d   |d
   t        j                  d|| _        t        d|D cg c]  }|dz  	 c}d|d
   |d   t        j                  d|| _        t        d|D cg c]  }|dz  	 c}d|d   |d   t        j                  d|| _        t        j$                  t'        j(                  d
d
|d   fi |      | _        t        j$                  t'        j(                  d
d
|d
   fi |      | _        t        j$                  t'        j(                  d
d
|d   fi |      | _        t        j$                  t'        j(                  d
d
|d   fi |      | _        t3        d|d   dd|| _        t3        d|d
   dd|| _        t3        d|d   dd|| _        t3        d|d   dd|| _        t=        d|d   |z  ||d|| _        t=        d|d
   |z  ||d|| _         t=        d|d   |z  ||d|| _!        t=        d|d   |z  ||d|| _"        |}tG        ||
||||      }t        jH                  tK        |d         D cg c].  }tM        d|d   |	d   | j4                  | j>                  d||0 c}      | _'        t        jH                  tK        |d
         D cg c].  }tM        d|d
   |	d
   | j6                  | j@                  d||0 c}      | _(        t        jH                  tK        |d         D cg c].  }tM        d|d   |	d   | j8                  | jB                  d||0 c}      | _)        t        jH                  tK        |d         D cg c].  }tM        d|d   |	d   | j:                  | jD                  d||0 c}      | _*        || _+        | jV                  dkD  rmt        jH                  tK        |      D cg c]?  }tY        d||	| j>                  | j@                  | jB                  | jD                  fd||A c}      | _-        nd | _-        | j                  sX| jZ                  # ||d
   fi || _.         ||d   fi || _/        nd x| _.        | _/         ||d   fi || _0        | jV                  dkD  r|d
   |d   cxk(  r	|d   k(  sJ  J t'        j                  jb                  ddd
d
d|| _2        t        jf                  |      | _4        |dkD  r!t        jj                  | j
                  |fi |nt        jl                         | _7        n`d | _2        t        jf                  |      | _4        |dkD  r!t        jj                  | j
                  |fi |nt        jl                         | _7        tq        | j*                  d       tq        | j,                  d       tq        | j.                  d       tq        | j0                  d       | js                  | jt                         y c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w )Nr   r   avgr!   rk   )rk         r@   r   )r   r   r   	embed_dimr   ru   r   r}   r   )rC   rz   )r   r   r   )r   rf   rh   rg   r   r   )rC   r   r   ri   )r   r   r   )in_channelsout_channelsr"   {Gz?stdr   );r&   r'   r   r   r   num_featureshead_hidden_sizer   r   r   r   r   r,   r   patch_embed1patch_embed2patch_embed3patch_embed4	ParameterrG   zeros
cls_token1
cls_token2
cls_token3
cls_token4r   cpe1cpe2cpe3cpe4r   crpe1crpe2crpe3crpe4r*   r-   ranger   serial_blocks1serial_blocks2serial_blocks3serial_blocks4r   r   parallel_blocksr   norm3norm4Conv1d	aggregaterp   	head_droprn   r   headr   apply_init_weights)r4   r   r   r   r   r   r   r   r   r   rf   r   r   r   r   r   r   r   r   r   r   r    r5   r:   dprskwargsr   r;   s                              r<   r'   zCoaT.__init__  sS   0 	/....!7qQ%7$8!($4>rNBD1& & X&& D*x mD@BD ' D&./a1f/A
ST mD@BD ' D&./a1f/A
ST mD@BD ' D'/0!a2g0QTU mD@BD
 ,,u{{1aA'M"'MN,,u{{1aA'M"'MN,,u{{1aA'M"'MN,,u{{1aA'M"'MN <:a=A<<	<:a=A<<	<:a=A<<	<:a=A<<	 #vJqMY,FR[dovsuv
"vJqMY,FR[dovsuv
"vJqMY,FR[dovsuv
"vJqMY,FR[dovsuv
$$!
 !mm =+,	-.   qM$Q-99 JJ	
  	-. 

 !mm =+,	-.   qM$Q-99 JJ	
  	-. 

 !mm =+,	-.   qM$Q-99 JJ	
  	-. 

 !mm =+,	-.   qM$Q-99 JJ	
  	-. 

 -"#%== ~.20   #)"&**djj$**djj!Q 	
 20 	$D  $(D  ((##/'
1<<
'
1<<
*..
TZ#JqM8R8DJ""Q&!!}
1FAFFFFF!&!dQQ\]!dac!d!#I!6OZ]^BIId&7&7KKdfdododq	 "&!#I!6OZ]^BIId&7&7KKdfdododq	 	doo3/doo3/doo3/doo3/

4%%&G 0 0 1@	-.	-.	-.	-.20s1   [
[
[
3[3[=3[3[A[$c                    t        |t        j                        rjt        |j                  d       t        |t        j                        r8|j
                  +t        j                  j                  |j
                  d       y y y t        |t        j                        rUt        j                  j                  |j
                  d       t        j                  j                  |j                  d       y y )Nr   r   r   r   )	r(   r,   rn   r   weightrl   init	constant_r   )r4   ms     r<   r"  zCoaT._init_weights3  s    a#!((,!RYY'AFF,>!!!&&!, -?'2<<(GGaffa(GGahh, )r]   c                 
    h dS )N>   r	  r
  r  r  r   r4   s    r<   no_weight_decayzCoaT.no_weight_decay<  s    GGr]   c                     |rJ d       y )Nz$gradient checkpointing not supportedr   )r4   enables     r<   set_grad_checkpointingzCoaT.set_grad_checkpointing@  s    AAAz6r]   c                 2    t        ddddddddd	d
g	      }|S )Nz#^cls_token1|patch_embed1|crpe1|cpe1z^serial_blocks1\.(\d+)z#^cls_token2|patch_embed2|crpe2|cpe2z^serial_blocks2\.(\d+)z#^cls_token3|patch_embed3|crpe3|cpe3z^serial_blocks3\.(\d+)z#^cls_token4|patch_embed4|crpe4|cpe4z^serial_blocks4\.(\d+))z^parallel_blocks\.(\d+)N)z^norm|aggregate)i )	stem1r  stem2r  stem3r  stem4r  r  )r*   )r4   coarsematchers      r<   group_matcherzCoaT.group_matcherD  s6    848484842.
 r]   returnc                     | j                   S r   )r   r+  s    r<   get_classifierzCoaT.get_classifierV  s    yyr]   c                     || _         ||dv sJ || _        |dkD  r&t        j                  | j                  |      | _        y t        j
                         | _        y )Nr   r   )r   r   r,   rn   r  r   r   )r4   r   r   s      r<   reset_classifierzCoaT.reset_classifierZ  sU    &""2222*DALqBIId//=	VXVaVaVc	r]   c                 H
   |j                   d   }| j                  |      }| j                  j                  \  }}t        || j                        }| j
                  D ]  } ||||f      } t        |      j                  |||d      j                  dddd      j                         }| j                  |      }| j                  j                  \  }	}
t        || j                        }| j                  D ]  } |||	|
f      } t        |      j                  ||	|
d      j                  dddd      j                         }| j                  |      }| j                  j                  \  }}t        || j                        }| j                  D ]  } ||||f      } t        |      j                  |||d      j                  dddd      j                         }| j!                  |      }| j                   j                  \  }}t        || j"                        }| j$                  D ]  } ||||f      } t        |      j                  |||d      j                  dddd      j                         }| j&                  t(        j*                  j-                         s\| j.                  rPi }d| j0                  v r||d<   d| j0                  v r||d<   d	| j0                  v r||d	<   d
| j0                  v r||d
<   |S | j3                  |      }|S | j&                  D ]\  }| j5                  ||	|
f      | j7                  |||f      | j9                  |||f      }}} |||||||f|	|
f||f||fg      \  }}}}^ t(        j*                  j-                         sQ| j.                  rDi }d| j0                  v rBt        |      j                  |||d      j                  dddd      j                         }||d<   d| j0                  v rBt        |      j                  ||	|
d      j                  dddd      j                         }||d<   d	| j0                  v rBt        |      j                  |||d      j                  dddd      j                         }||d	<   d
| j0                  v rBt        |      j                  |||d      j                  dddd      j                         }||d
<   |S | j;                  |      }| j=                  |      }| j3                  |      }|||gS )Nr   rv   r@   rk   r   r!   x1_noclsx2_noclsx3_noclsx4_nocls)r   )rD   r  	grid_size
insert_clsr	  r  
remove_clsrF   rw   
contiguousr  r
  r  r  r  r  r  r  r  r  rG   jitis_scriptingr   r   r  r  r  r  r   r  )r4   x0rO   r   H1W1blkr>  r   H2W2r?  r   H3W3r@  r   H4W4rA  feat_outs                        r<   forward_featureszCoaT.forward_featuresa  s   HHQK r""",,BDOO,&& 	(CRr2h'B	(b>))!RR8@@Aq!LWWY x("",,BDOO,&& 	(CRr2h'B	(b>))!RR8@@Aq!LWWY x("",,BDOO,&& 	(CRr2h'B	(b>))!RR8@@Aq!LWWY x("",,BDOO,&& 	(CRr2h'B	(b>))!RR8@@Aq!LWWY '99))+0I0I!2!22+3HZ(!2!22+3HZ(!2!22+3HZ(!2!22+3HZ( ZZ^	 '' 	aC2Bx0$))BR2I499UWZ\^`YaKbBB RRR2r(RQSHWY[]V^7_`NBB	a yy%%'D,E,EHT...%b>11!RR@HHAqRST__a'/$T...%b>11!RR@HHAqRST__a'/$T...%b>11!RR@HHAqRST__a'/$T...%b>11!RR@HHAqRST__a'/$OBBBBBBB<r]   x_feat
pre_logitsc           	      >   t        |t              r| j                  J | j                  dk(  r@t	        j
                  |D cg c]  }|d d dd f   j                  dd        c}d      }n,t	        j                  |D cg c]  }|d d df    c}d      }| j                  |      j                  d      }n3| j                  dk(  r|d d dd f   j                  d      n|d d df   }| j                  |      }|r|S | j                  |      S c c}w c c}w )Nr   r   T)rC   keepdimrB   r   )r(   listr  r   rG   rJ   meanstacksqueezer  r   )r4   rT  rU  xlr:   s        r<   forward_headzCoaT.forward_head  s    fd#>>---5(IIVTrr!QR%y~~!T~BTZ[\KKF ;bAqD ;Cq!))a)0A-1-=-=-Fq!"u""q")FSTVWSWLANN1q0DIIaL0 U ;s   #DDc                     t         j                  j                         s| j                  r| j	                  |      S | j	                  |      }| j                  |      }|S r   )rG   rF  rG  r   rS  r]  )r4   r:   rT  s      r<   r\   zCoaT.forward  sR    yy%%'D,E,E((++ **1-F!!&)AHr]   )TFr   ) r^   r_   r`   ra   r   r)   r   r   r   r   r,   r   r   r   strr*   r'   r"  rG   rF  ignorer,  r/  r7  r:  r<  rS  r   Tensorr]  r\   rb   rc   s   @r<   r   r     sD      #4G7C"#<H!!$&$&$&*3).04*.&-m'm' m' 	m'
 m' c3S01m' !c3!34m'  m' m' eUE589m' m' m' "m' "m' "m'  RYY!m'" #'#m'$ #49-%m'& "$'m'( )m'^- YYH H YYB B YY " YY		  dC dhsm dO b15tELL7I)I#J 1X\ 1ELL r]   c                 x    |j                  | j                  d   dd      }t        j                  || fd      } | S )z Insert CLS token. r   r@   r   rB   )expandrD   rG   rJ   )r:   r   
cls_tokenss      r<   rC  rC    s7    !!!''!*b"5J		:q/q)AHr]   c                     | ddddddf   S )z Remove CLS token. Nr   r   )r:   s    r<   rD  rD    s    QAX;r]   c                    i }| j                  d|       } | j                         D ]  \  }}|j                  d      s|j                  d      rt        |dd       x|j                  d      rt        |dd       Z|j                  d      rt        |dd       <|j                  d      rt        |dd       |j                  d      rt        |dd       |||<    |S )Nmodelr   r   r  r  r  r   )getr0   
startswithgetattr)
state_dictrh  out_dictrz   rN   s        r<   checkpoint_filter_fnrn    s    H4J  " 	1<< g&75'4+H+Pg&75'4+H+Pg&75'4+H+Pk*wuk4/P/Xf%'%*F*N	 Or]   c                 p    |j                  dd       rt        d      t        t        | |fdt        i|}|S )Nfeatures_onlyz<features_only not implemented for Vision Transformer models.pretrained_filter_fn)ri  RuntimeErrorr   r   rn  )variant
pretraineddefault_cfgkwargsrh  s        r<   _create_coatrw    sJ    zz/4(YZZ  2	
 E Lr]   c                 4    | ddd dddt         t        dddd	|S )
Nr   )rk   r   r   g?bicubicTzpatch_embed1.projr   z
apache-2.0)urlr   
input_size	pool_sizecrop_pctinterpolationfixed_input_sizerY  r   
first_conv
classifierlicenser	   )rz  rv  s     r<   	_cfg_coatr    s5    =t%.B)  r]   ztimm/)	hf_hub_id)rk     r  r   squash)r  r{  r}  	crop_mode)zcoat_tiny.in1kzcoat_mini.in1kzcoat_small.in1kzcoat_lite_tiny.in1kzcoat_lite_mini.in1kzcoat_lite_small.in1kzcoat_lite_medium.in1kzcoat_lite_medium_384.in1kr8  c           	      Z    t        dg dg dd      }t        dd| it        |fi |}|S )Nru   )   r  r  r  r!   r!   r!   r!   r   r   r   r   r   rt  )	coat_tinyr*   rw  rt  rv  	model_cfgrh  s       r<   r  r    :    !5\bceIYYtI?XQW?XYELr]   c           	      Z    t        dg dg dd      }t        dd| it        |fi |}|S )Nru   )r     r  r  r  r   r  rt  )	coat_minir  r  s       r<   r  r    r  r]   c           	      \    t        ddg dg ddd|}t        dd| it        |fi |}|S )	Nru   )r  r   r   r   r  r   r  rt  r   )
coat_smallr  r  s       r<   r  r    sI     o!5\bcogmoIZ*ZY@YRX@YZELr]   c           	      ^    t        dg dg dg d      }t        dd| it        |fi |}|S )Nru   )r   r      r   r  r}   r}   ru   ru   r   r   r   r   rt  )coat_lite_tinyr  r  s       r<   r  r  '  :    !4L]ikI^j^DD]V\D]^ELr]   c           	      ^    t        dg dg dg d      }t        dd| it        |fi |}|S )Nru   r   r  r  r  rt  )coat_lite_minir  r  s       r<   r  r  /  r  r]   c           	      ^    t        dg dg dg d      }t        dd| it        |fi |}|S )Nru   r   r   r  r  rt  )coat_lite_smallr  r  s       r<   r  r  7  s:    !4L]ikI_z_T)E^W]E^_ELr]   c           	      X    t        dg dg d      }t        dd| it        |fi |}|S )Nru   r   r  r   r   rk   r   
   r}   )r   r   r   rt  )coat_lite_mediumr  r  s       r<   r  r  ?  s7    !5]TI`
`d9F_X^F_`ELr]   c           	      Z    t        ddg dg d      }t        dd| it        |fi |}|S )Nr  ru   r  r  )r   r   r   r   rt  )coat_lite_medium_384r  r  s       r<   r  r  G  s:    /CS`bIdJd$yJc\bJcdELr]   )FN)r?   r_  )4ra   typingr   r   r   r   r   r   rG   torch.nnr,   torch.nn.functional
functionalrK   	timm.datar
   r   timm.layersr   r   r   r   r   r   r   _builderr   	_registryr   r   __all__r   r   re   r   r   r   r   rC  rD  rn  rw  r  default_cfgsr  r  r  r  r  r  r  r  r   r]   r<   <module>r     s   ; :     A _ _ _ * <(HBII HV2bii 2j D<")) <~FBII FRE299 EP

 	 %'2'2 73$w7$w7%8&9!* 3("&  T   T   d   $   $   4   D     r]   