
    ^j+A                        d Z 	 ddlmZmZmZmZ ddlZddlmZ ddl	m
Z
mZ ddlmZmZmZmZmZmZmZ ddlmZ ddlmZ dd	lmZmZ d
gZe G d dej8                               Z G d dej8                        Z G d dej8                        Z G d d
ej8                        Z ddZ!ddZ" e e"d       e"d       e"d      d      Z#edde fd       Z$edde fd       Z%edde fd       Z&y)a   ConViT Model

@article{d2021convit,
  title={ConViT: Improving Vision Transformers with Soft Convolutional Inductive Biases},
  author={d'Ascoli, St{'e}phane and Touvron, Hugo and Leavitt, Matthew and Morcos, Ari and Biroli, Giulio and Sagun, Levent},
  journal={arXiv preprint arXiv:2103.10697},
  year={2021}
}

Paper link: https://arxiv.org/abs/2103.10697
Original code: https://github.com/facebookresearch/convit, original copyright below

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
    )OptionalUnionTypeAnyNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)DropPathcalculate_drop_path_ratestrunc_normal_
PatchEmbedMlp	LayerNormHybridEmbed   )build_model_with_cfg)register_notrace_module)register_modelgenerate_default_cfgsConVitc                        e Zd Z	 	 	 	 	 	 	 ddedededededef fdZd Zd	 Zdd
Z	d Z
dedej                  fdZ xZS )GPSAdim	num_headsqkv_bias	attn_drop	proj_droplocality_strengthc	                 n   ||d}	t         |           || _        || _        ||z  }
|
dz  | _        || _        t        j                  ||dz  fd|i|	| _        t        j                  ||fd|i|	| _	        t        j                  |      | _        t        j                  ||fi |	| _        t        j                  d|fi |	| _        t        j                  |      | _        t        j                  t!        j"                  | j                  fi |	      | _        t!        j&                  di |	| _        y )Ndevicedtype         bias   )r   r   r   r&   )super__init__r   r   scaler   nnLinearqkvDropoutr   projpos_projr   	Parametertorchonesgating_paramzerosrel_indices)selfr   r   r   r   r   r   r!   r"   ddhead_dim	__class__s              ]/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/convit.pyr(   zGPSA.__init__)   s    /")#%
!2))Cq>x>2>39(9b9I.IIc3-"-			!Y5"5I.LLDNN)Ib)IJ).)F2)F    c                    |j                   \  }}}| j                  | j                  j                   d   |k7  r| j                  |      | _        | j                  |      }| j	                  |      j                  ||| j                  || j                  z        j                  dddd      }||z  j                  dd      j                  |||      }| j                  |      }| j                  |      }|S )Nr   r   r$   r&   )shaper6   get_rel_indicesget_attentionr-   reshaper   permute	transposer/   r   )r7   xBNCattnr-   s          r;   forwardzGPSA.forwardF   s    ''1a#t'7'7'='=a'@A'E#33A6D!!!$FF1IaDNNA4GHPPQRTUWXZ[\AX  A&..q!Q7IIaLNN1r<   c                    |j                   \  }}}| j                  |      j                  ||d| j                  || j                  z        j	                  ddddd      }|d   |d   }}| j
                  j                  |ddd      }| j                  |      j	                  dddd      }||j                  dd      z  | j                  z  }	|	j                  d      }	|j                  d      }| j                  j                  dddd      }
d	t        j                  |
      z
  |	z  t        j                  |
      |z  z   }||j                  d      j!                  d      z  }| j#                  |      }|S )
Nr$   r   r&   r      r         ?)r>   r,   rA   r   rB   r6   expandr0   rC   r)   softmaxr4   viewr2   sigmoidsum	unsqueezer   )r7   rD   rE   rF   rG   r,   qk	pos_scorepatch_scoregatingrH   s               r;   r@   zGPSA.get_attentionQ   sa   ''1aWWQZ1adnn9LMUUVWYZ\]_`bcd!ube1$$++Ar2r:	MM),44Q1a@	1;;r2..$**<!))b)1%%"%-	""''2q!4U]]6**k9EMM&<QT]<]]R **2..~~d#r<   c                     | j                  |      j                  d      }| j                  j                         d d d d df   dz  }t	        j
                  d||f      |j                  d      z  }|r||fS |S )Nr   rL         ?	nm,hnm->h)r@   meanr6   squeezer2   einsumsize)r7   rD   
return_mapattn_map	distancesdists         r;   get_attention_mapzGPSA.get_attention_mapa   sw    %%a(--a0$$,,.q!Rx8B>	||K)X)>?)..QRBSS>!Kr<   c                    | j                   j                  j                  j                  t	        j
                  | j                               d}t        | j                  dz        }|dz  dk(  r|dz
  dz  n|dz  }t        |      D ]  }t        |      D ]  }|||z  z   }d| j                  j                  j                  |df<   d||z
  z  |z  | j                  j                  j                  |df<   d||z
  z  |z  | j                  j                  j                  |df<     | j                  j                  xj                  | j                  z  c_        y )Nr   r\   r$   r   rL   )r-   weightdatacopy_r2   eyer   intr   ranger0   r   )r7   locality_distancekernel_sizecenterh1h2positions          r;   
local_initzGPSA.local_initj   s;     488!45$..B./*5/Q*>+/Q&KSTDT$ 	_BK( _b 009;$$))(A+69:b6k9JM^9^$$))(A+69:b6k9JM^9^$$))(A+6	_	_ 	!!T%;%;;!r<   num_patchesreturnc                     t        |dz        }t        j                  d||d      }t        j                  |t        j                        j                  dd      t        j                  |t        j                        j                  dd      z
  }|j                  ||      }|j                  |d      j                  |d      }|dz  |dz  z   }|j                  d      |d d d d d d df<   |j                  d      |d d d d d d df<   |j                  d      |d d d d d d df<   | j                  j                  j                  }| j                  j                  j                  }	|j                  ||		      S )
Nr\   r   r&   r"   rL   r   rN   r$   r    )rl   r2   r5   arangefloat32rR   repeatrepeat_interleaverU   r,   rh   r!   r"   to)
r7   ru   img_sizer6   indindxindyinddr!   r"   s
             r;   r?   zGPSA.get_rel_indicesx   sB   {b()kk![+qAXU]];@@BG,,xu}}=BB2qIJ 	 zz(H-$$X1$5GGVWGXqy419$"&.."3Aq!QJ"&.."3Aq!QJ"&.."3Aq!QJ&&$$~~V5~99r<   )   F        r   rO   NNF)__name__
__module____qualname__rl   boolfloatr(   rI   r@   rf   rt   r2   Tensorr?   __classcell__r:   s   @r;   r   r   '   s    
 "!!')GG G 	G
 G G  %G:	 <:3 :5<< :r<   r   c                   P     e Zd Z	 	 	 	 	 	 d	dededededef
 fdZd
dZd Z xZ	S )MHSAr   r   r   r   r   c                 :   ||d}t         
|           || _        ||z  }	|	dz  | _        t	        j
                  ||dz  fd|i|| _        t	        j                  |      | _        t	        j
                  ||fi || _	        t	        j                  |      | _
        y )Nr    r#   r&   r%   )r'   r(   r   r)   r*   r+   qkvr.   r   r/   r   )r7   r   r   r   r   r   r!   r"   r8   r9   r:   s             r;   r(   zMHSA.__init__   s     /")#%
99S#'??B?I.IIc3-"-	I.r<   c                 r   |j                   \  }}}| j                  |      j                  ||d| j                  || j                  z        j	                  ddddd      }|d   |d   |d   }	}}||j                  dd      z  | j                  z  }
|
j                  d      j                  d      }
t        |d	z        }t        j                  |t        j                  
      j                  dd      t        j                  |t        j                  
      j                  dd      z
  }|j                  ||      }|j                  |d      j                  |d      }|dz  |dz  z   }|d	z  }|j!                  |
j"                  |
j$                        }t        j&                  d||
f      |z  }|r||
fS |S )Nr&   r$   r   r   rK   rM   rL   rN   r\   rx   r]   )r>   r   rA   r   rB   rC   r)   rQ   r^   rl   r2   ry   rz   rR   r{   r|   r}   r!   r"   r`   )r7   rD   rb   rE   rF   rG   r   rV   rW   r-   rc   r~   r   r   r   r   rd   re   s                     r;   rf   zMHSA.get_attention_map   s   ''1ahhqk!!!Q4>>1;NOWWXY[\^_abdefa&#a&#a&a1B++tzz9###+003qBw<XU]];@@BG,,xu}}=BB2qIJ 	 zz(H-$$X1$5GGVWGXqy419$BJ	LL(..A	||K)X)>?!C>!Kr<   c                    |j                   \  }}}| j                  |      j                  ||d| j                  || j                  z        j	                  ddddd      }|j                  d      \  }}}||j                  dd      z  | j                  z  }	|	j                  d      }	| j                  |	      }	|	|z  j                  dd      j                  |||      }| j                  |      }| j                  |      }|S )	Nr&   r$   r   r   rK   rM   rL   rN   )r>   r   rA   r   rB   unbindrC   r)   rQ   r   r/   r   )
r7   rD   rE   rF   rG   r   rV   rW   r-   rH   s
             r;   rI   zMHSA.forward   s    ''1ahhqk!!!Q4>>1;NOWWXY[\^_abdef**Q-1aAKKB''4::5|||#~~d#AX  A&..q!Q7IIaLNN1r<   )r   Fr   r   NNr   )
r   r   r   rl   r   r   r(   rf   rI   r   r   s   @r;   r   r      sV     "!!// / 	/
 / /*0r<   r   c                        e Zd Zdddddej                  eddddfdeded	ed
edededede	ej                     de	ej                     dedef fdZd Z xZS )Block      @Fr   TrO   Nr   r   	mlp_ratior   r   r   	drop_path	act_layer
norm_layeruse_gpsar   c           	      ~   ||d}t         |            |	|fi || _        |
| _        | j                  rt	        |f|||||d|| _        nt        |f||||d|| _        |dkD  rt        |      nt        j                         | _
         |	|fi || _        t        ||z        }t        d||||d|| _        y )Nr    )r   r   r   r   r   )r   r   r   r   r   )in_featureshidden_featuresr   drop )r'   r(   norm1r   r   rH   r   r
   r*   Identityr   norm2rl   r   mlp)r7   r   r   r   r   r   r   r   r   r   r   r   r!   r"   r8   mlp_hidden_dimr:   s                   r;   r(   zBlock.__init__   s      /*r*
 ==#!##"3 DI #!## DI 1:B),BKKM*r*
S9_- 
*	

 
r<   c                     || j                  | j                  | j                  |                  z   }|| j                  | j                  | j	                  |                  z   }|S N)r   rH   r   r   r   r7   rD   s     r;   rI   zBlock.forward   sO    tyyA788txx

1677r<   )r   r   r   r*   GELUr   rl   r   r   r   Moduler(   rI   r   r   s   @r;   r   r      s      ""!!!)+*3!')0
0
 0
 	0

 0
 0
 0
 0
 BII0
 RYY0
 0
  %0
dr<   r   c            )           e Zd ZdZddddddddd	d
ddddddedddddfdededededededededededededededede	e
   deej                     d ed!ed"ef( fd#Zd$ Zej"                  j$                  d%        Zej"                  j$                  d/d&       Zej"                  j$                  d0d'       Zej"                  j$                  d(ej                  fd)       Zd1dede	e   fd*Zd+ Zd/d,efd-Zd. Z xZS )2r   zI Vision Transformer with support for patch or hybrid CNN input stage
          r&     tokeni      r   Fr   NrO   Tr~   
patch_sizein_chansnum_classesglobal_pool	embed_dimdepthr   r   r   	drop_ratepos_drop_rateproj_drop_rateattn_drop_ratedrop_path_ratehybrid_backboner   local_up_to_layerr   use_pos_embedc                    t         |           ||d}|dv sJ ||z  }|| _        || _        || _        || _        |x| _        x| _        | _        || _	        || _
        |t        |f|||d|| _        nt        d||||d|| _        | j                  j                  }|| _        t        j                   t#        j$                  dd|fi |      | _        t        j(                  |      | _        | j                  rGt        j                   t#        j$                  d||fi |      | _        t/        | j,                  d       t1        ||      }t        j2                  t5        |      D cg c]  }t7        d|||	|
||||   |||k  |d	
|  c}      | _         ||fi || _        t=        |d
d      g| _        t        j(                  |      | _         |d
kD  rt        jB                  ||fi |nt        jD                         | _#        t/        | j&                  d       | jI                  | jJ                         | jM                         D ]"  \  }}tO        |d      s|jQ                          $ y c c}w )Nr    ) avgr   )r~   r   r   )r~   r   r   r   r   )p{Gz?std)
r   r   r   r   r   r   r   r   r   r   r   head)num_chs	reductionmodulert   r   ))r'   r(   r   r   r   r   num_featureshead_hidden_sizer   r   r   r   patch_embedr   ru   r*   r1   r2   r5   	cls_tokenr.   pos_drop	pos_embedr   r   
ModuleListrm   r   blocksnormdictfeature_info	head_dropr+   r   r   apply_init_weightsnamed_moduleshasattrrt   )r7   r~   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r!   r"   r8   ru   dprinmr:   s                                r;   r(   zConVit.__init__  s   2 	/2222Y	& &!2ENNND1DN!2*&* !!#	 
  D  *  !%!#	 
  D &&22&ekk!Q	&HR&HI

]3\\%++ai*VSU*VWDN$..c2'>mm U|%%   ##!((a&%.."3 %% & y/B/	 ")qPQI.?JQBIIi;;TVT_T_Ta	dnn#.

4%%&&&( 	DAqq,'	/%%s   /#I7c                    t        |t        j                        rjt        |j                  d       t        |t        j                        r8|j
                  +t        j                  j                  |j
                  d       y y y t        |t        j                        rUt        j                  j                  |j
                  d       t        j                  j                  |j                  d       y y )Nr   r   r   rO   )	
isinstancer*   r+   r   rh   r%   init	constant_r   )r7   r   s     r;   r   zConVit._init_weights`  s    a#!((,!RYY'AFF,>!!!&&!, -?'2<<(GGaffa(GGahh, )r<   c                 
    ddhS )Nr   r   r   r7   s    r;   no_weight_decayzConVit.no_weight_decayi  s    [))r<   c                      t        dddg      S )Nz ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)i )stemr   )r   )r7   coarses     r;   group_matcherzConVit.group_matcherm  s    4-/CD
 	
r<   c                     |rJ d       y )Nz$gradient checkpointing not supportedr   )r7   enables     r;   set_grad_checkpointingzConVit.set_grad_checkpointingt  s    AAAz6r<   rv   c                     | j                   S r   )r   r   s    r;   get_classifierzConVit.get_classifierx  s    yyr<   c                     || _         ||dv sJ || _        |dkD  r&t        j                  | j                  |      | _        y t        j
                         | _        y )N)r   r   r   r   )r   r   r*   r+   r   r   r   )r7   r   r   s      r;   reset_classifierzConVit.reset_classifier|  sS    &""6666*D>IAoBIIdnnk:	SUS^S^S`	r<   c                    | j                  |      }| j                  r|| j                  z   }| j                  |      }| j                  j                  |j                  d   dd      }t        | j                        D ]5  \  }}|| j                  k(  rt        j                  ||fd      } ||      }7 | j                  |      }|S )Nr   rL   r   rN   )r   r   r   r   r   rP   r>   	enumerater   r   r2   catr   )r7   rD   
cls_tokensublks        r;   forward_featureszConVit.forward_features  s    QDNN"AMM!^^**1771:r2>
, 	FAsD***IIz1o15AA	 IIaLr<   
pre_logitsc                     | j                   r3| j                   dk(  r|d d dd f   j                  d      n|d d df   }| j                  |      }|r|S | j                  |      S )Nr   r   rN   r   )r   r^   r   r   )r7   rD   r   s      r;   forward_headzConVit.forward_head  s_    (,(8(8E(A!QR%!$qAwANN1q0DIIaL0r<   c                 J    | j                  |      }| j                  |      }|S r   )r   r   r   s     r;   rI   zConVit.forward  s'    !!!$a r<   r   )Tr   )r   r   r   __doc__r   rl   strr   r   r   r   r   r*   r   r(   r   r2   jitignorer   r   r   r   r   r   r   rI   r   r   s   @r;   r   r     s   
   #& !"!#%$&$&$&-1*3%&')"&/YY Y 	Y
 Y Y Y Y Y Y Y Y !Y "Y "Y  "!Y" &c]#Y$ RYY%Y&  #'Y(  %)Y*  +Yv- YY* * YY
 
 YYB B YY		  aC ahsm a1$ 1r<   c                 `    |j                  dd       rt        d      t        t        | |fi |S )Nfeatures_onlyz<features_only not implemented for Vision Transformer models.)getRuntimeErrorr   r   )variant
pretrainedkwargss      r;   _create_convitr    s0    zz/4(YZZFvFFr<   c                 0    | ddd t         t        ddddd
|S )Nr   )r&   r   r   Tzpatch_embed.projr   z
apache-2.0)
urlr   
input_size	pool_sizer^   r   fixed_input_size
first_conv
classifierlicenser   )r	  r  s     r;   _cfgr    s0    =t%.BX\(<	
  r<   ztimm/)	hf_hub_id)zconvit_tiny.fb_in1kzconvit_small.fb_in1kzconvit_base.fb_in1krv   c           	      T    t        dddd      }t        dd| dt        |fi |}|S )	N
   rO   0   rK   r   r   r   r   convit_tinyr  r  r   r   r  r  r  
model_argsmodels       r;   r  r    s;    rQPJf=Zf4PZKe^dKefELr<   c           	      T    t        dddd      }t        dd| dt        |fi |}|S )	Nr  rO   r  	   r  convit_smallr  r   r  r  s       r;   r  r    s;    rQPJg>jgDQ[Lf_eLfgELr<   c           	      T    t        dddd      }t        dd| dt        |fi |}|S )	Nr  rO   r  r   r  convit_baser  r   r  r  s       r;   r   r     s;    rRQJf=Zf4PZKe^dKefELr<   r   )r   )'r   typingr   r   r   r   r2   torch.nnr*   	timm.datar   r	   timm.layersr
   r   r   r   r   r   r   _builderr   _features_fxr   	_registryr   r   __all__r   r   r   r   r   r  r  default_cfgsr  r  r   r   r<   r;   <module>r*     s  ( . -   A s s s * 1 < * _:299 _: _:D:299 :z7BII 7tXRYY XvG %'2 73'2	&  v      v  r<   