
    ^jP                        d Z ddlmZmZmZmZ ddlZddlmZ ddl	m
Z
mZ ddlmZmZmZmZmZmZmZmZ ddlmZ ddlmZ dd	lmZmZ d
gZ G d dej:                        Z G d dej:                        Z G d dej:                        Z  G d d
ej:                        Z!ddZ"ddZ# e e#d       e#d      d      Z$edde!fd       Z%edde!fd       Z&y)z Visformer

Paper: Visformer: The Vision-friendly Transformer - https://arxiv.org/abs/2104.12533

From original at https://github.com/danczs/Visformer

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
    )OptionalUnionTypeAnyNIMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)	to_2tupletrunc_normal_DropPathcalculate_drop_path_rates
PatchEmbedLayerNorm2dcreate_classifieruse_fused_attn   )build_model_with_cfg)checkpoint_seq)register_modelgenerate_default_cfgs	Visformerc                        e Zd Zddej                  dddddfdedee   dee   deej                     d	e	d
ede
f fdZd Z xZS )
SpatialMlpN           Fin_featureshidden_featuresout_features	act_layerdropgroupspatial_convc
                    ||	d}
t         |           |xs |}|xs |}t        |      }|| _        || _        || _        | j
                  r|dk  r	|dz  dz  }n|dz  }|| _        || _        t        j                  ||dfdddd|
| _
         |       | _        t        j                  |d         | _        | j
                  r9t        j                  ||d	fdd| j                  dd
|
| _         |       | _        nd | _        d | _        t        j                  ||dfdddd|
| _        t        j                  |d         | _        y )Ndevicedtype         r   r   Fstridepaddingbias   )r+   r,   groupsr-   )super__init__r
   r   r   r"   r   r!   nnConv2dconv1act1Dropoutdrop1conv2act2conv3drop3)selfr   r   r   r   r    r!   r"   r%   r&   dd
drop_probs	__class__s               `/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/visformer.pyr1   zSpatialMlp.__init__   sX    /#2{)8[t_
&((qy"-/Q"6"-/.
YY{OQfqRSZ_fcef
K	ZZ
1.
!o<=qQUQ[Q[bgokmoDJ!DIDJDIYYagST[`gdfg
ZZ
1.
    c                    | j                  |      }| j                  |      }| j                  |      }| j                  "| j                  |      }| j	                  |      }| j                  |      }| j                  |      }|S N)r4   r5   r7   r8   r9   r:   r;   r<   xs     r@   forwardzSpatialMlp.forwardB   sl    JJqMIIaLJJqM::!

1A		!AJJqMJJqMrA   )__name__
__module____qualname__r2   GELUintr   r   Modulefloatboolr1   rF   __classcell__r?   s   @r@   r   r      s     .2*.)+!&'/'/ &c]'/ #3-	'/
 BII'/ '/ '/ '/R	rA   r   c                        e Zd ZU ej                  j
                  e   ed<   	 	 	 	 	 	 d	dedede	de	de	f
 fdZ
d Z xZS )
	Attention
fused_attndim	num_headshead_dim_ratio	attn_drop	proj_dropc                    ||d}t         
|           || _        || _        t	        ||z  |z        }	|	| _        |	dz  | _        t        d      | _        t        j                  ||	|z  dz  dfdddd	|| _        t        j                  |      | _        t        j                  | j
                  | j                  z  |dfdddd	|| _        t        j                  |      | _        y )
Nr$   g      T)experimentalr.   r   r   Fr*   )r0   r1   rT   rU   roundhead_dimscaler   rS   r2   r3   qkvr6   rW   projrX   )r<   rT   rU   rV   rW   rX   r%   r&   r=   r\   r?   s             r@   r1   zAttention.__init__Q   s     /"	)N:; %
(d;99S(Y"6":AeaQRY^ebdeI.IIdmmdnn<c1lQXY`elikl	I.rA   c                    |j                   \  }}}}| j                  |      j                  |d| j                  | j                  d      j                  ddddd      }|j                  d      \  }}}| j                  rzt        j                  j                  j                  |j                         |j                         |j                         | j                  r| j                  j                  nd      }nJ||j!                  d	d      z  | j"                  z  }	|	j%                  d
      }	| j                  |	      }	|	|z  }|j                  dddd      j                  |d||      }| j'                  |      }| j)                  |      }|S )Nr.   r   r   r'      r   )	dropout_p)rT   )shaper^   reshaperU   r\   permuteunbindrS   torchr2   
functionalscaled_dot_product_attention
contiguoustrainingrW   p	transposer]   softmaxr_   rX   )
r<   rE   BCHWqkvattns
             r@   rF   zAttention.forwardi   sG   WW
1aHHQK1dnndmmRHPPQRTUWXZ[]^_((1+1a??##@@.2mm$..** A A
 B++tzz9D<<B<'D>>$'DqAIIaAq!))!RA6IIaLNN1rA   )r         ?r   r   NN)rG   rH   rI   ri   jitFinalrN   __annotations__rK   rM   r1   rF   rO   rP   s   @r@   rR   rR   N   sd    		%%
 $&!!// / "	/
 / /0rA   rR   c                        e Zd Zdddddej                  edddddfdeded	ed
ededededeej                     deej                     dede
de
f fdZd Z xZS )Blockry         @r   r   FNrT   rU   rV   	mlp_ratiorX   rW   	drop_pathr   
norm_layerr!   attn_disabledr"   c           
      X   ||d}t         |           || _        |dkD  rt        |      nt	        j
                         | _        |rd | _        d | _        n$ |	|fi || _        t        |f||||d|| _         |	|fi || _
        t        d|t        ||z        |||
|d|| _        y )Nr$   r   )rU   rV   rW   rX   )r   r   r   r    r!   r"    )r0   r1   r"   r   r2   Identityr   norm1rx   rR   norm2r   rK   mlp)r<   rT   rU   rV   r   rX   rW   r   r   r   r!   r   r"   r%   r&   r=   r?   s                   r@   r1   zBlock.__init__   s    " /(09B),BKKMDJDI#C.2.DJ!#-## DI  *r*
 
i0%
 
rA   c                     | j                   2|| j                  | j                  | j                  |                  z   }|| j                  | j                  | j	                  |                  z   }|S rC   )rx   r   r   r   r   rD   s     r@   rF   zBlock.forward   sY    99 DNN499TZZ]#;<<Atxx

1677rA   )rG   rH   rI   r2   rJ   r   rK   rM   r   rL   rN   r1   rF   rO   rP   s   @r@   r~   r~      s    
 %'!!!!)+*5"'!&,
,
 ,
 "	,

 ,
 ,
 ,
 ,
 BII,
 RYY,
 ,
  ,
 ,
\rA   r~   c            1           e Zd Zddddddddd	d
d
d
d
d
eddddddddddfdededededee   dedeeef   dededededededede	e
j                     d ed!ed"ed#ed$ed%ed&ed'ee	e
j                        f. fd(Zd) Zej"                  j$                  d3d*       Zej"                  j$                  d4d+       Zej"                  j$                  d,e
j                  fd-       Zd5ded%efd.Zd/ Zd3d0efd1Zd2 Z xZS )6r         r.              r)   r   r   111TFr   avgNimg_size
patch_sizein_chansnum_classesinit_channels	embed_dimdepthrU   r   	drop_ratepos_drop_rateproj_drop_rateattn_drop_ratedrop_path_rater   
attn_stageuse_pos_embedr"   vit_stemr!   global_pool	conv_init
embed_normc                    t         |           ||d}t        |      }|| _        || _        || _        || _        || _        || _        || _	        t        |t        t        f      r!|\  | _        | _        | _        t!        |      }n2|dz  x| _        | _        || j                  z
  | j                  z
  | _        || _        d| _        t'        ||      }| j                  r2d | _        t+        d|||||dd|| _        |D cg c]  }||z  	 }}n| j                  :d | _        t+        d||dz  ||dz  |dd|| _        |D cg c]
  }||dz  z   }}nt/        j0                  t/        j2                  || j                  dfdddd|t/        j4                  | j                  fi |t/        j6                  d	            | _        |D cg c]  }|dz  	 }}t+        d||d
z  | j                  |dz  |dd|| _        |D cg c]
  }||d
z  z   }}| j"                  r| j                  r3t/        j8                  t;        j<                  d|g|i |      | _        n5t/        j8                  t;        j<                  d|dz  g|i |      | _        t/        j@                  |      | _!        nd | _        t/        j0                  tE        | j                        D cg c]+  }tG        d|dz  |d|	||||   |||d   dk(  |d   dk(  d|- c} | _$        | j                  syt+        d||dz  |dz  ||dd|| _%        |D cg c]
  }||dz  z   }}| j"                  r3t/        j8                  t;        j<                  d|g|i |      | _&        nd | _&        nd | _%        t/        j0                  tE        | j                  | j                  | j                  z         D cg c](  }tG        d||d|	||||   |||d   dk(  |d   dk(  d|* c} | _'        | j                  s|t+        d||dz  ||dz  |dd|| _(        |D cg c]
  }||dz  z   }}| j"                  r6t/        j8                  t;        j<                  d|dz  g|i |      | _)        nd | _)        nd | _(        t/        j0                  tE        | j                  | j                  z   |      D cg c]+  }tG        d|dz  |d|	||||   |||d   dk(  |d   dk(  d|- c} | _*        | j                  r|n|dz  x| _+        | _,         || jV                  fi || _-        t]        | jV                  | j                  |||      \  }}|| _/        t/        j@                  |
      | _0        || _1        | j"                  rQte        | j>                  d       | j                  s.te        | jL                  d       te        | jR                  d       | jg                  | jh                         y c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w )Nr$   r.   F)r   r   r   r   r   flattenr'      r*   T)inplacerb   r   )rn   g      ?r   01)rT   rU   rV   r   rX   rW   r   r   r!   r   r"   r   ry   	pool_typer%   r&   {Gz?stdr   )5r0   r1   r
   r   r   r   r   r   r   r   
isinstancelisttuple
stage_num1
stage_num2
stage_num3sumr   grad_checkpointingr   stemr   patch_embed1r2   
Sequentialr3   BatchNorm2dReLU	Parameterri   zeros
pos_embed1r6   pos_dropranger~   stage1patch_embed2
pos_embed2stage2patch_embed3
pos_embed3stage3num_featureshead_hidden_sizenormr   r   	head_dropheadr   apply_init_weights) r<   r   r   r   r   r   r   r   rU   r   r   r   r   r   r   r   r   r   r"   r   r!   r   r   r   r%   r&   r=   dprrE   ir   r?   s                                   r@   r1   zVisformer.__init__   s   8 	/X&& "*  "edE]+@E=DOT_doJE05
:DOdo#doo5GDO*"''>==DI * !!%!#%! !D 2::AZ:H:!!) 	$. %%)Q%'1n)!% %! =EEqA*/2EEMMIIh(:(:AeaQRY^ebdeNN4#5#5<<GGD)	
 -55qAF55$. %%)Q!//'1n)!% %! =EEqA*/2EE}}"$,,u{{1i/Y(/YVX/Y"Z"$,,u{{1il/\X/\Y[/\"]JJ7DM"DOmm 4??+&
   qL#"#((a&%)!}3*1o4 &
 & }} * !!%?"a#%! !D 9AA1jAo.AHA!!"$,,u{{1i/Y(/YVX/Y"Z"& $Dmm 4??DOODOO,KL&
   #"#((a&%)!}3*1o4 &
 & }} * !!%?"#a-%! !D 9AA1jAo.AHA!!"$,,u{{1ik/[H/[XZ/["\"& $Dmm 4??4??:EB&
   M#"#((a&%)!}3*1o4 &
 $ BFIT]`aTaaD1t007B7	 .!
T 'I.	 $//t4==doo48doo48

4%%&q ; F 6 F&
: B&
: B&
s6   XXXX$!0X)X.$-X3X80X=c                    t        |t        j                        rOt        |j                  d       |j
                  +t        j                  j                  |j
                  d       y y t        |t        j                        r| j                  r-t        j                  j                  |j                  dd       nt        |j                  d       |j
                  +t        j                  j                  |j
                  d       y y y )Nr   r   r   fan_outrelu)modenonlinearityr   )r   r2   Linearr   weightr-   init	constant_r3   r   kaiming_normal_)r<   ms     r@   r   zVisformer._init_weights  s    a#!((-vv!!!!&&!, "299%~~''yv'VahhD1vv!!!!&&"- " &rA   c                 .    t        d|rdndd fddg      S )Nz^patch_embed1|pos_embed1|stemz^stage(\d+)\.(\d+))z^(?:patch_embed|pos_embed)(\d+))r   )z^norm)i )r   blocks)dict)r<   coarses     r@   group_matcherzVisformer.group_matcher  s+    1*0&6KTR:$
 	
rA   c                     || _         y rC   )r   )r<   enables     r@   set_grad_checkpointingz Visformer.set_grad_checkpointing  s
    "(rA   returnc                     | j                   S rC   )r   )r<   s    r@   get_classifierzVisformer.get_classifier  s    yyrA   c                 T   || _         t        | j                  d      r | j                  j                  j                  nd }t        | j                  d      r | j                  j                  j
                  nd }t        | j                  | j                   |||      \  | _        | _        y )Nr   r   )	r   hasattrr   r   r%   r&   r   r   r   )r<   r   r   r%   r&   s        r@   reset_classifierzVisformer.reset_classifier  s    &,3DIIx,H!!((d*1$))X*F		  &&D&7t//;v]b'd#$)rA   c                    | j                   | j                  |      }| j                  |      }| j                  | j                  || j                  z         }| j                  r5t
        j                  j                         st        | j                  |      }n| j                  |      }| j                  ;| j                  |      }| j                  | j                  || j                  z         }| j                  r5t
        j                  j                         st        | j                  |      }n| j                  |      }| j                  ;| j                  |      }| j                  | j                  || j                  z         }| j                  r5t
        j                  j                         st        | j                  |      }n| j                  |      }| j!                  |      }|S rC   )r   r   r   r   r   ri   rz   is_scriptingr   r   r   r   r   r   r   r   r   rD   s     r@   forward_featureszVisformer.forward_features  s{   99 		!A a ??&a$//12A""599+A+A+Ct{{A.AAA (!!!$A*MM!doo"56""599+A+A+Ct{{A.AAA (!!!$A*MM!doo"56""599+A+A+Ct{{A.AAAIIaLrA   
pre_logitsc                 p    | j                  |      }| j                  |      }|r|S | j                  |      S rC   )r   r   r   )r<   rE   r   s      r@   forward_headzVisformer.forward_head  s5    QNN1q0DIIaL0rA   c                 J    | j                  |      }| j                  |      }|S rC   )r   r   rD   s     r@   rF   zVisformer.forward  s'    !!!$a rA   F)T)r   )rG   rH   rI   r   rK   r   r   r   rM   r   r2   rL   strrN   r1   r   ri   rz   ignorer   r   r   r   r   r   rF   rO   rP   s   @r@   r   r      s      #+- ')!!#%$&$&$&*5#"& %"$#485T'T' T' 	T'
 T' $C=T' T' e$T' T' T' T' !T' "T' "T' "T'  RYY!T'" #T'$  %T'& 'T'( )T'* +T', -T'. /T'0 !bii11T'l. YY
 
 YY) ) YY		  dC dc d"H1$ 1
rA   c                 d    |j                  dd       rt        d      t        t        | |fi |}|S )Nfeatures_onlyz<features_only not implemented for Vision Transformer models.)getRuntimeErrorr   r   )variant
pretraineddefault_cfgkwargsmodels        r@   _create_visformerr     s4    zz/4(YZZ GZJ6JELrA   c                 4    | ddddddt         t        ddd	d
|S )Nr   )r.   r   r   )r   r   g?bicubicTzstem.0r   z
apache-2.0)urlr   
input_size	pool_sizecrop_pctinterpolationfixed_input_sizemeanr   
first_conv
classifierlicenser   )r  r   s     r@   _cfgr    s5    =v%.Bf  rA   ztimm/)	hf_hub_id)zvisformer_tiny.in1kzvisformer_small.in1kr   c                     t        ddddddddt        j                  d	t        j                  
      }t        dd| it        |fi |}|S )Nr      r   rb   rb   r.   r   r   011100Tr   r   r   rU   r   r!   r   r"   r   r   r   r   )visformer_tinyr   r2   r   r   r   r   	model_cfgr   s       r@   r  r    sT    CyAQS[\uSW>>#I c:ciIb[aIbcELrA   c                     t        ddddddddt        j                  d	t        j                  
      }t        dd| it        |fi |}|S )Nr   r   r  r)   r   r   r  r  Tr  r   )visformer_smallr  r  s       r@   r  r    sT    CyAQS[\uSW>>#I dJd$yJc\bJcdELrA   )FN) r   )'__doc__typingr   r   r   r   ri   torch.nnr2   	timm.datar   r	   timm.layersr
   r   r   r   r   r   r   r   _builderr   _manipulater   	_registryr   r   __all__rL   r   rR   r~   r   r   r  default_cfgsr  r  r   rA   r@   <module>r%     s    . -   A R  R  R * ' <-3 3l.		 .b3BII 3lj		 jZ		 %'2 73&  )   9  rA   