
    ^jmJ                        d Z ddlmZ ddlmZmZmZmZ ddlZddl	m
Z
 ddlmZmZ ddlmZmZmZmZmZ ddlmZ dd	lmZmZmZ dd
lmZmZ ddlmZmZ ddl m!Z!  G d de
jD                        Z#d Z$dbdZ%dcdZ&	 	 dddee'ejP                  f   de!de'de)dee'ejP                  f   f
dZ*dedZ+dfdZ, ei d e,dddd      d  e,d!ddd"d#d$      d% e,d&dd'      d( e,d)dd"d#d*      d+ e,       d, e,d-dd"d#.      d/ e,d0dd'      d1 e,d2dd"d#d*      d3 e,d4dd5d6dd7      d8 e,d9dd5d6d:      d; e,ddd6<      d= e,d>dd5d6d:      d? e,eed@A      dB e,eed@A      dC e,eed@A      dD e,eed@A      dE e,dFdGdHdIdJdKdLM       e,dNdOdIdHdJdKdLP       e,ddIdJdKdLdHQ      dR      Z-edgde!fdS       Z.edgde!fdT       Z/edgde!fdU       Z0edgde!fdV       Z1edgde!fdW       Z2edgde!fdX       Z3edgde!fdY       Z4edgde!fdZ       Z5edgde!fd[       Z6edgde!fd\       Z7edgde!fd]       Z8edgde!fd^       Z9edgde!fd_       Z:edgde!fd`       Z; ee<d3d8d;d;d=d,da       y)ha   Hybrid Vision Transformer (ViT) in PyTorch

A PyTorch implement of the Hybrid Vision Transformers as described in:

'An Image Is Worth 16 x 16 Words: Transformers for Image Recognition at Scale'
    - https://arxiv.org/abs/2010.11929

`How to train your ViT? Data, Augmentation, and Regularization in Vision Transformers`
    - https://arxiv.org/abs/2106.10270

NOTE These hybrid model definitions depend on code in vision_transformer.py.
They were moved here to keep file sizes sane.

Hacked together by / Copyright 2020, Ross Wightman
    )partial)DictTupleTypeUnionN)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STD)StdConv2dSame	StdConv2dConvNormAct	to_ntupleHybridEmbed   )build_model_with_cfg)generate_default_cfgsregister_modelregister_model_deprecations)	resnet26d	resnet50d)ResNetV2create_resnetv2_stem)VisionTransformerc                       e Zd Zddddddej                  ej
                  ddf
dededeeeed	f   f   d
eeeed	f   f   deeeed	f   f   dee	eeed	f   f   de
ej                     de
ej                     f fdZ xZS )ConvStem   @   )   r   r    Nin_chansdepthchannels.kernel_sizestridepadding
norm_layer	act_layerc                 $   |	|
d}t         |           t        |t              r.t	        t        |      D cg c]
  }|d|z  z   c}d d d         } t        |      |      } t        |      |      }|t        |      cxk(  rt        |      cxk(  rt        |      k(  sJ  J |}t        t        |            D ]M  }|t        |      dz
  k(  }| j                  | t        |||   f||   ||   ||   || | ||d|       ||   }O y c c}w )Ndevicedtyper   r   )r"   r#   r$   bias
apply_norm	apply_actr%   r&   )
super__init__
isinstanceinttupleranger   len
add_moduler   )selfr   r    r!   r"   r#   r$   r%   r&   r)   r*   ddiin_chs	last_conv	__class__s                  p/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/vision_transformer_hybrid.pyr0   zConvStem.__init__!   s7    /h$eE1h!Q$.EddKLH&i&{3")E"7+FHs;'7H3x=HHHHHs8}% 	!AS]Q..IOOqcK% (Nay
(='-%#% %  a[F	! Fs   D)__name__
__module____qualname__nnBatchNorm2dReLUr2   r   r   strr   Moduler0   __classcell__)r<   s   @r=   r   r       s     46782;8:*,..)+'!'! '! CsCx01	'!
 sE#s(O34'! #uS#X./'! 3U38_45'! RYY'! BII'! '!    r   c                  L    | j                  dd       | j                  dd       dS )Nr)   r*   r(   )get)kwargss    r=   _dd_from_kwargsrK   K   s#    jj406::gt;TUUrG   c                 B   |j                  dd      }|rdnd}|rt        t        d      nt        t        d      }t	        |       r.t        d| dd|j                  dd	      d
||dt        di |}|S t        |j                  dd	      f|d
|dt        di |}|S )z ResNet-V2 backbone helperpadding_sameTsamer   g:0yE>)epsr   r   r   F)layersnum_classesglobal_poolr   preact	stem_type
conv_layer)rT   rS   rU    )rI   r   r
   r   r5   r   rK   r   )rP   rJ   rM   rT   rU   backbones         r=   	_resnetv2rX   O   s    ::nd3L&BI5AD1wy^bGcJ
6{ 	
ZZ
A.!	
 ''	
$ O (JJz1%
!	

 ''
 OrG   c                 "   i }| j                         D ]x  \  }}|j                  |      s|j                  |d      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd	      }|j                  d
d      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|dk(  rd}|j                  d      }d|v rU|j                  dd      }|j                  dd      }|j                  }t        j                  |j                  d         ||<   |||<   { |S ) Nr   z
patch_emb.zpatch_embed.backbone.z
block.convconvz
block.normbnzpost_transformer_norm.znorm.zpre_norm_mha.0norm1zpre_norm_mha.1attnzpre_norm_ffn.0norm2zpre_norm_ffn.1zmlp.fc1zpre_norm_ffn.4zmlp.fc2z	qkv_proj.zqkv.z	out_proj.zproj.ztransformer.zblocks.zpos_embed.pos_embed.pos_embed	pos_embedr   zclassifier.projz	head.biaszhead.weight)items
startswithreplacesqueezeTtorchzerosshape)
state_dictmodelprefixoutkvbias_ks          r=   _convert_mobileclipro   j   s   
C  " 1||F#IIfb!IIl$;<IIlF+IIlD)II.8II&0II&/II&0II&	2II&	2IIk6*IIk7+IIni0//A		!A!YY0+>F		+];AA++aggaj1CKA12 JrG   Trh   ri   interpolation	antialiasreturnc                 F    ddl m} d| v rt        | |      }  || |||      S )Nr   )checkpoint_filter_fnz1image_encoder.model.patch_emb.0.block.conv.weight)rp   rq   )vision_transformerrt   ro   )rh   ri   rp   rq   
_filter_fns        r=   rt   rt      s.     G:jH(U;
j%}PYZZrG   c                     |j                  dd      }|xs i }t        t        fd|i|}|j                  d|       |j                  dd       t	        t
        | |ft        t        |d      d	|S )
Nout_indicesr   rW   embed_layer
patch_sizer   getter)rx   feature_cls)pretrained_filter_fnfeature_cfg)popr   r   
setdefaultr   r   rt   dict)variantrW   
embed_args
pretrainedrJ   rx   ry   s          r=   !_create_vision_transformer_hybridr      s    **]A.K!rJ+GGJGK
m[1
lA& 2[hG  rG   c                 $    | ddd dddddddd	d
|S )Ni  )r      r   ?bicubicT)      ?r   r   zpatch_embed.backbone.stem.convheadz
apache-2.0)urlrQ   
input_size	pool_sizecrop_pctrp   fixed_input_sizemeanstd
first_conv
classifierlicenserV   )r   rJ   s     r=   _cfgr      s4    =t6f  rG   z*vit_tiny_r_s16_p8_224.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/R_Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npzztimm/zpatch_embed.backbone.conv)r   	hf_hub_idcustom_loadr   z*vit_tiny_r_s16_p8_384.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/R_Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npz)r     r         ?)r   r   r   r   r   r   z*vit_small_r26_s32_224.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/R26_S_32-i21k-300ep-lr_0.001-aug_light0-wd_0.03-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.03-res_224.npz)r   r   r   z*vit_small_r26_s32_384.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/R26_S_32-i21k-300ep-lr_0.001-aug_medium2-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npz)r   r   r   r   r   zvit_base_r26_s32_224.untrainedz'vit_base_r50_s16_384.orig_in21k_ft_in1kzthttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_vit_base_resnet50_384-9fd3c705.pth)r   r   r   r   z*vit_large_r50_s32_224.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/R50_L_32-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_224.npzz*vit_large_r50_s32_384.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/R50_L_32-i21k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_384.npzz"vit_tiny_r_s16_p8_224.augreg_in21kzohttps://storage.googleapis.com/vit_models/augreg/R_Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0.npziSU  r   )r   r   rQ   r   r   r   z"vit_small_r26_s32_224.augreg_in21kzshttps://storage.googleapis.com/vit_models/augreg/R26_S_32-i21k-300ep-lr_0.001-aug_medium2-wd_0.03-do_0.0-sd_0.0.npz)r   r   rQ   r   r   zvit_base_r50_s16_224.orig_in21k)r   rQ   r   z"vit_large_r50_s32_224.augreg_in21kzrhttps://storage.googleapis.com/vit_models/augreg/R50_L_32-i21k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.0-sd_0.0.npzz!vit_small_resnet26d_224.untrainedzpatch_embed.backbone.conv1.0)r   r   r   z%vit_small_resnet50d_s16_224.untrainedz vit_base_resnet26d_224.untrainedz vit_base_resnet50d_224.untrainedzvit_base_mci_224.apple_mclip_ltzapple/mobileclip_b_lt_timmzYhttps://docs-assets.developer.apple.com/ml-research/datasets/mobileclip/mobileclip_blt.ptz
apple-amlri   )        r   r   )r   r   r   zpatch_embed.backbone.0.conv)r   r   r   rQ   r   r   r   zapple/mobileclip_b_timmzWhttps://docs-assets.developer.apple.com/ml-research/datasets/mobileclip/mobileclip_b.pt)r   r   rQ   r   r   r   r   )r   rQ   r   r   r   r   )zvit_base_mci_224.apple_mclipz%vit_base_mci_224.apple_mclip2_dfndr2bc           	      p    t        dddi|}t        dddd      }t        	 d	|| dt        |fi |}|S )
z3 R+ViT-Ti/S16 w/ 8x8 patch hybrid @ 224 x 224.
    rP   rV            r   rz   	embed_dimr    	num_headsrW   r   )vit_tiny_r_s16_p8_224rX   r   r   r   rJ   rW   
model_argsri   s        r=   r   r   	  Y     --f-HcqIJ-i*2ziMQR\Mg`fMgiELrG   c           	      p    t        dddi|}t        dddd      }t        	 d	|| dt        |fi |}|S )
z3 R+ViT-Ti/S16 w/ 8x8 patch hybrid @ 384 x 384.
    rP   rV   r   r   r   r   r   r   )vit_tiny_r_s16_p8_384r   r   s        r=   r   r     r   rG   c           	      j    t        di |}t        ddd      }t        	 d|| dt        |fi |}|S ) R26+ViT-S/S32 hybrid.
    r   r      r   r    r   r   )r   r   r   r   )vit_small_r26_s32_224r   r   s        r=   r   r     R     00H2;J-i*2ziMQR\Mg`fMgiELrG   c           	      j    t        di |}t        ddd      }t        	 d|| dt        |fi |}|S )r   r   r   r   r   r   r   )vit_small_r26_s32_384r   r   s        r=   r   r   *  r   rG   c           	      j    t        di |}t        ddd      }t        	 d|| dt        |fi |}|S )z R26+ViT-B/S32 hybrid.
       r   r   r   r   )vit_base_r26_s32_224r   r   s        r=   r   r   5  sR     00H2<J-h)1jhLPQ[Lf_eLfhELrG   c           	      j    t        di |}t        ddd      }t        	 d|| dt        |fi |}|S )zR R50+ViT-B/S16 hybrid from original paper (https://arxiv.org/abs/2010.11929).
    r   r   r   r   )r      	   )vit_base_r50_s16_224r   r   s        r=   r   r   @  sR     -f-H2<J-h)1jhLPQ[Lf_eLfhELrG   c           	      j    t        di |}t        ddd      }t        	 d|| dt        |fi |}|S )z R50+ViT-B/16 hybrid from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    r   r   r   r   r   )vit_base_r50_s16_384r   r   s        r=   r   r   K  sR    
 -f-H2<J-h)1jhLPQ[Lf_eLfhELrG   c           	      j    t        di |}t        ddd      }t        	 d|| dt        |fi |}|S ) R50+ViT-L/S32 hybrid.
             r   r   )r   r   r   r   )vit_large_r50_s32_224r   r   s        r=   r   r   W  R     00HB"=J-i*2ziMQR\Mg`fMgiELrG   c           	      j    t        di |}t        ddd      }t        	 d|| dt        |fi |}|S )r   r   r   r   r   r   r   )vit_large_r50_s32_384r   r   s        r=   r   r   b  r   rG   c           	          t        d
| |j                  dd      ddgdt        d
i |}t        dddd      }t	        	 d|| d	t        |fi |}|S )zL Custom ViT small hybrid w/ ResNet26D stride 32. No pretrained weights.
    r   r   Tr   r   r   features_onlyrx   r   r   r   r    r   	mlp_ratior   rV   )vit_small_resnet26d_224r   rI   rK   r   r   r   s        r=   r   r   m  s      J*C	
 
#F
#H 1QGJ-!k,4kOST^OibhOikELrG   c           	          t        d	| |j                  dd      ddgdt        d	i |}t        dddd      }t	        	 d
|| dt        |fi |}|S )zV Custom ViT small hybrid w/ ResNet50D 3-stages, stride 16. No pretrained weights.
    r   r   Tr   r   r   r   r   rV   )vit_small_resnet50d_s16_224r   rI   rK   r   r   r   s        r=   r   r   ~  s      J*C	
 
#F
#H 1QGJ-%o08ZoSWXbSmflSmoELrG   c           	          t        d
| |j                  dd      ddgdt        d
i |}t        ddd      }t	        	 d|| d	t        |fi |}|S )zK Custom ViT base hybrid w/ ResNet26D stride 32. No pretrained weights.
    r   r   Tr   r   r   r   r   r   rV   )vit_base_resnet26d_224r   r   s        r=   r   r           J*C	
 
#F
#H 2<J- j+3
jNRS]NhagNhjELrG   c           	          t        d
| |j                  dd      ddgdt        d
i |}t        ddd      }t	        	 d|| d	t        |fi |}|S )K Custom ViT base hybrid w/ ResNet50D stride 32. No pretrained weights.
    r   r   Tr   r   r   r   r   r   rV   )vit_base_resnet50d_224r   r   s        r=   r   r     r   rG   c                     t        ddddd|j                  dd      t        j                  dt	        di |}t        dddd	
      }t        	 d|t        d      | dt        |fi |}|S )r   )r   r   r   )r   r   r   r   r   r   )r!   r#   r"   r$   r   r&   r   r   T)r   r    r   no_embed_classF)proj)rW   r   r   rV   )vit_base_mci_224)r   rI   rA   GELUrK   r   r   r   s        r=   r   r     s      &J*'' 
#F
#H 2DQJ-%-$E:J!%j!;F!;E LrG   )vit_tiny_r_s16_p8_224_in21kvit_small_r26_s32_224_in21kvit_base_r50_s16_224_in21kvit_base_resnet50_224_in21kvit_large_r50_s32_224_in21kvit_base_resnet50_384r   )zimage_encoder.model.)r   T)NF)r   )F)=__doc__	functoolsr   typingr   r   r   r   re   torch.nnrA   	timm.datar   r	   timm.layersr
   r   r   r   r   _builderr   	_registryr   r   r   resnetr   r   resnetv2r   r   ru   r   
Sequentialr   rK   rX   ro   rD   Tensorboolrt   r   r   default_cfgsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r>   rV   rG   r=   <module>r      s    + +   A U U * Y Y ( 4 1(!r}} (!VV6B '	[ell*+[ [ [ 	[
 
#u||
[ 	 % T&0$ f.	30T& 1$ f.=SVdh3jT& 1$ i3T&  1$ j 3D3B!T&( %df)T&* .t C 300+T&2 1$ i33T&< 1$ i 3D3=T&J )$}C4O]a+cKT&R )$ BCT+;ST&Z &t(%[T&b )$ ACT+;cT&n ("(<Ig*ioT&r ,T"(<Ig.isT&v '"(<Ig)iwT&z '"(<Ig)i{T&@ &t.g|8U(AT&N %)+e|8U% .2|8U	.]T& Tn 9J   9J   9J   9J   8I   8I   8I   9J   9J   ;L    ?P    :K    :K    4E  ( H#G#G"C#D#GF' rG   