
    ^j;                    @B   d Z ddlZddlZddlZddlZddlmZ ddlmZ ddl	m
Z
mZmZmZmZmZmZmZmZ 	 ddl	mZ ddlZddlmZ ddlmc mZ ddlmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$ dd	l%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z= d
dl>m?Z? d
dl@mAZA d
dlBmCZCmDZDmEZEmFZF d
dlGmHZHmIZImJZJ dgZK ej                  eM      ZNe&e&e'dZO	 	 	 	 	 	 	 	 dde<dePdePdeQdeQdeQdeQdeRdeRdeeej                        dePdej                  fdZT G d d ej                        ZU G d! d"ej                        ZV G d# d$ej                        ZW G d% d&ej                        ZX G d' d(ej                        ZY	 	 	 dd)ej                  d*e[d+ePd,eQfd-Z\ G d. dej                        Z]dd0ej                  d1e[d2eQddfd3Z^	 	 	 dd0ej                  d1e[d4eRd2eQddf
d5Z_dd0ej                  d1e[d2eQddfd6Z`dd0ej                  d1e[d2eQddfd7Zadd8e[d4eRd2eQdefd9Zb	 	 	 	 dd;ej                  d<ej                  d+ePd=eePePf   d>e[d?eQdej                  fd@Zc ej                         ddAe]dBe[dCe[dDeQddf
dE       Ze	 ddFee[ej                  f   dAe]dCe[dee[ej                  f   fdGZfdFee[ej                  f   dAe]dee[ej                  f   fdHZgdFee[ej                  f   dAe]dee[ej                  f   fdIZhdFeifdJZj	 	 	 ddFee[ej                  f   dAe]dKeQd>e[d?eQdee[ej                  f   fdLZkddMe[dee[e
f   fdNZli dO eldPQ      dR eldPQ      dS eldTdPdU      dV eldWdPddXdYZ      d[ eld\dPdU      d] eld^dPddXdYZ      d_ eld`dPdU      da eldbdPddXdYZ      dc eldddPdU      de eldfdPddXdYZ      dg eldhdPdU      di eldjdPddXdYZ      dk eldldPdU      dm eldndPdU      do eldpdPddXdYZ      dq eldrdPs      dt eldudPdXdYv      i dw eldxdPdXdYv      dy eldzdPdU      d{ eld|dPddXdYZ      d} eld~dPdU      d elddPddXdYZ      d elddPdU      d elddPddXdYZ      d eld/      d eld/      d eld/      d eld/      d eldPd      d eldPd      d eldPd      d eldPd      d eldPd      d elddPdd      i d elddPdd      d elddPdd      d elddPdd      d elddPdd      d elddPdd      d elddPdd      d eldddP      d eldddP      d elddPee d      d elddPee d      d elddPee d      d elddPee d      d elddPdee dddY      d elddPdee dddY      d elddPdee dddY      d elddPdee dddY      d elddPdee dddY      i d elddPdee dddY      d elddPdee dddY      d elddPdee dddY      d eldPddddddYū      d eldPddddddYū      d eldPddddddYū      d eldPddddddYū      d elddPdddddͬΫ      d elddPdddd̬ѫ      d elddPs      d eldPddd׬ث      d eldPdd֬۫      d eldPdXddݬޫ      d eldd֬      d el       d eldPe#e$      d eldPe#e$dYdX      i d eldPe#e$dYdĬ      d eldPe#e$d֬      d eldPe#e$dYdXdݬ      d eldPe!e"dY      d eldPe!e"dYddݬ      d eldPe#e$dY      d eldPe#e$dYddݬ      d ele#e$      d eldPe#e$ddXdݬ      d eldPe#e$d֬      d eldPe#e$ddXdݬ      d eldPe#e$dY      d eldPe#e$dYddݬ      d eldPe#e$      d eldPe#e$dY      d eldPe#e$dYdXdݬ      d eldPe!e"dY      i d eldPe!e"dYddݬ      d eldPe#e$dY      d eld/e#e$dYddݬ      d eldPe#e$      d eldPe#e$      d  eldPe#e$dYdXdݬ      d eldPe#e$dY      d eldPe#e$dא      d eldPe!e"dYdא      d eldPe#e$dYdא      d eldPe#e$dא      d eldPe#e$dYdא      d	 eldPe#e$d
      d eldPe#e$dYd
      d eldPe!e"dYd      d eldPe#e$dYd      d eldPe#e$dYd      i d eldPe#e$dYd      d eldPdde#e$d
      d eldPde#e$dYd
      d eldPde#e$ddYd      d eldPde#e$dYd      d eldPe#e$dYd
      d eldPe#e$dYdڐd
      d  eldPe#e$dYd
      d! eldPe#e$dYd      d" eldPd#e#e$dYd
      d$ eldPd#e#e$dYd      d% eldPd#de#e$dYd&      d' eldPd#de#e$dYd&      d( eldPe#e$d#ddYd)d*      d+ eldPd,de#e$dYd&      d- eldPd,e#e$d)dYdݐd.      d/ eldPd,e#e$dYd      i d0 eldPd,e#e$d)dYdݐd.      d1 eldPd,de#e$dYd
&      d2 eldPd,de#e$dYd
&      d3 eldPd,de#e$dYd&      d4 eldPd,de#e$dYd&      d5 eldPd,e#e$dYd      d6 eldPd,de#e$dYd&      d7 eldPd,de#e$dYd
&      d8 eldPd,de#e$dYd
&      d9 eldPd,de#e$dYd&      d: eldPde#e$d
;      d< eldPde#e$d
;      d= eldPde#e$dYd>      d? eldPde#e$dYdd@      dA eldPde#e$dYdBC      dD eld/dd֐E      dF eld/dd֐E      i dG eld/      dH eld/      dI eld/      dJ eldPde#e$dKdYL      dM eldPde#e$ddYdݐN      dO eldPde#e$dKdYL      dP eldPde#e$ddYdݐN      dQ eldRddPdd֐S      dT eldUddPdd֐S      dV eldWddPdd֐S      dX eldYddPdd֐S      dZ eld[ddPdd֐S      d\ eld]ddPdd֐S      d^ eld_ddPddd`      da eldbddPddd`      dc elddddPdd֐S      de eldfddPdd֐S      i dg eldhddPdd֐S      di eldjddPddd`      dk eldlddPddd`      dm eld/      dn eld/      do eld/      dp eldqdPd,ee dr      ds eldtdPd,ee dr      du eldvdPd,ee dr      dw eldxd,ee dy      dz eld{d,ee dy      d| eld}d,ddYee d~      d eldd,ee dy      d eldPdd      d eldPd      d eldPd      d eldPdd      i d eldPdd      d eldPdd      d eldPdXd      d eldPdXd      d eldPdd      d eldPdd      d eldPdd      d eldPdd      d eldPdXd      d eldPdXd      d eldPdd      d eldPd      d eldPd      d eldPd)d      d eldPd)d      d eldPdXd      d eldPdd      i d eldPdd      d eldPdXd      d eldPdd      d eldPdd      d eldPdXd      d eldPdd      d eldPd      d eldPd      d eldPdd      d eldPdd      d eldPdd      d eldPdXd      d eldPdXd      d eldPdd      d eldPdd      d eldPdd      d eldPdd      i d eldPdXd      d eldPdXd      d eldPdd      d eldPd      d eldPd      d eldPd      d eldPd      d eldPd      d eldPd      d eldPd)d      d eldPd)dYdث      d eldPdXdYdث      d eldPddYdث      d eldPddYdث      d eldPddYdث      d eldPddYdث      d eldPddYdث      i d eldPddYdث      d eldPddYdث      d eldPddYdث      d eldPddYdث      d eldPddYdث      d eldPddYdث      d eldPddYdث      d eldPddYdث      d eldPdd      d eldPdd      d eldPdXd      d eldPdd      d eldPdd      d eldPdXd      d eldPd)dYdݬޫ      d eldPd)dYdݬޫ      d eldPde#e$d
̫      i d eldPde#e$d
̫      d eldPde#e$d
̫      d eldPde#e$d
̫      d eldPdd֬۫      d eldPdd֬۫      d eldPdd֬۫      d eldPdd֬۫      d eldPdd֬۫      d eldPdd֬۫      d eldPdd֬۫      d eldPddd֐ث      d eldPdd֬۫      d eldPdd֬۫      d eldPdd֬۫      d eldPdd֬۫      d eldPdd֬۫      d eldPddd֐ث      i d eldPdd֬۫      d eldPdd֬۫      d eldPddd֐ث      d eldPddd֐ث      d eldPdXdY۫      d eldPdd֬۫      d eldPdd֬۫      d eldPdd֬۫      d eldPdd֬۫      d eldPddd֐ث      d eldPddd֐ث      d eldPdXdY۫      d eldڐ      d eldPdd֬۫      d eldPddd֐ث      d eldPdXdY۫      d eldڐ      i d eldPddY۫      d eldPdddYث      d eldPdXdY۫      d eldPddYdݬޫ      d eldPdee ddYd      d eldPee ddYd      d eldPe#e$d#dYd      d eldPe#e$d#dYd      d eldPe#e$d#dYd      d eldPe#e$d#dYd      d eldPe#e$d#dYd      d eldPe#e$d#ddYd      d  eldPe#e$d#ddYd      d eldPe#e$d#ddYd      d eldPe#e$d#ddYd      d eldPe#e$d#ddYd      d eldPe#e$d#ddYd      i d eldPe#e$d#ddYd      d eldPe#e$d#ddYd      d eldPe#e$d#ddYd      d eldPd	d֬۫      d
 eldPd	d֬۫      d eldPd	d֬۫      d eld	d֬      d eldPee dY      d eldPee dY      d eldPee dY      d eldPee dY      d eld/ee dY      d eld/dee dY      d eldPee dYd      d eldPee dYd      d eldPee dYd      d eldPee dYd      Zmemj                         D  cg c](  \  } }|j                  dd:      sd|d   d   v s'| * c}} ZpepD ]@  Zq ej                  emeq         Zsesd   dPk(  r	dPeqz   esd<   esemeqj                  dd      <   B  eHem      Zmej                  j                  dd      j                         d k(  Zw	 	 dd!e[d"eQd#eeQ   dee]d$f   fd%ZxeIdd"eQde]fd&       ZyeIdd"eQde]fd'       ZzeIdd"eQde]fd(       Z{eIdd"eQde]fd)       Z|eIdd"eQde]fd*       Z}eIdd"eQde]fd+       Z~eIdd"eQde]fd,       ZeIdd"eQde]fd-       ZeIdd"eQde]fd.       ZeIdd"eQde]fd/       ZeIdd"eQde]fd0       ZeIdd"eQde]fd1       ZeIdd"eQde]fd2       ZeIdd"eQde]fd3       ZeIdd"eQde]fd4       ZeIdd"eQde]fd5       ZeIdd"eQde]fd6       ZeIdd"eQde]fd7       ZeIdd"eQde]fd8       ZeIdd"eQde]fd9       ZeIdd"eQde]fd:       ZeIdd"eQde]fd;       ZeIdd"eQde]fd<       ZeIdd"eQde]fd=       ZeIdd"eQde]fd>       ZeIdd"eQde]fd?       ZeIdd"eQde]fd@       ZeIdd"eQde]fdA       ZeIdd"eQde]fdB       ZeIdd"eQde]fdC       ZeIdd"eQde]fdD       ZeIdd"eQde]fdE       ZeIdd"eQde]fdF       ZeIdd"eQde]fdG       ZeIdd"eQde]fdH       ZeIdd"eQde]fdI       ZeIdd"eQde]fdJ       ZeIdd"eQde]fdK       ZeIdd"eQde]fdL       ZeIdd"eQde]fdM       ZeIdd"eQde]fdN       ZeIdd"eQde]fdO       ZeIdd"eQde]fdP       ZeIdd"eQde]fdQ       ZeIdd"eQde]fdR       ZeIdd"eQde]fdS       ZeIdd"eQde]fdT       ZeIdd"eQde]fdU       ZeIdd"eQde]fdV       ZeIdd"eQde]fdW       ZeIdd"eQde]fdX       ZeIdd"eQde]fdY       ZeIdd"eQde]fdZ       ZeIdd"eQde]fd[       ZeIdd"eQde]fd\       ZeIdd"eQde]fd]       ZeIdd"eQde]fd^       ZeIdd"eQde]fd_       ZeIdd"eQde]fd`       ZeIdd"eQde]fda       ZeIdd"eQde]fdb       ZeIdd"eQde]fdc       ZeIdd"eQde]fdd       ZeIdd"eQde]fde       ZeIdd"eQde]fdf       ZeIdd"eQde]fdg       ZeIdd"eQde]fdh       ZeIdd"eQde]fdi       ZeIdd"eQde]fdj       ZeIdd"eQde]fdk       ZeIdd"eQde]fdl       ZeIdd"eQde]fdm       ZeIdd"eQde]fdn       ZeIdd"eQde]fdo       ZeIdd"eQde]fdp       ZeIdd"eQde]fdq       ZeIdd"eQde]fdr       ZeIdd"eQde]fds       ZeIdd"eQde]fdt       ZeIdd"eQde]fdu       ZeIdd"eQde]fdv       ZeIdd"eQde]fdw       ZeIdd"eQde]fdx       ZeIdd"eQde]fdy       ZeIdd"eQde]fdz       ZeIdd"eQde]fd{       ZeIdd"eQde]fd|       ZeIdd"eQde]fd}       ZeIdd"eQde]fd~       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       Z eIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       Z	eIdd"eQde]fd       Z
eIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       ZeIdd"eQde]fd       Z eJeMi ddddddddddddddddddd̐d͓dΐdϓddddddddddɓdՐd	dddd֜       y# e$ r
 ddlmZ Y  w xY wc c}} w (  a   Vision Transformer (ViT) in PyTorch

A PyTorch implement of Vision Transformers as described in:

'An Image Is Worth 16 x 16 Words: Transformers for Image Recognition at Scale'
    - https://arxiv.org/abs/2010.11929

`How to train your ViT? Data, Augmentation, and Regularization in Vision Transformers`
    - https://arxiv.org/abs/2106.10270

`FlexiViT: One Model for All Patch Sizes`
    - https://arxiv.org/abs/2212.08013

The official jax code is released and available at
  * https://github.com/google-research/vision_transformer
  * https://github.com/google-research/big_vision

Acknowledgments:
  * The paper authors for releasing code and weights, thanks!
  * I fixed my class token impl based on Phil Wang's https://github.com/lucidrains/vit-pytorch
  * Simple transformer style inspired by Andrej Karpathy's https://github.com/karpathy/minGPT
  * Bert reference code checks against Huggingface Transformers and Tensorflow Bert

Hacked together by / Copyright 2020, Ross Wightman
    N)OrderedDict)partial)	AnyCallableDictOptionalSetTupleTypeUnionList)Literal)Final)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STDIMAGENET_INCEPTION_MEANIMAGENET_INCEPTION_STDOPENAI_CLIP_MEANOPENAI_CLIP_STD)	AttentionDiffAttentionAttentionPoolLatentAttentionPoolPrr
PatchEmbedMlpSwiGLUPackedSwiGLU	LayerNormRmsNormDropPathcalculate_drop_path_ratesPatchDropouttrunc_normal_lecun_normal_resample_patch_embedresample_abs_pos_embeduse_fused_attnget_act_layerget_norm_layermaybe_add_maskresolve_self_attn_mask	LayerType
LayerScale   )build_model_with_cfg)feature_take_indices)named_apply
checkpointcheckpoint_seqadapt_input_conv)generate_default_cfgsregister_modelregister_model_deprecationsVisionTransformer) attndiffT
attn_layerdim	num_headsqkv_biasqk_norm
scale_norm	proj_bias	attn_drop	proj_drop
norm_layerdepthreturnc                     t        | t              r"t        j                  | d       } | 
J d|         t	        | t
              r|
|d<    | |f||||||||	d|S )NzUnknown attn_layer: rF   )r>   r?   r@   rA   rB   rC   rD   rE   )
isinstancestrATTN_LAYERSget
issubclassr   )r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   kwargss               i/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/vision_transformer.py_create_attnrP   \   s     *c" __Z6
%J)=j\'JJ% *m,w      c            %       X    e Zd ZdZddddddddddej
                  eeedddfde	d	e	d
e
dededededede
de
dee
   de
deej                     deej                     deej                     dede	ddf$ fdZ	 	 ddej$                  deej$                     dedej$                  fdZ xZS ) Blockz)Transformer block with pre-normalization.      @FT        Nr   r=   r>   	mlp_ratior?   r@   scale_attn_normscale_mlp_normrB   rD   rC   init_values	drop_path	act_layerrE   	mlp_layerr<   rF   rG   c                 8   t         |           ||d} ||fi || _        t        ||f||||||
|	||d	|| _        |rt        |fd|i|nt        j                         | _        |dkD  rt        |      nt        j                         | _
         ||fi || _         |d|t        ||z        ||r|nd||	d|| _        |rt        |fd|i|nt        j                         | _        |dkD  rt        |      | _        yt        j                         | _        y)af  Initialize Block.

        Args:
            dim: Number of input channels.
            num_heads: Number of attention heads.
            mlp_ratio: Ratio of mlp hidden dim to embedding dim.
            qkv_bias: If True, add a learnable bias to query, key, value.
            qk_norm: If True, apply normalization to query and key.
            proj_bias: If True, add bias to output projection.
            proj_drop: Projection dropout rate.
            attn_drop: Attention dropout rate.
            init_values: Initial values for layer scale.
            drop_path: Stochastic depth rate.
            act_layer: Activation layer.
            norm_layer: Normalization layer.
            mlp_layer: MLP layer.
            attn_layer: Attention layer type (class or string).
            depth: Block index, passed to attention layer for depth-dependent init.
        devicedtype	r>   r?   r@   rA   rB   rC   rD   rE   rF   rY   rU   Nin_featureshidden_featuresr[   rE   biasdrop )super__init__norm1rP   r:   r-   nnIdentityls1r    
drop_path1norm2intmlpls2
drop_path2selfr=   r>   rV   r?   r@   rW   rX   rB   rD   rC   rY   rZ   r[   rE   r\   r<   rF   r_   r`   dd	__class__s                        rO   ri   zBlock.__init__   s=   R 	/*r*
 
  &!
 
	 FQ:cA{AbAVXVaVaVc1:R(9-R[[]*r*
 
i0%3z
 
 FQ:cA{AbAVXVaVaVc1:R(9-R[[]rQ   x	attn_mask	is_causalc           
         || j                  | j                  | j                  | j                  |      ||                  z   }|| j	                  | j                  | j                  | j                  |                        z   }|S Nry   rz   )rn   rm   r:   rj   rs   rr   rq   ro   ru   rx   ry   rz   s       rO   forwardzBlock.forward   sj     4::a=Iaj)k lmm$**Q-)@ ABBrQ   NF__name__
__module____qualname____doc__rk   GELUr   r   r   rp   floatboolr   r   Moduler,   ri   torchTensorr   __classcell__rw   s   @rO   rS   rS      s}   3  ""!$)#("!!+/!)+*3),$-)ISIS IS 	IS
 IS IS "IS !IS IS IS IS "%IS IS BIIIS RYYIS  BII!IS" "#IS$ %IS* 
+IS\ 15#	||  - 	
 
rQ   rS   c            %       \    e Zd Zddddddddddej                  eeedddfdeded	e	d
e
de
de
de
de
de	de	dee	   de	deej                     deej                     deej                     dededdf$ fdZddZ	 	 d dej$                  deej$                     de
dej$                  fdZ xZS )!ResPostBlockrT   FTrU   Nr   r=   r>   rV   r?   r@   rW   rX   rB   rD   rC   rY   rZ   r[   rE   r\   r<   rF   rG   c                    t         |           ||d}|| _        t        ||f||||||
|	||d	|| _         ||fi || _        |dkD  rt        |      nt        j                         | _	         |d|t        ||z        ||r|nd ||	d|| _         ||fi || _        |dkD  rt        |      nt        j                         | _        | j                          y )Nr^   ra   rU   rb   rg   )rh   ri   rY   rP   r:   rj   r    rk   rl   rn   rp   rq   ro   rs   init_weightsrt   s                        rO   ri   zResPostBlock.__init__   s    , 	/& 
  &!
 
	  *r*
1:R(9-R[[] 
i0%3z
 
  *r*
1:R(9-R[[]rQ   c                    | j                   }t        j                  j                  | j                  j
                  | j                          t        j                  j                  | j                  j
                  | j                          y y N)rY   rk   init	constant_rj   weightro   ru   s    rO   r   zResPostBlock.init_weights  s[    'GGdjj//1A1ABGGdjj//1A1AB (rQ   rx   ry   rz   c           
          || j                  | j                  | j                  |||                  z   }|| j                  | j	                  | j                  |                  z   }|S r|   )rn   rj   r:   rs   ro   rq   r~   s       rO   r   zResPostBlock.forward  sY     

499Q)W`9+a bcc

488A; 788rQ   rG   Nr   )r   r   r   rk   r   r   r   r   rp   r   r   r   r   r   r,   ri   r   r   r   r   r   r   s   @rO   r   r      sZ   
  ""!$)#("!!+/!)+*3),$-)77 7 	7
 7 7 "7 !7 7 7 7 "%7 7 BII7 RYY7  BII!7" "#7$ %7* 
+7rC 15#	||  - 	
 
rQ   r   c            '           e Zd ZU dZee   ed<   ddddddddddej                  e	ddddddfd	e
d
e
dededededededededee   dedeej                     deej                     deeej                        dee   de
deddf& fdZd"dZ	 	 d#dej&                  deej&                     d edej&                  fd!Z xZS )$ParallelScalingBlockz Parallel ViT block (MLP & Attention in parallel)
    Based on:
      'Scaling Vision Transformers to 22 Billion Parameters` - https://arxiv.org/abs/2302.05442
    
fused_attnrT   FTrU   Nr   r=   r>   rV   r?   r@   rW   rX   rB   rD   rC   rY   rZ   r[   rE   r\   r<   rF   fuse_out_projrG   c                    t         |           ||d}||z  dk(  sJ d       |s|rJ d       || _        ||z  | _        | j                  dz  | _        t               | _        t        ||z        }|d|z  z   } ||fi || _        t        j                  ||fd|i|| _        |g|gdz  z   | _        |r| j                  dd        n.t        j                  t        j                   |fi |      | _        |r || j                  fi |nt        j$                         | _        |r || j                  fi |nt        j$                         | _        t        j*                  |
      | _        t        j*                  |	      | _         |       | _        |r0t        j                  ||z   |fd|i|| _        d | _        d | _        nCd | _        t        j                  ||fd|i|| _        t        j                  ||fd|i|| _        |t9        |fd	|i|nt        j$                         | _        |d
kD  rt=        |      nt        j$                         | _        | jA                          y )Nr^   r   $dim should be divisible by num_headsScale norms not supported         re   mlp_biasrY   rU   )!rh   ri   r>   head_dimscaler'   r   rp   in_normrk   Linearin_projin_splitregister_parameter	Parameterr   emptyr   rl   q_normk_normDropoutrC   mlp_dropmlp_actout_projattn_out_projmlp_out_projr-   lsr    rZ   reset_parameters)ru   r=   r>   rV   r?   r@   rW   rX   rB   rD   rC   rY   rZ   r[   rE   r\   r<   rF   r   r_   r`   rv   mlp_hidden_dimin_proj_out_dimrw   s                           rO   ri   zParallelScalingBlock.__init__+  s.   . 	/Y!#K%KK#">V;VV9"y(]]d*
(*Y_-(1s72!#,,yyoKHKK'(C5194##J5LL^)Jr)JKDM9@j5"5bkkm9@j5"5bkkmI.

9- {IIcN&:CViVSUVDM!%D $D !DM!#3!J)!Jr!JD "		.# TI TQS TDDOD[*S@k@R@acalalan09B),BKKM 	rQ   c                 p    | j                   *t        j                  j                  | j                          yy)"Initialize parameters and buffers.N)r   rk   r   zeros_r   s    rO   r   z%ParallelScalingBlock.reset_parametersn  s%    ==$GGNN4==) %rQ   rx   ry   rz   c                 \   |j                   \  }}}| j                  |      }| j                  |      }t        j                  || j
                  d      \  }}	}
}| j                  || j                  z   }| j                  |	j                  ||| j                  | j                              j                  dd      }	| j                  |
j                  ||| j                  | j                              j                  dd      }
|j                  ||| j                  | j                        j                  dd      }| j                  r>t        j                  |	|
||| j                   r| j"                  j$                  nd|      }ng|	| j&                  z  }	|	|
j                  dd      z  }t)        ||||      }t+        ||      }|j-                  d      }| j#                  |      }||z  }|j                  dd      j/                  |||      }| j1                  |      }| j3                  |      }| j4                  )| j5                  t        j6                  ||fd            }n#| j9                  |      | j;                  |      z   }|| j=                  | j?                  |            z   }|S )	Nr=   r.      rU   ry   	dropout_prz   rz   ) shaper   r   r   splitr   r   r   viewr>   r   	transposer   r   Fscaled_dot_product_attentiontrainingrC   pr   r+   r*   softmaxreshaper   r   r   catr   r   rZ   r   )ru   rx   ry   rz   BNCyx_mlpqkvx_attnr:   	attn_biass                  rO   r   zParallelScalingBlock.forwards  s@    ''1a LLOLLOQ2>q!Q==$DMM)E KKq!T^^T]]CDNNqRSTKKq!T^^T]]CDNNqRSTFF1a7AA!QG??331a#.2mm$..**#	F DJJAq{{2r**D.q$	YWI!$	2D<<B<'D>>$'DAXF!!!Q'//1a8 U#e$ ==$eiiR@AA""6*T->->u-EEA twwqz**rQ   r   r   )r   r   r   r   r   r   __annotations__rk   r   r   rp   r   r   r   r   r,   ri   r   r   r   r   r   r   s   @rO   r   r   $  s    d  ""!$)#("!!+/!)+*337.2"'+A A  A  	A 
 A  A  "A  !A  A  A  A  "%A  A  BIIA  RYYA    RYY0!A " !+#A $ %A &  'A , 
-A F* 15#	1||1  -1 	1
 
1rQ   r   c            '           e Zd ZU dZee   ed<   ddddddddddej                  e	ddddddfd	e
d
e
dededededededededee   dedeej                     deej                     deeej                        dee   de
deddf& fdZde
fdZd$dZdej(                  fdZ	 	 d%d ej(                  d!eej(                     d"edej(                  fd#Z xZS )&DiffParallelScalingBlockaM   Parallel ViT block with Differential Attention (MLP & Attention in parallel).

    Combines the parallel MLP+Attention structure from 'Scaling Vision Transformers to
    22 Billion Parameters' (https://arxiv.org/abs/2302.05442) with differential attention
    from 'Differential Transformer' (https://arxiv.org/abs/2410.05258).
    r   rT   FTrU   Nr   r=   r>   rV   r?   r@   rW   rX   rB   rD   rC   rY   rZ   r[   rE   r\   r<   rF   dual_lambdarG   c                 T   t         |           ||d}||z  dk(  sJ d       |s|rJ d       || _        ||z  dz  | _        | j                  dz  | _        t               | _        t        ||z        }|d|z  z   } ||fi || _        t        j                  ||fd|i|| _        |g|gdz  z   | _        |r| j                  d	d        n.t        j                  t        j                   |fi |      | _        |r || j                  fi |nt        j$                         | _        |r || j                  fi |nt        j$                         | _        t        j*                  |
      | _        |
| _        t1        d| j                  z  fd
di|| _        || _        |rt        j                  t        j                   dt        j6                  |            | _        t        j                  t        j                   dt        j6                  |            | _        d x| _        x| _        x| _         | _!        n.d x| _        | _        t        j                  t        j                   | j                  t        j6                  |            | _        t        j                  t        j                   | j                  t        j6                  |            | _        t        j                  t        j                   | j                  t        j6                  |            | _         t        j                  t        j                   | j                  t        j6                  |            | _!        t        j*                  |	      | _"         |       | _#        t        j                  ||z   |fd|i|| _$        |tK        |fd|i|nt        j$                         | _&        |dkD  rtO        |      nt        j$                         | _(        d| _)        | jU                  |       | jW                          y )Nr^   r   r   r   r   r   r   re   r   epsh㈵>rg   )r`   r_   rY   rU   皙?),rh   ri   r>   r   r   r'   r   rp   r   rk   r   r   r   r   r   r   r   r   rl   r   r   r   rC   attn_drop_pr   sub_normr   float32lambda_alambda_b	lambda_q1	lambda_k1	lambda_q2	lambda_k2r   r   r   r-   r   r    rZ   lambda_initset_lambda_initr   )ru   r=   r>   rV   r?   r@   rW   rX   rB   rD   rC   rY   rZ   r[   rE   r\   r<   rF   r   r_   r`   rv   r   r   rw   s                           rO   ri   z!DiffParallelScalingBlock.__init__  s-   . 	/Y!#K%KK#">V;VV9"y(A-]]d*
(*Y_-(1s72!#,,yyoKHKK'(C5194##J5LL^)Jr)JKDM9@j5"5bkkm9@j5"5bkkmI.$  DMM 1BtBrB&LLRu}}U[)\]DMLLRu}}U[)\]DMPTTDNTT^Tdnt~,00DMDM\\%++dmm5==ag*hiDN\\%++dmm5==ag*hiDN\\%++dmm5==ag*hiDN\\%++dmm5==ag*hiDN

9- { 		#"6R)RrRDOD[*S@k@R@acalalan09B),BKKMU# 	rQ   c                 J    ddt        j                  d|z        z  z
  | _        y )Nr   g333333?g333333ӿ)mathexpr   )ru   rF   s     rO   r   z(DiffParallelScalingBlock.set_lambda_init  s!    txxu'=!==rQ   c                    | j                   )t        j                  j                  | j                          | j                  rSt        j                  j                  | j
                         t        j                  j                  | j                         yt        j                  j                  | j                  dd       t        j                  j                  | j                  dd       t        j                  j                  | j                  dd       t        j                  j                  | j                  dd       y)r   Nr   皙?meanstd)r   rk   r   r   r   r   r   normal_r   r   r   r   r   s    rO   r   z)DiffParallelScalingBlock.reset_parameters   s    ==$GGNN4==)GGNN4==)GGNN4==)GGOODNNO<GGOODNNO<GGOODNNO<GGOODNNO<rQ   c                    | j                   ?t        j                  | j                         }t        j                  | j                        }nt        j                  t        j                  | j
                  | j                  z  d      j                               }t        j                  t        j                  | j                  | j                  z  d      j                               }||z
  | j                  z   S )Nr   r   )r   r   r   r   sumr   r   r   r   r   r   )ru   lambda_1lambda_2s      rO   _compute_lambdaz(DiffParallelScalingBlock._compute_lambda  s    ==$yy/Hyy/Hyy4>>DNN+JPR!S!Y!Y![\Hyy4>>DNN+JPR!S!Y!Y![\H("T%5%555rQ   rx   ry   rz   c                 :   |j                   \  }}}| j                  |      }| j                  |      }t        j                  || j
                  d      \  }}	}
}| j                  || j                  z   }|	j                  ||d| j                  z  | j                        j                  dd      }	|
j                  ||d| j                  z  | j                        j                  dd      }
|j                  ||| j                  d| j                  z        j                  dd      }| j                  |	      | j                  |
      }
}	| j                         j                  |	      }| j                  r|	j                  || j                  d|| j                        }	|
j                  || j                  d|| j                        }
|	j!                  d      \  }}|
j!                  d      \  }}| j"                  r| j$                  nd}t'        j(                  ||||||      }t'        j(                  ||||||      }|||z  z
  }n|	| j*                  z  }	|	|
j                  dd      z  }t-        ||||      }t/        ||      }|j1                  d      }| j3                  |      }|j5                  || j                  d||      }|d d d d d	f   ||d d d d df   z  z
  }||z  }| j7                  |      }|d| j8                  z
  z  }|j                  dd      j                  |||      }| j;                  |      }| j=                  |      }| j?                  t        j@                  ||fd            }|| jC                  | jE                  |            z   }|S )
Nr   r   r   r.   rU   r   r   r   r   )#r   r   r   r   r   r   r   r   r>   r   r   r   r   r   type_asr   unbindr   r   r   r   r   r+   r*   r   rC   r   r   r   r   r   r   r   rZ   r   )ru   rx   ry   rz   r   r   r   r   r   r   r   r   lambda_fullq1q2k1k2r   attn1attn2r   r:   r   s                          rO   r   z DiffParallelScalingBlock.forward  s'    ''1a LLOLLOQ2>q!Q==$DMM)E IIaA.>HHANIIaA.>HHANIIaDNNA,=>HHAN{{1~t{{1~1**,44Q7??		!T^^Q4==AA		!T^^Q4==AAXXa[FBXXa[FB,0MM((sI222r1	]fr{|E222r1	]fr{|E[500FDJJAq{{2r**D.q$	YWI!$	2D<<B<'D>>$'D99Q1a8D1a=;aAg#>>DAXFv&1t///0!!!Q'//1a8 U#e$ MM%))VUO<= twwqz**rQ   r   r   )r   r   r   r   r   r   r   rk   r   r   rp   r   r   r   r   r,   ri   r   r   r   r   r   r   r   r   s   @rO   r   r     s    d  ""!$)#("!!+/!)+*337.2 %+K K  K  	K 
 K  K  "K  !K  K  K  K  "%K  K  BIIK  RYYK    RYY0!K " !+#K $ %K & 'K , 
-K Z>S >=6 6 15#	<||<  -< 	<
 
<rQ   r   c            '       ^    e Zd ZdZdddddddddddej
                  eeedddfd	e	d
e	de	de
dedededededee
   de
de
de
deej                     deej                     deej                     dede	ddf& fdZ	 	 d!dej$                  deej$                     dedej$                  fd Z xZS )"ParallelThingsBlockz Parallel ViT block (N parallel attention followed by N parallel MLP)
    Based on:
      `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795
    r   rT   FTNrU   r   r=   r>   num_parallelrV   r?   r@   rW   rX   rB   rY   rD   rC   rZ   r[   rE   r\   r<   rF   rG   c                 ^   ||d}t         |           || _        t        j                         | _        t        j                         | _        t        |      D ]Q  }| j
                  j                  t        j                  t        d ||fi |fdt        ||f|||||	||||d	|fd|
rt        |fd|
i|nt        j                         fd|dkD  rt        |      nt        j                         fg                   | j                  j                  t        j                  t        d ||fi |fd	 ||ft        ||z        ||r|nd |	|d
|fd|
rt        |fd|
i|nt        j                         fd|dkD  rt        |      nt        j                         fg                   T y )Nr^   normr:   ra   r   rY   rZ   rU   rq   )rd   r[   rE   re   rf   )rh   ri   r  rk   
ModuleListattnsffnsrangeappend
Sequentialr   rP   r-   rl   r    rp   )ru   r=   r>   r  rV   r?   r@   rW   rX   rB   rY   rD   rC   rZ   r[   rE   r\   r<   rF   r_   r`   rv   _rw   s                          rO   ri   zParallelThingsBlock.__init__Z  s   . /(]]_
MMO	|$ !	AJJbmmKC.2./ (%#.''')   z#E;E"EZ\ZeZeZghY^hy1W#9 -  & IIR]];C.2./	$'i$8'-;z""   z#E;E"EZ\ZeZeZghY^hy1W8 ,  )!	rQ   rx   ry   rz   c           	      b   ||rg }| j                   D ]Z  }|j                  |      }|j                  |||      }|j                  |      }|j	                  |      }|j                  |       \ |t        j                  |      j                  d      z   }nF|t        j                  | j                   D cg c]
  } ||       c}      j                  d      z   }|t        j                  | j                  D cg c]
  } ||       c}      j                  d      z   }|S c c}w c c}w )Nr}   r   r   )
r
  r  r:   r   rZ   r  r   stackr   r  )ru   rx   ry   rz   attn_outr:   r   ffns           rO   r   zParallelThingsBlock.forward  s     IH

 (16Y)T/'( EKK)--!-44AEKKTZZ @Ta @AEE!ELLAtyy9SV9:>>1>EE !A9s   7D'
=D,
r   r   r   s   @rO   r  r  U  sf    !"!"!$)#("+/!!!)+*3),$-+== = 	=
 = = = "= != = "%= = = = BII=  RYY!=" BII#=$ "%=& '=, 
-=D 15#	||  - 	
 
rQ   r  rx   	pool_typenum_prefix_tokensreduce_include_prefixc                 &   |s| S |dk(  r| d d df   } | S |r| n
| d d |d f   } |dk(  r| j                  d      } | S |dk(  r*d| j                  d      | j                  d      z   z  } | S |dk(  r| j                  d      } | S |r
J d	|        | S )
Ntokenr   avgr.   r   avgmaxg      ?maxzUnknown pool type )r   amax)rx   r  r  r  s       rO   global_pool_nlcr    s     GadG H 'AAa1B1C.C,D1A H ("qvv!v}qvv!v}45A H %1A H !B$6yk"BB=HrQ   c            S           e Zd ZU dZee   ed<   ddddddd	d	d
ddddddddddddddddddddddddedddee	e
ddf*deeeeef   f   deeeeef   f   dededed   dedededededededed ed!ee   d"ed#ed$ed%ed&ed'ed(ee   d)eded*ed+ed,ed-ed.ed/ed0ed1ed2   d3ed4ed5ee   d6ee   d7ee   d8eej,                     d9eej,                     d:ed;dfR fd<Zdfd=Zdgd>ed?ed;dfd@ZdAej,                  d;dfdBZej8                  j;                         dhdCedDed;dfdE       Zej8                  j:                  d;ee   fdF       Z ej8                  j:                  didGed;e!eeee"f   f   fdH       Z#ej8                  j:                  djdIed;dfdJ       Z$ej8                  j:                  d;ej,                  fdK       Z%dkdedee   d;dfdLZ&	 	 dldeeeef      deeeef      d;dfdMZ'dNejP                  d;ejP                  fdOZ)	 	 	 	 	 	 	 	 	 dmdNejP                  dPeeee"e   f      dQedRedSedTedUedVedWeejP                     dXed;ee"ejP                     eejP                  e"ejP                     f   e!ee*f   f   fdYZ+	 	 	 dndPeee"e   f   dZed[ed;e"e   fd\Z,	 	 	 	 	 dodNejP                  d]eee"e   ee   f   d^edQedRedWeejP                     d;e"ejP                     fd_Z-	 	 dpdNejP                  dWeejP                     dXed;ejP                  fd`Z.dkdNejP                  daee   d;ejP                  fdbZ/didNejP                  dced;ejP                  fddZ0	 	 dpdNejP                  dWeejP                     dXed;ejP                  fdeZ1 xZ2S )qr8   z Vision Transformer

    A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale`
        - https://arxiv.org/abs/2010.11929
    dynamic_img_size      r     r        rT   TFNlearnr   rU   r9   img_size
patch_sizein_chansnum_classesglobal_poolr9   r  r  r  r  mapprr	embed_dimrF   r>   rV   r?   r@   rW   rX   rB   rY   class_token	pos_embedno_embed_class
reg_tokenspre_norm
final_normfc_normpool_include_prefixdynamic_img_pad	drop_ratepos_drop_ratepatch_drop_rateproj_drop_rateattn_drop_ratedrop_path_rateweight_init)skipresetjaxjax_nlhbmocor9   fix_initembed_layerembed_norm_layerrE   r[   block_fnr\   r<   rG   c+                    t         3|           |)|*d}+|dv sJ |s|dk7  sJ |dv sJ ||dv n|},t        |$      xs t        }$t        |#      }#t	        |%      xs t
        j                  }%|| _        || _        || _	        |x| _
        x| _        | _        |rdnd| _        | xj                  |z  c_        || _        || _        || _        || _        || _        d	| _        i }-|r|-j)                  t+        d	d
             |#|#|-d<    |"d-||||| |d|-|+| _        | j,                  j.                  }.t1        | j,                  d      r| j,                  j3                         n|}/|r+t        j4                  t7        j8                  dd|fi |+      nd| _        |r+t        j4                  t7        j8                  d||fi |+      nd| _        |r|.n|.| j                  z   }0|r|dk(  rd| _        n0t        j4                  t7        j8                  d|0|fi |+      | _        t        j@                  |      | _!        |dkD  rtE        || j                        | _#        nt        jH                         | _#        |r	 |$|fi |+nt        jH                         | _%        tM        ||      }1t        jN                  tQ        |      D 2cg c]@  }2 |&d-i d|d|d|	d|
d|d|d|d|d|d|d|d|1|2   d|$d|%d|'d |(d!|2|+B c}2 | _)        tQ        |      D 2cg c]  }2t+        d"|2 ||/#       c}2| _*        |r|,s	 |$|fi |+nt        jH                         | _+        |d$k(  r!tY        | j                  f||	|$|%d%|+| _-        n7|d&k(  r+t]        | j                  f||rdnd'|$d(|+| _-        d)| _        nd| _-        |r|,r	 |$|fi |+nt        jH                         | _/        t        j@                  |      | _0        |dkD  r!t        jb                  | j                  |fi |+nt        jH                         | _2        | d*k(  rd+n| | _3        |!| _4        | d*k7  r| jk                  d	,       yyc c}2w c c}2w ).a  
        Args:
            img_size: Input image size.
            patch_size: Patch size.
            in_chans: Number of image input channels.
            num_classes: Number of classes for classification head.
            global_pool: Type of global pooling for final sequence (default: 'token').
            embed_dim: Transformer embedding dimension.
            depth: Depth of transformer.
            num_heads: Number of attention heads.
            mlp_ratio: Ratio of mlp hidden dim to embedding dim.
            qkv_bias: Enable bias for qkv projections if True.
            init_values: Layer-scale init values (layer-scale enabled if not None).
            class_token: Use class token.
            no_embed_class: Don't include position embeddings for class (or reg) tokens.
            reg_tokens: Number of register tokens.
            pre_norm: Enable norm after embeddings, before transformer blocks (standard in CLIP ViT).
            final_norm: Enable norm after transformer blocks, before head (standard in most ViT).
            fc_norm: Move final norm after pool (instead of before), if None, enabled when global_pool == 'avg'.
            drop_rate: Head dropout rate.
            pos_drop_rate: Position embedding dropout rate.
            attn_drop_rate: Attention dropout rate.
            drop_path_rate: Stochastic depth rate.
            weight_init: Weight initialization scheme.
            fix_init: Apply weight initialization fix (scaling w/ layer index).
            embed_layer: Patch embedding layer.
            embed_norm_layer: Normalization layer to use / override in patch embed module.
            norm_layer: Normalization layer.
            act_layer: MLP activation layer.
            block_fn: Transformer block layer.
        r^   r+  r  )r9   noner%  N)r  r  r  r.   r   FNHWC)strict_img_size
output_fmtrE   )r&  r'  r(  r.  re   r7  
feat_ratiorI  )r   )r  r=   r>   rV   r?   r@   rW   rX   rB   rY   rD   rC   rZ   r[   r\   r<   rF   blocks.)modulenum_chs	reductionr,  )r>   rV   rE   r[   r-  r  )r>   r  rE   Tr?  r@  needs_resetrg   )6rh   ri   r)   r   r(   rk   r   r)  r(  r*  num_featureshead_hidden_sizer.  r  num_reg_tokenshas_class_tokenr1  r6  r  grad_checkpointingupdatedictpatch_embednum_patcheshasattrrM  r   r   r   	cls_token	reg_tokenr0  r   pos_dropr"   
patch_droprl   norm_prer!   r  r  blocksfeature_infor  r   	attn_poolr   r5  	head_dropr   headweight_init_moderD  r   )4ru   r&  r'  r(  r)  r*  r.  rF   r>   rV   r?   r@   rW   rX   rB   rY   r/  r0  r1  r2  r3  r4  r5  r6  r  r7  r8  r9  r:  r;  r<  r=  r>  rD  rE  rF  rE   r[   rG  r\   r<   r_   r`   rv   use_fc_norm
embed_argsr\  rQ  	embed_lendprirw   s4                                                      rO   ri   zVisionTransformer.__init__  s   X 	/QQQQkW4441111AHk%==V]#J/<9
)*:;!),7	& &ENNND1DN&1q*,(*,#6  0"'
d5VLM''7J|$& 	
!+	
 	
 	
 &&225<T=M=M|5\D$$//1bl	MXekk!Q	&HR&HI^bV`ekk!Z&Qb&QRfj#1K{TE[E[7[	I/!DN\\%++aI*TQS*TUDN

]3Q*"&"8"8DO
 !kkmDO7?
933R[[]'>mm* 5\+&#* )  # $ "	
   !0  . $ ( ) ) a& & $ $  &!" %&# $. Y^^cXdfSTD'!yINf3=kJy/B/WYWbWbWd	 %0##%# DN E!-#%0'e%	
 DN (,D$!DN6@[z)2r2VXVaVaVcI.DORSOBIIdnnk@R@Y[YdYdYf	+6&+@k & %0 !m&#,fs   AQ.Q3c                    t        j                         5  t        | j                        D ]~  \  }}t	        j
                  d|dz   z        }|j                  j                  j                  j                  |       |j                  j                  j                  j                  |        	 ddd       y# 1 sw Y   yxY w)z9Apply weight initialization fix (scaling w/ layer index).g       @r.   N)r   no_grad	enumeraterc  r   sqrtr:   projr   div_rq   fc2)ru   layer_idlayerr   s       rO   fix_init_weightz!VisionTransformer.fix_init_weight  s    ]]_ 	1#,T[[#9 1%		#A"67

&&++E2		$$))%01	1 	1 	1s   BB66B?moderS  c                    |xs | j                   }|dv sJ d|v r t        j                  | j                         nd}| j                  t        | j                  d       | j                  +t        j                  j                  | j                  d       | j                  +t        j                  j                  | j                  d       t        t        |||      |        | j                  r| j                          yy)	a*  Initialize model weights.

        Args:
            mode: Weight initialization mode ('jax', 'jax_nlhb', 'moco', or '').
            needs_reset: If True, call reset_parameters() on modules that have it.
                Set to False when modules have already self-initialized in __init__.
        )rA  rB  rC  r@  r9   nlhbrU   N{Gz?r   ư>rR  )rh  r   logr)  r0  r#   r^  rk   r   r   r_  r1   get_init_weights_vitrD  rw  )ru   rx  rS  	head_biass       rO   r   zVisionTransformer.init_weights  s     ,t,,????39T>TXXd..//r	>>%$..c2>>%GGOODNNO5>>%GGOODNNO5(ykRTXY==  " rQ   mc                     t        |       y)z>Initialize weights for a single module (compatibility method).N)init_weights_vit_timm)ru   r  s     rO   _init_weightszVisionTransformer._init_weights  s     	a rQ   checkpoint_pathprefixc                     t        | ||       y)zLoad pretrained weights.

        Args:
            checkpoint_path: Path to checkpoint.
            prefix: Prefix for state dict keys.
        N)_load_weights)ru   r  r  s      rO   load_pretrainedz!VisionTransformer.load_pretrained  s     	dOV4rQ   c                 
    h dS )z3Set of parameters that should not use weight decay.>   r^  r0  r_  
dist_tokenrg   r   s    rO   no_weight_decayz!VisionTransformer.no_weight_decay  s     EDrQ   coarsec                      t        dddg      S )zCreate regex patterns for parameter grouping.

        Args:
            coarse: Use coarse grouping.

        Returns:
            Dictionary mapping group names to regex patterns.
        z*^cls_token|reg_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)i )stemrc  )rZ  )ru   r  s     rO   group_matcherzVisionTransformer.group_matcher  s     >-/CD
 	
rQ   enablec                 v    || _         t        | j                  d      r| j                  j                  |       yy)zEnable or disable gradient checkpointing.

        Args:
            enable: Whether to enable gradient checkpointing.
        set_grad_checkpointingN)rX  r]  r[  r  )ru   r  s     rO   r  z(VisionTransformer.set_grad_checkpointing  s6     #)4##%=>33F; ?rQ   c                     | j                   S )zGet the classifier head.)rg  r   s    rO   get_classifierz VisionTransformer.get_classifier  s     yyrQ   c                 J   || _         |V|dv sJ |dv r| j                  J d       |dvr| j                  d| _        n|dv r| j                  |k7  rJ d       || _        |dkD  r&t        j                  | j
                  |      | _        yt        j                         | _        y)zReset the classifier head.

        Args:
            num_classes: Number of classes for new classifier.
            global_pool: Global pooling type.
        Nr+  )r,  r-  z=Cannot currently add attention pooling in reset_classifier().zECannot currently change attention pooling type in reset_classifier().r   )r)  re  r*  rk   r   r.  rl   rg  )ru   r)  r*  s      rO   reset_classifierz"VisionTransformer.reset_classifier  s     '""UUUUn,1G]]]uN2t~~7Q!%.43C3C{3Reeeu*D>IAoBIIdnnk:	SUS^S^S`	rQ   c           	         | j                   j                  }| j                   j                  ||       | j                  | j                  rdn| j
                  }| j                   j                  |z   }|| j                  j                  d   k7  rGt        j                  t        | j                  | j                   j                  ||d            | _        yyy)zUpdate the input image resolution and patch size.

        Args:
            img_size: New input resolution, if None current resolution is used.
            patch_size: New patch size, if None existing patch size is used.
        )r&  r'  Nr   r.   T)new_sizeold_sizer  verbose)r[  	grid_sizeset_input_sizer0  r1  r  r\  r   rk   r   r&   )ru   r&  r'  prev_grid_sizer  num_new_tokenss         rO   r  z VisionTransformer.set_input_size  s     ))33''j'Q>>%%)%8%8d>T>T!--99<MMN!5!5a!88!#.DNN!--77+&7 / " 9 &rQ   rx   c           	         g }| j                   9|j                  | j                   j                  |j                  d   dd             | j                  9|j                  | j                  j                  |j                  d   dd             | j
                  Ft        j                  ||j                  |j                  d   d|j                  d         gz   d      S | j                  rn|j                  \  }}}}| j                  j                  }t        | j
                  ||f|| j                  rdn| j                        }|j                  |d|      }n| j
                  }| j                  r#||z   }|r>t        j                  ||gz   d      }n"|rt        j                  ||gz   d      }||z   }| j                  |      S )z$Apply positional embedding to input.r   r   r.   r   )r  r  r  )r^  r  expandr   r_  r0  r   r   r   r  r[  r  r&   r1  r  r`  )	ru   rx   to_catr   HWr   r  r0  s	            rO   
_pos_embedzVisionTransformer._pos_embed  s   >>%MM$..//
BCD>>%MM$..//
BCD>>!99Vqvvaggaj"aggbk'J&KKQRSS  JAq!Q!--77N.Q''+':':!@V@V	I q"a AI IAIIfsl2 IIfsl2IA}}QrQ   indicesreturn_prefix_tokensr  
stop_earlyrL  intermediates_onlyoutput_dictry   rz   c           	      *   |dv sJ d       |dk(  }g }t        t        | j                        |      \  }}|j                  \  }}}}| j	                  |      }| j                  |      }| j                  |      }| j                  |      }t        j                  j                         s|s| j                  }n| j                  d|dz    }t        |      D ]}  \  }}|	|
r |||	|
      }n?| j                  r+t        j                  j                         st        ||      }n ||      }||v sZ|j                  |r| j                  |      n|        | j                   rE|D cg c]  }|ddd| j                   f    }}|D cg c]  }|dd| j                   df    }}nd}|ra| j                  j#                  ||f      \  }}|D cg c]6  }|j%                  |||d      j'                  dd	dd
      j)                         8 }}|r*i }||d<   ||r||d<   |s| j                  |      }||d<   |S t        j                  j                         s|r|t+        t-        ||            }|r|S | j                  |      }||fS c c}w c c}w c c}w )a,   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            return_prefix_tokens: Return both prefix and spatial intermediate tokens
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
            output_dict: Return outputs as a dictionary with 'image_features' and 'image_intermediates' keys
            attn_mask: Optional attention mask for masked attention (e.g., for NaFlex)
            is_causal: If True, use causal (autoregressive) masking in attention
        Returns:
            A tuple with (final_features, intermediates), a list of intermediate features, or a dictionary containing
            'image_features' and 'image_intermediates' (and optionally 'image_intermediates_prefix')
        )NCHWNLCz)Output format must be one of NCHW or NLC.r  Nr.   r}   r   r   r   r   image_intermediatesimage_intermediates_prefiximage_features)r0   lenrc  r   r[  r  ra  rb  r   jitis_scriptingrp  rX  r2   r  r  r  dynamic_feat_sizer   permute
contiguouslistzip)ru   rx   r  r  r  r  rL  r  r  ry   rz   r   intermediatestake_indices	max_indexr   r  heightwidthrc  rm  blkr   prefix_tokensr  r  result_dictx_finals                               rO   forward_intermediatesz'VisionTransformer.forward_intermediates5  s   < _,Y.YY,&"6s4;;7G"Qi  gg1feQOOAOOAMM!99!!#:[[F[[)a-0F' 		BFAs$	Y)D((1G1G1IsA&FL $$TTYYq\qA		B !!ERSQq!D$:$:"::;SMSDQRqQq$"8"8"99:RMR M##55vuoFDAq^klYZQYYq!Q3;;Aq!QGRRTlMl K1>K-.(-A<I89 &))A,07,- yy%%',@]E^ ]M!BCM  IIaL-G TR ms   $JJ;J
prune_norm
prune_headc                    t        t        | j                        |      \  }}| j                  d|dz    | _        |rt        j                         | _        |r+t        j                         | _        | j                  dd       |S )aE  Prune layers not required for specified intermediates.

        Args:
            indices: Indices of intermediate layers to keep.
            prune_norm: Whether to prune normalization layer.
            prune_head: Whether to prune the classifier head.

        Returns:
            List of indices that were kept.
        Nr.   r   r9   )r0   r  rc  rk   rl   r  r5  r  )ru   r  r  r  r  r  s         rO   prune_intermediate_layersz+VisionTransformer.prune_intermediate_layers  sj      #7s4;;7G"Qikk.9q=1DI;;=DL!!!R(rQ   nr   c           	      :    | j                  |||||rdndd|      S )a  Get intermediate layer outputs (DINO interface compatibility).

        NOTE: This API is for backwards compat, favour using forward_intermediates() directly.

        Args:
            x: Input tensor.
            n: Number or indices of layers.
            reshape: Reshape to NCHW format.
            return_prefix_tokens: Return prefix tokens.
            norm: Apply normalization.

        Returns:
            List of intermediate features.
        r  r  T)r  r  rL  r  ry   )r  )ru   rx   r  r   r  r  ry   s          rO   get_intermediate_layersz)VisionTransformer.get_intermediate_layers  s4    . ))q!5!(ve# * 
 	
rQ   c                    | j                  |      }| j                  |      }| j                  |      }| j                  |      }||r| j                  D ]  } ||||      } nR| j
                  r5t        j                  j                         st        | j                  |      }n| j	                  |      }| j                  |      }|S )z\Forward pass through feature layers (embeddings, transformer blocks, post-transformer norm).r}   )r[  r  ra  rb  rc  rX  r   r  r  r3   r  )ru   rx   ry   rz   r  s        rO   forward_featuresz"VisionTransformer.forward_features  s     QOOAOOAMM! I{{ EY)DE$$UYY-C-C-Et{{A.AAAIIaLrQ   r  c                     | j                   4| j                  s|dd| j                  df   }| j                  |      }|S || j                  n|}t	        ||| j                  | j                        }|S )zApply pooling to feature tokens.

        Args:
            x: Feature tensor.
            pool_type: Pooling type override.

        Returns:
            Pooled features.
        N)r  r  r  )re  r6  r  r*  r  )ru   rx   r  s      rO   poolzVisionTransformer.pool  s}     >>%++a//001q!AH(1(9D$$y	"44"&":":	
 rQ   
pre_logitsc                     | j                  |      }| j                  |      }| j                  |      }|r|S | j                  |      S )zForward pass through classifier head.

        Args:
            x: Feature tensor.
            pre_logits: Return features before final classifier.

        Returns:
            Output tensor.
        )r  r5  rf  rg  )ru   rx   r  s      rO   forward_headzVisionTransformer.forward_head  sA     IIaLLLONN1q0DIIaL0rQ   c                 P    | j                  |||      }| j                  |      }|S r|   )r  r  r~   s       rO   r   zVisionTransformer.forward  s0     !!!yI!Na rQ   r   r9   Tr9   F)Tr   )NN)	NFFFr  FFNF)r.   FT)r.   FFFNr   )3r   r   r   r   r   r   r   r   rS   r   r   r   rp   r
   r   r   r   rJ   r   r,   r   rk   r   ri   rw  r   r  r   r  ignorer  r	   r  r   r   r  r  r  r  r  r   r  r   r  r  r  r  r  r  r   r   r   s   @rO   r8   r8     s   
 Dk! 5868#V] !!!$)#("+/ $$#("#&*(-%*$)!#%%'$&$&$&SU"$.48.2-1(-),$-W1CsCx011 c5c?231 	1
 1 !!RS1 1 1 1 1 1 1 "1 !1 1  "%!1" #1$ %1& !'1( )1* +1, -1. d^/10 "&112 #314 "516 718 !91: #;1< "=1> "?1@ "A1B !!OPC1D E1F "G1H 'y1I1J !+K1L  	*M1N 299oO1P BIIQ1R "S1X 
Y1B1# # # #.!ryy !T !
 YY5s 5C 5 5 5 YYES E E YY
D 
T#uS$Y?O:O5P 
 
 YY<T <T < < YY		  aC ahsm aW[ a* 3748uS#X/ !sCx1 
	2% ELL % U\\ % T 8<).$$', %04#_ ||_  eCcN34_  #'	_ 
 _  _  _  !%_  _   -_  _  
tELL!5tELL7I)I#JDQTVYQYNZ	[_ F ./$#	3S	>*  	
 
c8 45!).04
||
 S$s)U3Z/0
 	

 #'
 
  -
 
ell	
F 15#	||  - 	
 
0ell x}  01ell 1 1 1$ 15#	||  - 	
 
rQ   r9   rO  namerS  c                 N   t        | t        j                        rNt        | j                  d       | j
                  *t        j                  j                  | j
                         yyt        | d      r| j                          y|rt        | d      r| j                          yyy)zViT weight initialization, original timm impl (for reproducibility).

    Args:
        module: Module to initialize.
        name: Module name for context.
        needs_reset: If True, call reset_parameters() on modules that have it.
    r{  r|  Nr   r   )rI   rk   r   r#   r   re   r   r   r]  r   r   rO  r  rS  s      rO   r  r    sy     &"))$fmm-;;"GGNN6;;' #		(	);<! =rQ   r  c                 f   t        | t        j                        r|j                  d      rTt        j                  j                  | j                         t        j                  j                  | j                  |       yt        j                  j                  | j                         | j                  Yd|v r+t        j                  j                  | j                  d      n(t        j                  j                  | j                         yyt        | t        j                        rLt        | j                         | j                  *t        j                  j                  | j                         yyt        | d      r| j                          y|rt        | d      r| j                          yyy)a  ViT weight initialization, matching JAX (Flax) impl.

    Args:
        module: Module to initialize.
        name: Module name for context.
        head_bias: Bias value for head layer.
        needs_reset: If True, call reset_parameters() on modules that have it.
    rg  Nrq   r}  r|  r   r   )rI   rk   r   
startswithr   r   r   r   re   xavier_uniform_r   Conv2dr$   r]  r   r   )rO  r  r  rS  s       rO   init_weights_vit_jaxr  +  s    &"))$??6"GGNN6==)GGfkk95GG##FMM2{{&:?4-6RWW^^\b\g\gMh '	FBII	&fmm$;;"GGNN6;;' #		(	);<! =rQ   c                 ~   t        | t        j                        rd|v rt        j                  dt        | j                  j                  d   dz  | j                  j                  d   z         z        }t        j                  j                  | j                  | |       n)t        j                  j                  | j                         | j                  *t        j                  j                  | j                         yyt        | d      r| j                          y|rt        | d      r| j                          yyy)	zViT weight initialization, matching moco-v3 impl minus fixed PatchEmbed.

    Args:
        module: Module to initialize.
        name: Module name for context.
        needs_reset: If True, call reset_parameters() on modules that have it.
    qkvg      @r   r   r.   Nr   r   )rI   rk   r   r   rq  r   r   r   r   uniform_r  re   r   r]  r   r   )rO  r  rS  vals       rO   init_weights_vit_mocor  K  s     &"))$D=))Bv}}':':1'='BV]]EXEXYZE['[!\\]CGGV]]SD#6GG##FMM2;;"GGNN6;;' #		(	);<! =rQ   c                 D    |rt        | d      r| j                          y y y )Nr   )r]  r   r  s      rO   init_weights_reset_parametersr  b  s"    wv'9:! ;{rQ   rx  c                     | j                  d      rt        t        ||      S | j                  d      rt        t        |      S | dk(  rt        t        |      S t        t
        |      S )NrA  )r  rS  rC  rR  r@  )r  r   r  r  r  r  )rx  r  rS  s      rO   r  r  g  s]    u+ykZZ		 ,+FF	4+NN ,+FFrQ   rg   posemb
posemb_newgs_newinterpolation	antialiasc           	         |j                   d   |z
  }| j                   d   |z
  }t        t        j                  |            gdz  }t	        |      s"t        t        j                  |            gdz  }t        | |||||d      S )z Rescale the grid of position embeddings when loading from state_dict.
    *DEPRECATED* This function is being deprecated in favour of using resample_abs_pos_embed
    r.   r   T)r  r  r  r  )r   rp   r   rq  r  r&   )	r  r  r  r  r  r  ntok_newntok_oldgs_olds	            rO   resize_pos_embedr  t  s     "%66H||A!22H$))H%&'!+Fv;dii)*+a/!+# rQ   modelr  r  load_bfloat16c                    ddl r
ddlm  ddldIfd	}rj                  |      }nj                  |      }d}d}d}|sd|v rd}nd	|v rd
}d}nd|v rd}d}t	        | j
                  d      r| j
                  j                  }	t	        |	d       }
|
r|	n|	j                  }|j                  j                  j                  t        |j                  j                  j                  d    ||| d                      |j                  j                  j                   ||| d                |j                  j                  j                   ||| d                |
st        |	j                         D ]  \  }}t        |j"                        D ]  \  }}| d|dz    d|dz    d}t%        d      D ]  }t'        |d|dz          j                  j                   ||| d|dz    d                t'        |d|dz          j                  j                   ||| d|dz    d                t'        |d|dz          j                  j                   ||| d|dz    d                 |j(                  |j(                  j                  j                  j                   ||| d                |j(                  j                  j                  j                   ||| d                |j(                  j                  j                  j                   ||| d                   ||| d          }nCt        | j
                  j*                  j                  j                  d    ||| d                }|j                  d!d | j
                  j*                  j                  j                  d!d k7  r;t-        || j
                  j*                  j                  j                  d!d ||d"      }| j
                  j*                  j                  j                  |       | j
                  j*                  j                  j                   ||| d#                | j.                  )| j.                  j                   ||| d$   d%             |r ||| d&   d%      }n ||| d'   d%      }|j                  | j0                  j                  k7  rAt'        | d(d      rdnt'        | d)d      }t3        || j
                  j4                  |||d*      }| j0                  j                  |       | j                  j                  j                   ||| d+                | j                  j                  j                   ||| d,                t7        | j8                  t:        j<                        r| d-|v r| j8                  j                  j                  d   || d-   j                  d.   k(  rb| j8                  j                  j                   ||| d/                | j8                  j                  j                   ||| d-                t7        | j>                  t@              rn| d0}|d1z   }| j>                  jB                  j                   ||| d2   d%             | j>                  jD                  j                  j                  tG        jH                  d3D cg c]-  } ||| | d   d%      jK                  d      jL                  / c}             | j>                  jD                  j                  j                  tG        jH                  d3D cg c]#  } ||| | d   d%      jO                  d.      % c}             | j>                  jP                  j                  j                   ||| d4   d%      jK                  d      jL                         | j>                  jP                  j                  j                   ||| d5   d%      jO                  d.             | j>                  j*                  j                  j                   ||| d6         jK                  d             | j>                  j*                  j                  j                   ||| d7                | j>                  j                  j                  j                   ||| d8                | j>                  j                  j                  j                   ||| d9                t%        d:      D ]  }t'        | j>                  jR                  d;|dz          j                  j                   ||| d<| d                t'        | j>                  jR                  d;|dz          j                  j                   ||| d<| d                 |rd=nd>\  }}}t        | j"                  jU                               D ]  \  }}| d?|v r| d@}|}n
| dA| d}d}|dB| dz   }|jV                  j                  j                   ||| d8   |C             |jV                  j                  j                   ||| d9   |C             |jX                  jZ                  j                  j                  tG        jH                  dDD cg c].  } ||| | d   d|E      jK                  d      jL                  0 c}             |jX                  jZ                  j                  j                  tG        jH                  dDD cg c]$  } ||| | d   d|E      jO                  d.      & c}             |jX                  j*                  j                  j                   ||| d6   |C      jK                  d             |jX                  j*                  j                  j                   ||| d7   |C             |j\                  j                  j                   ||| dF| d   |C             |j\                  j                  j                   ||| dF| d   |C             t%        d:      D ]  }t'        |jR                  d;|dz          j                  j                   ||| dG| dH| d   |C             t'        |jR                  d;|dz          j                  j                   ||| dG| dH| d   |C               yc c}w c c}w c c}w c c}w )JzV Load weights from .npz checkpoints for official Google Brain Flax implementation
    r   NTc                 V   || |   } rE| j                  j                        j                  j                        } j	                  |       } | j
                  dk(  rK| j                  d   | j                  d   cxk(  r| j                  d   cxk(  rdk(  rn n| j                         } |rh| j
                  dk(  r| j                  g d      } nE| j
                  dk(  r| j                  g d      } n"| j
                  dk(  r| j                  ddg      } t        j                  |       } | S )N   r   r.   r   )r   r   r   r.   r   )r   r   r.   )r   bfloat16astyper   arrayndimr   flattenr   r   
from_numpy)_wtidxjnpr  	ml_dtypesnps      rO   _n2pz_load_weights.<locals>._n2p  s    ?CB++,33CKK@B"B77a<BHHQK288A;J"((1+JJBww!|\\,/A\\),A\\1a&)b!	rQ   bilinearFzopt/target/embedding/kernelzopt/target/zparams/embedding/kernelzparams/zparams/img/embedding/kernelzparams/img/backboner  r.   zconv_root/kernelzgn_root/scalezgn_root/biasblockz/unit/r   convz/kernelr  gnz/scalez/biaszconv_proj/kernelzgn_proj/scalezgn_proj/biaszembedding/kernelr   r  r  r  zembedding/biascls)r  pos_embeddingz(Transformer/posembed_input/pos_embeddingr1  r  r  r  r  r  r  zTransformer/encoder_norm/scalezTransformer/encoder_norm/biasz	head/biasr   zhead/kernelz
MAPHead_0/zMultiHeadDotProductAttention_0/probe)keyvaluezquery/kernelz
query/biasz
out/kernelzout/biaszLayerNorm_0/scalezLayerNorm_0/biasr   fczMlpBlock_0/Dense_)r   r   r.   )r.   r   r   z*Transformer/encoderblock/LayerNorm_0/scalezTransformer/encoderblock/zTransformer/encoderblock_MultiHeadDotProductAttention_)r  )queryr  r  )r  r  
LayerNorm_	MlpBlock_z/Dense_)TN)/numpy	jax.numpyr  loadr]  r[  r  r  r  r   copy_r4   r   r  re   rp  stagesrc  r  getattr
downsamplerr  r%   r^  r0  r&   r  rI   rg  rk   r   re  r   latentkvr   r   r  Tr   r   rq   childrenrj   r:   r  ro   )r  r  r  r  r  wr  r  
big_visionr  	stem_onlyr  rm  stagejr  bprembed_conv_wpos_embed_wr  block_prefix
mha_prefixr  mha_subb_subln1_subr  r  r  r   s      `                        @@@rO   r  r    sn     * HH_%GGO$MIJ(A-"F&!+FJ*a/"FJu  *-$$--&11	$x(--		/		0@0@0F0Fq0I4PQU[T\\lRmPnKopq		tA&>$?@A		T!vhl$;"<=>%hoo6 
W5 )%,, 7 	WHAu"85QuQUG1=B"1X ba!eW~6==CCDbTQUVWZ[V[U\\cKdIeDfga!eW~6==CCDbTQSTUXYTYSZZ`KaIbDcda!eW~6;;AA$qB4rRSVWRWQXX]I^G_B`ab ''3((--44::4RDHXBY@Z;[\((--44::4RDBV@W;XY((--2288a2$l@S>T9UV	W
W A(89:;'""))//2DfXEU;V9W4XZ"#%"3"3"8"8"?"?"E"Ebc"JJ+""))//4'
 
!!''5	%%d1x~-F+G&HI"d1xs^#4>?1x}56%@1x'OPQUZ[EOO111!(0@%!HAgV[]prsNt,&&00/'
 
OO+&	JJDfX-K#L!MNO	JJOO$qF8+H!IJKL5::ryy)hi A%JJOO!!!$VHI*>(?(E(Eb(II

Q&'=%> ?@

d1xy%9#:;<
 %//#67 ,!&EF
$$T!|nE,B*Cu%MN!!''		N^3`IJDj\!G,-7??BDD3` )a 	b%%eiiK[1]FGDj\!E*+u5==bA1] '^ 	_  &&tAL.I,Je'T'\'\]^'_'a'ab$$T!zl*,E*F%%P%X%XY[%\]##))$qJ<z1J/K*L*T*TUV*WX!!''Q*X/F-G(HI##))$qL>AR1S/T*UV!!''Q,?O/P-Q(RSq 	wAEOO''2a!eW6==CCDl^[lmnloovKwIxDyzEOO''2a!eW6;;AA$qL>YjkljmmrIsGtBuv	w ,6i9GUGell3356 T5X?@AE$X%>?LC$X%>qcCLC!&CG9A$NN
  a<.8I(J&KQT!UVtA6F&G$HcRS

##EIIWp/rRSDj\!G,-C@HHKMM/r %s 	t

!!%))Tm-oOPDj\!E*+u#>FFrJ-o #p 	q

$$T!zl*,E*FC%P%X%XYZ%[\

""4ZL*A(B#LM  a<.
7)6(R&SY\!]^tAj	&O$PVYZ[q 	TAEIIAE7|,3399Q,ywqcIJPSTVEIIAE7|,1177Q,ywqcGHcRT	T%T3`1]./r-os   	2t?(u63u	0)u
state_dictc                    i }g d}| j                         D ]  \  }}|j                  |      s|j                  |d      }|D ]  }|j                  |d   |d         } |dk(  r:d}|j                  dd      }t	        j
                  |j                  d         |d<   n>|dk(  r#d	}|j                  d      j                  d      }n|d
k(  r|j                  d      }|||<    |S )N)
)conv1patch_embed.proj)positional_embeddingr0  )ztransformer.resblocks.rN  )ln_prerb  )ln_postr  )ln_r  )in_proj_zqkv.r   rr  )zmlp.c_fcmlp.fc1)z
mlp.c_projzmlp.fc2r9   r   r.   rr  head.weight	head.biasclass_embeddingr^  r0  )itemsr  replacer   r   zerosr   	unsqueeze)r-  r  r  out_dictswapsr   r   sps           rO   _convert_openai_cliprB  (  s    
 HE   " 1||F#IIfb! 	(B		"Q%A'A	( ;AAq!A$)KK
$;H[!##AA((+A+AA!" OrQ   c                    dd l }i }| j                  dd        d| v rR| j                  d      |d<   | j                  d      | d   d d df   z   |d<   | j                  d      d d dd f   |d<   | j                         D ]Z  \  }}|j                  d|      r|||j	                  d	d
      <   .|j                  d|      r|||j	                  dd      <   V|||<   \ |S )Nr   
mask_tokenregister_tokensr_  r^  r0  r.   z(blocks\.(\d+)\.mlp\.w12\.(?:weight|bias)w12fc1z'blocks\.(\d+)\.mlp\.w3\.(?:weight|bias)w3rt  )repopr;  matchr<  )r-  r  rI  r?  r   r   s         rO   _convert_dinov2rL  N  s     HNN<&J& */@ A *{ ;j>UVWYZVZ>[ [ *{ ;AqrE B  " 188?C01HQYYue,-XX@!D/0HQYYtU+, OrQ   c                 d   i }| j                         D ]  \  }}|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  d	d
      }|j                  dd      }|j                  dd      }|j                  dd      }|||<    |S )Nnorm_1rj   norm_2ro   zpreprocessor.patchifier.patch_embed.zpreprocessor.pos_embedr0  ztrunk.r9   zpost_trunk_norm.norm.r7  z	mlp.fc1_gzmlp.fc3z	mlp.fc1_x)r;  r<  )r-  r  r?  r   r   s        rO   _convert_aimv2rR  e  s     H  " 	1IIh(IIh(II0.AII.<IIh#II('2IIi-IIi-	 OrQ   c                    ddl }| j                  d|       } dD ]  }| j                  |d        g d}i }| j                         D ]I  \  }}d|v r|D ]  \  }}|j	                  |||      } |dk(  r|dd j                  d      |d	<   E|||<   K i i }
}	|j                  d
      }|j                         D ]  \  }}|j                  |      }|s||	|<   |j                         \  }}}|
j                  ||fi       }|||<   t        |      dk(  s[t        j                  |d   |d   |d   gd      |	d| d| <    |	S )zP
    Turn a BEiT-3 checkpoint into a standard VisionTransformer state-dict.
    r   Nr  )zbeit3.text_embed.weightzbeit3.vision_embed.mask_token))zbeit3\.r9   )zvision_embed\.cls_tokenr^  )zvision_embed\.rP  )zembed_positions\.z
pos_embed.)z	encoder\.r9   )zlayers\.rN  )zffn_layernorm\.rQ  )zffn\.zmlp.)zself_attn_layer_norm\.znorm1.)zself_attn\.zattn.)zfinal_layer_norm\.znorm2.)inner_attn_lnr  r6  )z\.A\..z.B.zpos_embed.weightr   r0  z1blocks\.(\d+)\.attn\.(q|k|v)_proj\.(weight|bias)$r   r   r   r   r   rN  z
.attn.qkv.)rI  rL   rJ  r;  subr>  compile	fullmatchgroups
setdefaultr  r   r   )r-  r  rI  r   rulestmpr   oldnewoutbufpatr  r  whichkindstashs                    rO   _convert_beit3re  w  s    4J J  q$ E  C  " 	1A: 	$HCsC#A	$"" uq1CCF	 2C
**I
JC		 1MM!CF88:UDT{B/eu:?3899sU3Zs4!4C'#j/0 JrQ   adapt_layer_scalec           	      b   ddl }i }| j                  d|       } | j                  d|       } d}d| v rt        | |      } n-d| v rt        | |d	      } nd
| v rt        | |      } nd| v rPd}| j	                         D 	
ci c]&  \  }	}
|	j                  |      s|	t        |      d |
( } }	}
t        | |      } nt        d | j                         D              rt        | |      } nd| v r| d   } d}nzd| v sd| v r[d}d| v rlt        |j                  t        j                        rH| d   |d<   t        j                  | d   j                   d         |d<   nd| v rd}nd| v rt#        | |      } |rA| j	                         D 	
ci c]&  \  }	}
|	j                  |      s|	t        |      d |
( } }	}
| j	                         D ]`  \  }	}
d|	v r|j$                  j&                  j(                  j                   \  }}}}t        |
j                         dk  rC|j$                  j&                  j(                  j                   \  }}}}|
j+                  |d||      }
|
j                   d   |k7  s|
j                   d   |k7  rt-        |
||f||d      }
n|	dk(  rk|
j                   d    |j.                  j                   d    k7  rBt1        |d!d"      rdnt1        |d#d       }t3        |
|j$                  j4                  |||d$      }
n |rd%|	v r|j7                  d&d'|	      }	nd(|	v r\|
||	<   c |S c c}
}	w c c}
}	w ))zJ convert patch embedding weight from manual patchify + linear proj to convr   Nr  r-  r9   zvisual.class_embeddingzmodule.visual.class_embeddingzmodule.visual.)r  rD  zvision_encoder.mask_tokenzvision_encoder.c              3   $   K   | ]  }d |v  
 yw)zbeit3.Nrg   ).0r   s     rO   	<genexpr>z'checkpoint_filter_fn.<locals>.<genexpr>  s     6qX]6s   encoderzmodule.zvisual.trunk.pos_embedz"visual.trunk.blocks.0.norm1.weightzvisual.trunk.zvisual.head.proj.weightr8  r9  zmodule.visual.trunk.pos_embedzmodule.visual.trunk.z#preprocessor.patchifier.proj.weightzpatch_embed.proj.weightr  r   r   Tr  r0  r.   r1  Fr  r  gamma_zgamma_([0-9])z
ls\1.gammar  )rI  rL   rB  rL  r;  r  r  anykeysre  rI   rg  rk   r   r   r=  r   rR  r[  rr  r   r   r%   r0  r  r&   r  rV  )r-  r  rf  r  r  rI  r?  r  	ve_prefixr   r   OIr  r  r  s                   rO   checkpoint_filter_fnrr    sG    H4Jj9JF:-)*e<
	(J	6)*eDTU
		#$Z7
	$
	2%	8B8H8H8Jf1all[dNeaI(!+f
f$Z7
	6JOO$56	6#J6
	j	 	*
	!Z	/3W[e3e $
2z%**bii7X&01J&KH]#$)KK
;T0U0[0[\]0^$_H[!	(J	6'	.*	<#J6
5?5E5E5G`TQ1<<X^K_aFoq(`
`  "  1$)**//66<<JAq!Q177|a"..33::@@
1aIIaQ*wwr{a1772;!#3(F"/'  +!''!*0E0Ea0H"H%,U4De%LRYZ_atvwRx&**44"3+#A 8q=':AQA B Ou g. as   >L%L%L+L+urlc                 4    | ddd dddt         t        dddd	|S )
Nr"  )r   r   r   g?bicubicTr0  rg  
apache-2.0)rs  r)  
input_size	pool_sizecrop_pctr  fixed_input_sizer   r   
first_conv
classifierlicense)r   r   )rs  rN   s     rO   _cfgr~    s:    #" '%(  rQ   z*vit_base_patch16_224.augreg2_in21k_ft_in1kztimm/)	hf_hub_idz)vit_base_patch8_224.augreg2_in21k_ft_in1kz)vit_tiny_patch16_224.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npz)rs  r  custom_loadz)vit_tiny_patch16_384.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npz)r     r        ?)rs  r  r  rw  ry  z*vit_small_patch32_224.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/S_32-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npzz*vit_small_patch32_384.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/S_32-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npzz*vit_small_patch16_224.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/S_16-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npzz*vit_small_patch16_384.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/S_16-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npzz)vit_base_patch32_224.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/B_32-i21k-300ep-lr_0.001-aug_medium1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npzz)vit_base_patch32_384.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/B_32-i21k-300ep-lr_0.001-aug_light1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npzz)vit_base_patch16_224.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/B_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_224.npzz)vit_base_patch16_384.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/B_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_384.npzz(vit_base_patch8_224.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/B_8-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_224.npzz*vit_large_patch16_224.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/L_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_224.npzz*vit_large_patch16_384.augreg_in21k_ft_in1kzhttps://storage.googleapis.com/vit_models/augreg/L_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_384.npzz'vit_base_patch16_224.orig_in21k_ft_in1kzohttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_vit_base_p16_224-80ecf9dd.pth)rs  r  z'vit_base_patch16_384.orig_in21k_ft_in1kzohttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_vit_base_p16_384-83fb41ba.pth)rs  r  rw  ry  z(vit_large_patch32_384.orig_in21k_ft_in1kzphttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_vit_large_p32_384-9b920ba8.pthz!vit_small_patch16_224.augreg_in1kzhttps://storage.googleapis.com/vit_models/augreg/S_16-i1k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_224.npzz!vit_small_patch16_384.augreg_in1kzhttps://storage.googleapis.com/vit_models/augreg/S_16-i1k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_384.npzz vit_base_patch32_224.augreg_in1kzhttps://storage.googleapis.com/vit_models/augreg/B_32-i1k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_224.npzz vit_base_patch32_384.augreg_in1kzhttps://storage.googleapis.com/vit_models/augreg/B_32-i1k-300ep-lr_0.001-aug_medium2-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_384.npzz vit_base_patch16_224.augreg_in1kzhttps://storage.googleapis.com/vit_models/augreg/B_16-i1k-300ep-lr_0.001-aug_strong2-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_224.npzz vit_base_patch16_384.augreg_in1kzhttps://storage.googleapis.com/vit_models/augreg/B_16-i1k-300ep-lr_0.001-aug_strong2-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_384.npzzvit_large_patch14_224.untrained)rs  zvit_huge_patch14_224.untrainedzvit_giant_patch14_224.untrainedz"vit_gigantic_patch14_224.untrainedzvit_base_patch32_224.orig_in21k)r  r)  zvit_base_patch16_224.orig_in21kz vit_large_patch32_224.orig_in21kz vit_large_patch16_224.orig_in21kzvit_huge_patch14_224.orig_in21kz!vit_tiny_patch16_224.augreg_in21kzmhttps://storage.googleapis.com/vit_models/augreg/Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0.npziSU  )rs  r  r  r)  z"vit_small_patch32_224.augreg_in21kznhttps://storage.googleapis.com/vit_models/augreg/S_32-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0.npzz"vit_small_patch16_224.augreg_in21kznhttps://storage.googleapis.com/vit_models/augreg/S_16-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0.npzz!vit_base_patch32_224.augreg_in21kzohttps://storage.googleapis.com/vit_models/augreg/B_32-i21k-300ep-lr_0.001-aug_medium1-wd_0.03-do_0.0-sd_0.0.npzz!vit_base_patch16_224.augreg_in21kznhttps://storage.googleapis.com/vit_models/augreg/B_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0.npzz vit_base_patch8_224.augreg_in21kzmhttps://storage.googleapis.com/vit_models/augreg/B_8-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0.npzz"vit_large_patch16_224.augreg_in21kznhttps://storage.googleapis.com/vit_models/augreg/L_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1.npzzvit_base_patch32_224.sam_in1kz:https://storage.googleapis.com/vit_models/sam/ViT-B_32.npz)rs  r  r  zvit_base_patch16_224.sam_in1kz:https://storage.googleapis.com/vit_models/sam/ViT-B_16.npzzvit_small_patch16_224.dinoz[https://dl.fbaipublicfiles.com/dino/dino_deitsmall16_pretrain/dino_deitsmall16_pretrain.pth)rs  r  r   r   r)  zvit_small_patch8_224.dinozYhttps://dl.fbaipublicfiles.com/dino/dino_deitsmall8_pretrain/dino_deitsmall8_pretrain.pthzvit_base_patch16_224.dinozWhttps://dl.fbaipublicfiles.com/dino/dino_vitbase16_pretrain/dino_vitbase16_pretrain.pthzvit_base_patch8_224.dinozUhttps://dl.fbaipublicfiles.com/dino/dino_vitbase8_pretrain/dino_vitbase8_pretrain.pthz vit_small_patch14_dinov2.lvd142mzNhttps://dl.fbaipublicfiles.com/dinov2/dinov2_vits14/dinov2_vits14_pretrain.pthrv  )r     r  )rs  r  r}  r   r   r)  rw  ry  zvit_base_patch14_dinov2.lvd142mzNhttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitb14/dinov2_vitb14_pretrain.pthz vit_large_patch14_dinov2.lvd142mzNhttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_pretrain.pthz vit_giant_patch14_dinov2.lvd142mzNhttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitg14/dinov2_vitg14_pretrain.pthz%vit_small_patch14_reg4_dinov2.lvd142mzShttps://dl.fbaipublicfiles.com/dinov2/dinov2_vits14/dinov2_vits14_reg4_pretrain.pthz$vit_base_patch14_reg4_dinov2.lvd142mzShttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitb14/dinov2_vitb14_reg4_pretrain.pthz%vit_large_patch14_reg4_dinov2.lvd142mzShttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitl14/dinov2_vitl14_reg4_pretrain.pthz%vit_giant_patch14_reg4_dinov2.lvd142mzShttps://dl.fbaipublicfiles.com/dinov2/dinov2_vitg14/dinov2_vitg14_reg4_pretrain.pthz"vit_base_patch14_reg1_tipsv2.webli)rU   rU   rU   )r  r  r  )r     r  )r  r}  r   r   r)  rw  ry  z#vit_large_patch14_reg1_tipsv2.webliz$vit_so400m_patch14_reg1_tipsv2.webliz#vit_giant_patch14_reg1_tipsv2.weblizvit_base_patch16_224_miil.in21kz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tresnet/vit_base_patch16_224_in21k_miil-887286df.pthg      ?r  i+  )rs  r  r   r   ry  r  r)  z'vit_base_patch16_224_miil.in21k_ft_in1kzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tresnet/vit_base_patch16_224_1k_miil_84_4-2deb18e3.pth)rs  r  r   r   ry  r  z vit_base_patch16_rpn_224.sw_in1kz}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_base_patch16_rpn_224-sw-3b07e89d.pthz#vit_medium_patch16_gap_240.sw_in12k)r      r  gffffff?i-.  )r  rw  ry  r)  z+vit_medium_patch16_gap_256.sw_in12k_ft_in1k)r      r  )r  rw  ry  z+vit_medium_patch16_gap_384.sw_in12k_ft_in1ksquash)r  rw  ry  	crop_modez%vit_betwixt_patch16_gap_256.untrained)rw  ry  z"vit_base_patch16_gap_224.untrainedz/vit_base_patch32_clip_224.laion2b_ft_in12k_in1k)r  r   r   z/vit_base_patch32_clip_384.laion2b_ft_in12k_in1k)r  r   r   ry  rw  z/vit_base_patch32_clip_448.laion2b_ft_in12k_in1kz/vit_base_patch16_clip_224.laion2b_ft_in12k_in1k)r  r   r   ry  z/vit_base_patch16_clip_384.laion2b_ft_in12k_in1k)r  r   r   ry  rw  r  z0vit_large_patch14_clip_224.laion2b_ft_in12k_in1kz0vit_large_patch14_clip_336.laion2b_ft_in12k_in1k)r   P  r  z/vit_huge_patch14_clip_224.laion2b_ft_in12k_in1kz/vit_huge_patch14_clip_336.laion2b_ft_in12k_in1kz.vit_base_patch32_clip_224.openai_ft_in12k_in1kr   z.vit_base_patch32_clip_384.openai_ft_in12k_in1kz.vit_base_patch16_clip_224.openai_ft_in12k_in1kz.vit_base_patch16_clip_384.openai_ft_in12k_in1kz/vit_large_patch14_clip_224.openai_ft_in12k_in1kz/vit_large_patch14_clip_336.openai_ft_in12k_in1kz)vit_base_patch32_clip_224.laion2b_ft_in1kz)vit_base_patch16_clip_224.laion2b_ft_in1kz)vit_base_patch16_clip_384.laion2b_ft_in1kz*vit_large_patch14_clip_224.laion2b_ft_in1kz*vit_large_patch14_clip_336.laion2b_ft_in1kz)vit_huge_patch14_clip_224.laion2b_ft_in1kz)vit_huge_patch14_clip_336.laion2b_ft_in1kz(vit_base_patch32_clip_224.openai_ft_in1kz(vit_base_patch16_clip_224.openai_ft_in1kz(vit_base_patch16_clip_384.openai_ft_in1kz)vit_large_patch14_clip_224.openai_ft_in1kz*vit_base_patch16_clip_224.laion2b_ft_in12k)r  r   r   r)  z+vit_large_patch14_clip_224.laion2b_ft_in12k)r  r   r   ry  r)  z*vit_huge_patch14_clip_224.laion2b_ft_in12kz)vit_base_patch16_clip_224.openai_ft_in12kz*vit_large_patch14_clip_224.openai_ft_in12kz!vit_base_patch32_clip_224.laion2b   z!vit_base_patch16_clip_224.laion2bz"vit_large_patch14_clip_224.laion2br#  z!vit_huge_patch14_clip_224.laion2b   z"vit_giant_patch14_clip_224.laion2bz%vit_gigantic_patch14_clip_224.laion2b   z'vit_base_patch32_clip_224.laion400m_e32mit)zDnatively QuickGELU, use quickgelu model variant for original results)r  r}  notesr   r   r)  z'vit_base_patch16_clip_224.laion400m_e32)r  r}  r   r   ry  r)  z,vit_base_patch16_plus_clip_240.laion400m_e32  )r  r}  r   r   rw  ry  r)  z(vit_large_patch14_clip_224.laion400m_e32z$vit_base_patch32_clip_224.datacompxlz$vit_base_patch32_clip_256.datacompxl)r  r   r   ry  rw  r)  z$vit_base_patch16_clip_224.datacompxlz%vit_large_patch14_clip_224.datacompxlzvit_base_patch16_clip_224.dfn2bz
apple-asclz%vit_large_patch14_clip_224.dfn2b_s39bz vit_large_patch14_clip_224.dfn2b)r  r}  r  r   r   ry  r)  zvit_huge_patch14_clip_224.dfn5bzvit_huge_patch14_clip_378.dfn5b)r   z  r  )r  r   r   r}  r  ry  rw  r)  z-vit_huge_patch14_clip_224.metaclip2_worldwidezcc-by-nc-4.0z-vit_huge_patch14_clip_378.metaclip2_worldwide)r  r}  r   r   rw  ry  r  r)  z1vit_gigantic_patch14_clip_224.metaclip2_worldwidez1vit_gigantic_patch14_clip_378.metaclip2_worldwidez(vit_base_patch32_clip_224.metaclip_2pt5bz(vit_base_patch16_clip_224.metaclip_2pt5bz)vit_large_patch14_clip_224.metaclip_2pt5bz(vit_huge_patch14_clip_224.metaclip_2pt5bz-vit_huge_patch14_clip_224.metaclip_altogetherz,vit_gigantic_patch14_clip_224.metaclip_2pt5bz'vit_base_patch32_clip_224.metaclip_400mz'vit_base_patch16_clip_224.metaclip_400mz(vit_large_patch14_clip_224.metaclip_400mz vit_base_patch32_clip_224.openai)r  r  r   r   r)  z vit_base_patch16_clip_224.openaiz!vit_large_patch14_clip_224.openai)r  r  r   r   ry  r)  z!vit_large_patch14_clip_336.openai)r  r  r   r   ry  rw  r)  z/vit_large_patch14_clip_224.apple_mclip2_dfndr2bz
apple-amlr)r  r)  r   r   ry  r}  z#vit_base_patch32_plus_256.untrained)rs  rw  ry  z#vit_base_patch16_plus_240.untrainedz$vit_small_patch16_36x1_224.untrainedz$vit_small_patch16_18x2_224.untrainedz#vit_base_patch16_18x2_224.untrainedz)eva_large_patch14_196.in22k_ft_in22k_in1k)r      r  )r  r}  r   r   rw  ry  z)eva_large_patch14_336.in22k_ft_in22k_in1k)r  r}  r   r   rw  ry  r  z#eva_large_patch14_196.in22k_ft_in1kz#eva_large_patch14_336.in22k_ft_in1kzflexivit_small.1200ep_in1kzEhttps://storage.googleapis.com/big_vision/flexivit/flexivit_s_i1k.npz)rs  r  r  rw  ry  zflexivit_small.600ep_in1kzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_s_i1k_600ep.npzzflexivit_small.300ep_in1kzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_s_i1k_300ep.npzzflexivit_base.1200ep_in1kzEhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i1k.npzzflexivit_base.600ep_in1kzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i1k_600ep.npzzflexivit_base.300ep_in1kzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i1k_300ep.npzzflexivit_base.1000ep_in21kzMhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i21k_1000ep.npz)rs  r  r  rw  ry  r)  zflexivit_base.300ep_in21kzLhttps://storage.googleapis.com/big_vision/flexivit/flexivit_b_i21k_300ep.npzzflexivit_large.1200ep_in1kzEhttps://storage.googleapis.com/big_vision/flexivit/flexivit_l_i1k.npzzflexivit_large.600ep_in1kzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_l_i1k_600ep.npzzflexivit_large.300ep_in1kzKhttps://storage.googleapis.com/big_vision/flexivit/flexivit_l_i1k_300ep.npzzflexivit_base.patch16_in21kzIhttps://storage.googleapis.com/big_vision/flexivit/vit_b16_i21k_300ep.npzzflexivit_base.patch30_in21kzIhttps://storage.googleapis.com/big_vision/flexivit/vit_b30_i21k_300ep.npzz!vit_base_patch16_xp_224.untrainedz"vit_large_patch14_xp_224.untrainedz!vit_huge_patch14_xp_224.untrainedzvit_base_patch16_224.maezEhttps://dl.fbaipublicfiles.com/mae/pretrain/mae_pretrain_vit_base.pth)rs  r  r}  r   r   r)  zvit_large_patch16_224.maezFhttps://dl.fbaipublicfiles.com/mae/pretrain/mae_pretrain_vit_large.pthzvit_huge_patch14_224.maezEhttps://dl.fbaipublicfiles.com/mae/pretrain/mae_pretrain_vit_huge.pthz#vit_huge_patch14_gap_224.in1k_ijepaz?https://dl.fbaipublicfiles.com/ijepa/IN1K-vit.h.14-300e.pth.tar)rs  r}  r   r   r)  z$vit_huge_patch14_gap_224.in22k_ijepaz@https://dl.fbaipublicfiles.com/ijepa/IN22K-vit.h.14-900e.pth.tarz#vit_huge_patch16_gap_448.in1k_ijepazEhttps://dl.fbaipublicfiles.com/ijepa/IN1K-vit.h.16-448px-300e.pth.tar)rs  r}  rw  ry  r   r   r)  z%vit_giant_patch16_gap_224.in22k_ijepaz@https://dl.fbaipublicfiles.com/ijepa/IN22K-vit.g.16-600e.pth.tarz$vit_base_patch32_siglip_256.v2_webli)r  rw  r)  z$vit_base_patch16_siglip_224.v2_webliz!vit_base_patch16_siglip_224.webliz$vit_base_patch16_siglip_256.v2_webliz!vit_base_patch16_siglip_256.webliz&vit_base_patch16_siglip_256.webli_i18nz$vit_base_patch16_siglip_384.v2_webliz!vit_base_patch16_siglip_384.webliz$vit_base_patch16_siglip_512.v2_webli)r   r  r  z!vit_base_patch16_siglip_512.webliz%vit_large_patch16_siglip_256.v2_webliz"vit_large_patch16_siglip_256.webliz%vit_large_patch16_siglip_384.v2_webliz"vit_large_patch16_siglip_384.webliz%vit_large_patch16_siglip_512.v2_webliz&vit_so400m_patch14_siglip_224.v2_webliz#vit_so400m_patch14_siglip_224.webliz&vit_so400m_patch14_siglip_378.v2_webliz#vit_so400m_patch14_siglip_378.webliz#vit_so400m_patch14_siglip_384.webliz&vit_so400m_patch16_siglip_256.v2_webliz(vit_so400m_patch16_siglip_256.webli_i18nz&vit_so400m_patch16_siglip_384.v2_webliz&vit_so400m_patch16_siglip_512.v2_webliz(vit_giantopt_patch16_siglip_256.v2_webliz(vit_giantopt_patch16_siglip_384.v2_webliz(vit_base_patch32_siglip_gap_256.v2_webliz(vit_base_patch16_siglip_gap_224.v2_webliz%vit_base_patch16_siglip_gap_224.webliz(vit_base_patch16_siglip_gap_256.v2_webliz%vit_base_patch16_siglip_gap_256.webliz*vit_base_patch16_siglip_gap_256.webli_i18nz(vit_base_patch16_siglip_gap_384.v2_webliz%vit_base_patch16_siglip_gap_384.webliz(vit_base_patch16_siglip_gap_512.v2_webliz%vit_base_patch16_siglip_gap_512.webliz)vit_large_patch16_siglip_gap_256.v2_webliz&vit_large_patch16_siglip_gap_256.webliz)vit_large_patch16_siglip_gap_384.v2_webliz&vit_large_patch16_siglip_gap_384.webliz)vit_large_patch16_siglip_gap_512.v2_webliz*vit_so400m_patch14_siglip_gap_224.v2_webliz'vit_so400m_patch14_siglip_gap_224.webliz*vit_so400m_patch14_siglip_gap_224.pali_mixz)vit_so400m_patch14_siglip_gap_224.pali_ptz-vit_so400m_patch14_siglip_gap_224.pali2_3b_ptz.vit_so400m_patch14_siglip_gap_224.pali2_10b_ptz*vit_so400m_patch14_siglip_gap_378.v2_webliz'vit_so400m_patch14_siglip_gap_378.webliz'vit_so400m_patch14_siglip_gap_384.webliz*vit_so400m_patch14_siglip_gap_448.pali_mixz)vit_so400m_patch14_siglip_gap_448.pali_ptz2vit_so400m_patch14_siglip_gap_448.pali_refcoco_segz-vit_so400m_patch14_siglip_gap_448.pali_ocrvqaz-vit_so400m_patch14_siglip_gap_448.pali2_3b_ptz.vit_so400m_patch14_siglip_gap_448.pali2_10b_ptz0vit_so400m_patch14_siglip_gap_448.pali2_3b_docciz1vit_so400m_patch14_siglip_gap_448.pali2_10b_docciz)vit_so400m_patch14_siglip_gap_896.pali_pt)r     r  z2vit_so400m_patch14_siglip_gap_896.pali_refcoco_segz-vit_so400m_patch14_siglip_gap_896.pali_ocrvqaz-vit_so400m_patch14_siglip_gap_896.pali2_3b_ptz.vit_so400m_patch14_siglip_gap_896.pali2_10b_ptz*vit_so400m_patch16_siglip_gap_256.v2_webliz,vit_so400m_patch16_siglip_gap_256.webli_i18nz*vit_so400m_patch16_siglip_gap_384.v2_webliz*vit_so400m_patch16_siglip_gap_512.v2_webliz,vit_giantopt_patch16_siglip_gap_256.v2_webliz,vit_giantopt_patch16_siglip_gap_384.v2_webliz+vit_so400m_patch14_siglip_378.webli_ft_in1kz/vit_so400m_patch14_siglip_gap_378.webli_ft_in1kz,vit_xsmall_patch16_clip_224.tinyclip_yfcc15m)r  r}  r   r   r)  z.vit_medium_patch32_clip_224.tinyclip_laion400mz,vit_medium_patch16_clip_224.tinyclip_yfcc15mz/vit_betwixt_patch32_clip_224.tinyclip_laion400mz%vit_wee_patch16_reg1_gap_256.sbb_in1kz/vit_dwee_patch16_reg1_gap_256.sbb_nadamuon_in1kz&vit_dwee_patch16_reg1_gap_256.sbb_in1kz&vit_pwee_patch16_reg1_gap_256.sbb_in1kz0vit_dpwee_patch16_reg1_gap_256.sbb_nadamuon_in1kz'vit_dpwee_patch16_reg1_gap_256.sbb_in1kz1vit_little_patch16_reg1_gap_256.sbb_in12k_ft_in1kz)vit_little_patch16_reg1_gap_256.sbb_in12k)r  r)  rw  ry  z(vit_little_patch16_reg4_gap_256.sbb_in1kz2vit_dlittle_patch16_reg1_gap_256.sbb_nadamuon_in1kz(vit_medium_patch16_reg1_gap_256.sbb_in1kz1vit_medium_patch16_reg4_gap_256.sbb_in12k_ft_in1kz(vit_medium_patch16_reg4_gap_256.sbb_in1kz)vit_medium_patch16_reg4_gap_256.sbb_in12kz8vit_mediumd_patch16_reg4_gap_256.sbb2_e200_in12k_ft_in1kz2vit_mediumd_patch16_reg4_gap_256.sbb_in12k_ft_in1kz0vit_mediumd_patch16_reg4_gap_256.sbb2_e200_in12kz*vit_mediumd_patch16_reg4_gap_256.sbb_in12kz8vit_mediumd_patch16_reg4_gap_384.sbb2_e200_in12k_ft_in1kz)vit_betwixt_patch16_reg1_gap_256.sbb_in1kz8vit_betwixt_patch16_reg4_gap_256.sbb2_e200_in12k_ft_in1kz2vit_betwixt_patch16_reg4_gap_256.sbb_in12k_ft_in1kz)vit_betwixt_patch16_reg4_gap_256.sbb_in1kz0vit_betwixt_patch16_reg4_gap_256.sbb2_e200_in12kz*vit_betwixt_patch16_reg4_gap_256.sbb_in12kz8vit_betwixt_patch16_reg4_gap_384.sbb2_e200_in12k_ft_in1kz'vit_base_patch16_reg4_gap_256.untrained)rw  z6vit_so150m_patch16_reg4_gap_256.sbb_e250_in12k_ft_in1kz.vit_so150m_patch16_reg4_gap_256.sbb_e250_in12kz6vit_so150m_patch16_reg4_gap_384.sbb_e250_in12k_ft_in1kz)vit_so150m_patch16_reg4_map_256.untrainedz7vit_so150m2_patch16_reg1_gap_256.sbb_e200_in12k_ft_in1kz/vit_so150m2_patch16_reg1_gap_256.sbb_e200_in12kz7vit_so150m2_patch16_reg1_gap_384.sbb_e200_in12k_ft_in1kz7vit_so150m2_patch16_reg1_gap_448.sbb_e200_in12k_ft_in1kz$vit_intern300m_patch14_448.ogvl_distz$vit_intern300m_patch14_448.ogvl_2pt5)r  r   r   rw  ry  r)  z aimv2_large_patch14_224.apple_pt)r  r   r   r}  ry  r)  z%aimv2_large_patch14_224.apple_pt_distzaimv2_huge_patch14_224.apple_ptzaimv2_1b_patch14_224.apple_ptzaimv2_3b_patch14_224.apple_ptz aimv2_large_patch14_336.apple_pt)r  r   r   r}  rw  ry  r)  z%aimv2_large_patch14_336.apple_pt_distzaimv2_huge_patch14_336.apple_ptzaimv2_1b_patch14_336.apple_ptzaimv2_3b_patch14_336.apple_ptz aimv2_large_patch14_448.apple_ptzaimv2_huge_patch14_448.apple_ptzaimv2_1b_patch14_448.apple_ptzaimv2_3b_patch14_448.apple_ptztest_vit.r160_in1k)r      r  ztest_vit2.r160_in1kztest_vit3.r160_in1kztest_vit4.r160_in1kz$beit3_base_patch16_224.in22k_ft_in1kz-beit3_base_patch16_224.indomain_in22k_ft_in1kz%beit3_large_patch16_224.in22k_ft_in1kz.beit3_large_patch16_224.indomain_in22k_ft_in1kz!beit3_giant_patch14_224.untrained)rs  r   r   ry  z!beit3_giant_patch14_336.untrained)rs  rw  r   r   ry  zbeit3_base_patch16_224.ptz"beit3_base_patch16_224.indomain_ptzbeit3_large_patch16_224.ptz#beit3_large_patch16_224.indomain_ptr  	quickgelur  _clip__clip_quickgelu_TIMM_USE_NAFLEXVITfalsetruevariant
pretrained
use_naflex	NaFlexVitc           	      0   |t         }|rddlm}  || |fi |S |j                  dd      }d| v rt	        t
        dd      }nt
        }|j                  d	d
      }d| v r|j                  dd       dk7  rd}t        t        | |f||t        |d      d|S )Nr.   )_create_naflexvit_from_classicout_indicesr   flexir  F)r  r  pretrained_strictTsiglipr*  r,  getter)r  feature_cls)pretrained_filter_fnr  feature_cfg)
_USE_NAFLEX_DEFAULT	naflexvitr  rJ  r   rr  rL   r/   r8   rZ  )r  r  r  rN   r  r  
_filter_fnstricts           rO   _create_vision_transformerr    s     (
=-gzLVLL**]A.K' 1W\]
)
 ZZ+T2F7vzz->%G ( [hG  rQ   c           	      R    t        dddd      }t        dd| it        |fi |}|S )z ViT-Tiny (Vit-Ti/16)
    r!     r$  r   r'  r.  rF   r>   r  )vit_tiny_patch16_224rZ  r  r  rN   
model_argsr  s       rO   r  r     8     s"JJ&s*sX\]gXrkqXrsELrQ   c           	      R    t        dddd      }t        dd| it        |fi |}|S )z% ViT-Tiny (Vit-Ti/16) @ 384x384.
    r!  r  r$  r   r  r  )vit_tiny_patch16_384r  r  s       rO   r  r  )  r  rQ   c           	      R    t        dddd      }t        dd| it        |fi |}|S )z ViT-Small (ViT-S/32)
        r  r$     r  r  )vit_small_patch32_224r  r  s       rO   r  r  2  8     s"JJ&t:tY]^hYslrYstELrQ   c           	      R    t        dddd      }t        dd| it        |fi |}|S )z& ViT-Small (ViT-S/32) at 384x384.
    r  r  r$  r  r  r  )vit_small_patch32_384r  r  s       rO   r  r  ;  r  rQ   c           	      R    t        dddd      }t        dd| it        |fi |}|S ) ViT-Small (ViT-S/16)
    r!  r  r$  r  r  r  )vit_small_patch16_224r  r  s       rO   r  r  D  r  rQ   c           	      R    t        dddd      }t        dd| it        |fi |}|S )r  r!  r  r$  r  r  r  )vit_small_patch16_384r  r  s       rO   r  r  M  r  rQ   c           	      R    t        dddd      }t        dd| it        |fi |}|S )z ViT-Small (ViT-S/8)
       r  r$  r  r  r  )vit_small_patch8_224r  r  s       rO   r  r  V  s8     cqIJ&s*sX\]gXrkqXrsELrQ   c           	      R    t        dddd      }t        dd| it        |fi |}|S )z ViT-Base (ViT-B/32) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k, source https://github.com/google-research/vision_transformer.
    r  r#  r$  r  r  )vit_base_patch32_224r  r  s       rO   r  r  _  8    
 s"KJ&s*sX\]gXrkqXrsELrQ   c           	      R    t        dddd      }t        dd| it        |fi |}|S )z ViT-Base model (ViT-B/32) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    r  r#  r$  r  r  )vit_base_patch32_384r  r  s       rO   r  r  i  r  rQ   c           	      R    t        dddd      }t        dd| it        |fi |}|S )z ViT-Base (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 224x224, source https://github.com/google-research/vision_transformer.
    r!  r#  r$  r  r  )vit_base_patch16_224r  r  s       rO   r  r  s  r  rQ   c           	      R    t        dddd      }t        dd| it        |fi |}|S )z ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    r!  r#  r$  r  r  )vit_base_patch16_384r  r  s       rO   r  r  }  r  rQ   c           	      R    t        dddd      }t        dd| it        |fi |}|S )z ViT-Base (ViT-B/8) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 224x224, source https://github.com/google-research/vision_transformer.
    r  r#  r$  r  r  )vit_base_patch8_224r  r  s       rO   r  r    s8    
 crJJ&rrW[\fWqjpWqrELrQ   c           	      R    t        dddd      }t        dd| it        |fi |}|S )zo ViT-Large model (ViT-L/32) from original paper (https://arxiv.org/abs/2010.11929). No pretrained weights.
    r  r     r!  r  r  )vit_large_patch32_224r  r  s       rO   r  r    8     t2LJ&t:tY]^hYslrYstELrQ   c           	      R    t        dddd      }t        dd| it        |fi |}|S )z ViT-Large model (ViT-L/32) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    r  r  r  r!  r  r  )vit_large_patch32_384r  r  s       rO   r  r    8    
 t2LJ&t:tY]^hYslrYstELrQ   c           	      R    t        dddd      }t        dd| it        |fi |}|S )z ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 224x224, source https://github.com/google-research/vision_transformer.
    r!  r  r  r  r  )vit_large_patch16_224r  r  s       rO   r  r    r  rQ   c           	      R    t        dddd      }t        dd| it        |fi |}|S )z ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929).
    ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer.
    r!  r  r  r  r  )vit_large_patch16_384r  r  s       rO   r  r    r  rQ   c           	      R    t        dddd      }t        dd| it        |fi |}|S )z  ViT-Large model (ViT-L/14)
       r  r  r!  r  r  )vit_large_patch14_224r  r  s       rO   r  r    r  rQ   c           	      R    t        dddd      }t        dd| it        |fi |}|S )zW ViT-Huge model (ViT-H/14) from original paper (https://arxiv.org/abs/2010.11929).
    r  r  r  r!  r  r  )vit_huge_patch14_224r  r  s       rO   r  r    s8     t2LJ&s*sX\]gXrkqXrsELrQ   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	zq ViT-Giant (little-g) model (ViT-g/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560
    r    tE]t@(   r!  r'  r.  rV   rF   r>   r  )vit_giant_patch14_224r  r  s       rO   r  r    s;     tuBZ\]J&t:tY]^hYslrYstELrQ   c           	      V    t        ddddd      }t        	 dd| it        |fi |}|S )	zq ViT-Gigantic (big-G) model (ViT-G/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560
    r    ;;@0   r!  r  r  )vit_gigantic_patch14_224r  r  s       rO   r  r    sG     tuBZ\]J&"Y/9Y=A*=WPV=WYELrQ   c           	      V    t        ddddd      }t        	 dd| it        |fi |}|S )z ViT-Base (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929).
    Weights taken from: https://github.com/Alibaba-MIIL/ImageNet21K
    r!  r#  r$  F)r'  r.  rF   r>   r?   r  )vit_base_patch16_224_miilr  r  s       rO   r  r    sG    
 s"UZ[J&#Z0:Z>B:>XQW>XZELrQ   c                 ^    t        ddddddddd	      }t        	 d
d	| it        |fi |}|S )zB ViT-Medium (ViT-M/16) w/o class token, w/ avg-pool @ 240x240
    r!  r  r$  r  Fr  r}  	r'  r.  rF   r>   r/  r*  r?   rY   r5  r  )vit_medium_patch16_gap_240r  r  s       rO   r  r    U     B!EtULJ '$[1;[?CJ?YRX?Y[ELrQ   c                 ^    t        ddddddddd	      }t        	 d
d	| it        |fi |}|S )zB ViT-Medium (ViT-M/16) w/o class token, w/ avg-pool @ 256x256
    r!  r  r$  r  Fr  r}  r  r  )vit_medium_patch16_gap_256r  r  s       rO   r  r    r  rQ   c                 ^    t        ddddddddd	      }t        	 d
d	| it        |fi |}|S )zB ViT-Medium (ViT-M/16) w/o class token, w/ avg-pool @ 384x384
    r!  r  r$  r  Fr  r}  r  r  )vit_medium_patch16_gap_384r  r  s       rO   r  r     r  rQ   c                 ^    t        ddddddddd	      }t        	 d
d	| it        |fi |}|S )zC ViT-Betwixt (ViT-b/16) w/o class token, w/ avg-pool @ 256x256
    r!  r  r$  
   Fr  r}  r  r  )vit_betwixt_patch16_gap_256r  r  s       rO   r  r    sU     B"%EtULJ '%\2<\@DZ@ZSY@Z\ELrQ   c           	      Z    t        ddddddd      }t        	 dd| it        |fi |}|S )	z@ ViT-Base (ViT-B/16) w/o class token, w/ avg-pool @ 224x224
    r!  r#  r$  Fr  r'  r.  rF   r>   r/  r*  r5  r  )vit_base_patch16_gap_224r  r  s       rO   r  r    sP     B"%]blqsJ&"Y/9Y=A*=WPV=WYELrQ   c           	      Z    t        ddddddd      }t        	 d	d| it        |fi |}|S )
z; ViT-Huge model (ViT-H/14) w/ no class token, avg pool
    r  r  r  r!  Fr  r  r  )vit_huge_patch14_gap_224r  r  s       rO   r  r  #  P     R25^cmrtJ&"Y/9Y=A*=WPV=WYELrQ   c           	      Z    t        ddddddd      }t        	 dd| it        |fi |}|S )	zE ViT-Huge model (ViT-H/16) w/ no class token, avg pool @ 448x448
    r!  r  r  Fr  r  r  )vit_huge_patch16_gap_448r  r  s       rO   r   r   .  r  rQ   c           
      \    t        dddddddd      }t        	 d	d| it        |fi |}|S )
zH ViT-Giant (little-gg) model (ViT-g/16) w/ no class token, avg pool
    r!  r  r  r  Fr  r'  r.  rF   r>   rV   r/  r*  r5  r  )vit_giant_patch16_gap_224r  r  s       rO   r  r  9  sR     R2ue=J '#Z0:Z>B:>XQW>XZELrQ   c           
      t    t        ddddt        t        d            }t        	 d	d| it        |fi |}|S )
Nr  r  r  Tr   r   r.  rF   r>   r3  rE   r  )vit_xsmall_patch16_clip_224rZ  r   r   r  r  s       rO   r  r  E  P     2TV]^gmqVrsJ&%\2<\@DZ@ZSY@Z\ELrQ   c                 v    t        dddddt        t        d            }t        	 d
d	| it        |fi |}|S )Nr  r  r$  r  Tr   r  r'  r.  rF   r>   r3  rE   r  )vit_medium_patch32_clip_224r  r  s       rO   r  r  N  sV     B!dW^_hnrWsuJ&%\2<\@DZ@ZSY@Z\ELrQ   c           
      t    t        ddddt        t        d            }t        	 d	d| it        |fi |}|S )
Nr  r$  r  Tr   r  r  r  )vit_medium_patch16_clip_224r  r  s       rO   r  r  X  r	  rQ   c                 v    t        dddddt        t        d            }t        	 d
d	| it        |fi |}|S )Nr  r  r$  r  Tr   r  r  r  )vit_betwixt_patch32_clip_224r  r  s       rO   r  r  a  sV     B"tX_`iosXtvJ&&]3=]AEjA[TZA[]ELrQ   c                 v    t        dddddt        t        d            }t        	 d	d| it        |fi |}|S )
) ViT-B/32 CLIP image tower @ 224x224
    r  r#  r$  Tr   r  r  r  )vit_base_patch32_clip_224r  r  s       rO   r  r  k  V     B"tX_`iosXtvJ&#Z0:Z>B:>XQW>XZELrQ   c                 v    t        dddddt        t        d            }t        	 d	d| it        |fi |}|S )
z) ViT-B/32 CLIP image tower @ 256x256
    r  r#  r$  Tr   r  r  r  )vit_base_patch32_clip_256r  r  s       rO   r  r  v  r  rQ   c                 v    t        dddddt        t        d            }t        	 d	d| it        |fi |}|S )
z) ViT-B/32 CLIP image tower @ 384x384
    r  r#  r$  Tr   r  r  r  )vit_base_patch32_clip_384r  r  s       rO   r  r    r  rQ   c                 v    t        dddddt        t        d            }t        	 d	d| it        |fi |}|S )
z) ViT-B/32 CLIP image tower @ 448x448
    r  r#  r$  Tr   r  r  r  )vit_base_patch32_clip_448r  r  s       rO   r  r    r  rQ   c                 v    t        dddddt        t        d            }t        	 d	d| it        |fi |}|S )
z ViT-B/16 CLIP image tower
    r!  r#  r$  Tr   r  r  r  )vit_base_patch16_clip_224r  r  s       rO   r  r    r  rQ   c                 v    t        dddddt        t        d            }t        	 d	d| it        |fi |}|S )
z) ViT-B/16 CLIP image tower @ 384x384
    r!  r#  r$  Tr   r  r  r  )vit_base_patch16_clip_384r  r  s       rO   r  r    r  rQ   c                 v    t        dddddt        t        d            }t        	 d
d	| it        |fi |}|S )z5 ViT-Base (ViT-B/16+) CLIP image tower @ 240x240
    r!  r  r$  r  Tr   r  r  r  )vit_base_patch16_plus_clip_240r  r  s       rO   r   r     sV     B"tX_`iosXtvJ&(_5?_CG
C]V\C]_ELrQ   c                 v    t        dddddt        t        d            }t        	 d
d	| it        |fi |}|S )z1 ViT-Large model (ViT-L/14) CLIP image tower
    r  r  r  r!  Tr   r  r  r  )vit_large_patch14_clip_224r  r  s       rO   r"  r"    V     R2Y`ajptYuwJ&$[1;[?CJ?YRX?Y[ELrQ   c                 v    t        dddddt        t        d            }t        	 d
d	| it        |fi |}|S )z; ViT-Large model (ViT-L/14) CLIP image tower @ 336x336
    r  r  r  r!  Tr   r  r  r  )vit_large_patch14_clip_336r  r  s       rO   r%  r%    r#  rQ   c                 v    t        dddddt        t        d            }t        	 d
d	| it        |fi |}|S )z1 ViT-Huge model (ViT-H/14) CLIP image tower.
    r  r  r  r!  Tr   r  r  r  )vit_huge_patch14_clip_224r  r  s       rO   r'  r'    V     R2Y`ajptYuwJ&#Z0:Z>B:>XQW>XZELrQ   c                 v    t        dddddt        t        d            }t        	 d
d	| it        |fi |}|S )z: ViT-Huge model (ViT-H/14) CLIP image tower @ 336x336
    r  r  r  r!  Tr   r  r  r  )vit_huge_patch14_clip_336r  r  s       rO   r*  r*    r(  rQ   c                 v    t        dddddt        t        d            }t        	 d
d	| it        |fi |}|S )z: ViT-Huge model (ViT-H/14) CLIP image tower @ 378x378
    r  r  r  r!  Tr   r  r  r  )vit_huge_patch14_clip_378r  r  s       rO   r,  r,    r(  rQ   c                 x    t        ddddddt        t        d      	      }t        	 dd
| it        |fi |}|S )z ViT-Giant (little-g) model (ViT-g/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560
    Pretrained weights from CLIP image tower.
    r  r  r  r  r!  Tr   r  r'  r.  rV   rF   r>   r3  rE   r  )vit_giant_patch14_clip_224r  r  s       rO   r/  r/    sX    
 bBY]9$/J '$[1;[?CJ?YRX?Y[ELrQ   c                 x    t        ddddddt        t        d      	      }t        	 dd
| it        |fi |}|S ) ViT-bigG model (ViT-G/14) from `Scaling Vision Transformers` - https://arxiv.org/abs/2106.04560
    Pretrained weights from CLIP image tower.
    r  r  r  r  r!  Tr   r  r.  r  )vit_gigantic_patch14_clip_224r  r  s       rO   r2  r2    X    
 bBY]9$/J ''^4>^BFzB\U[B\^ELrQ   c                 x    t        ddddddt        t        d      	      }t        	 dd
| it        |fi |}|S )r1  r  r  r  r  r!  Tr   r  r.  r  )vit_gigantic_patch14_clip_378r  r  s       rO   r5  r5    r3  rQ   c                 x    t        dddddt        t        d      d      }t        	 d
d	| it        |fi |}|S )r  r  r#  r$  Tr   r  
quick_gelur'  r.  rF   r>   r3  rE   r[   r  )#vit_base_patch32_clip_quickgelu_224r  r  s       rO   r9  r9    W     B"t9$/<J '-d:DdHLZHb[aHbdELrQ   c                 x    t        dddddt        t        d      d      }t        	 d
d	| it        |fi |}|S )z0 ViT-B/16 CLIP image tower w/ QuickGELU act
    r!  r#  r$  Tr   r  r7  r8  r  )#vit_base_patch16_clip_quickgelu_224r  r  s       rO   r<  r<  &  r:  rQ   c                 x    t        dddddt        t        d      d	      }t        	 dd
| it        |fi |}|S )zB ViT-Large model (ViT-L/14) CLIP image tower w/ QuickGELU act
    r  r  r  r!  Tr   r  r7  r8  r  )$vit_large_patch14_clip_quickgelu_224r  r  s       rO   r>  r>  3  W     R29$/<J '.e;EeIMjIc\bIceELrQ   c                 x    t        dddddt        t        d      d	      }t        	 dd
| it        |fi |}|S )zL ViT-Large model (ViT-L/14) CLIP image tower @ 336x336 w/ QuickGELU act
    r  r  r  r!  Tr   r  r7  r8  r  )$vit_large_patch14_clip_quickgelu_336r  r  s       rO   rA  rA  @  r?  rQ   c                 x    t        dddddt        t        d      d	      }t        	 dd
| it        |fi |}|S )zB ViT-Huge model (ViT-H/14) CLIP image tower w/ QuickGELU act.
    r  r  r  r!  Tr   r  r7  r8  r  )#vit_huge_patch14_clip_quickgelu_224r  r  s       rO   rC  rC  M  W     R29$/<J '-d:DdHLZHb[aHbdELrQ   c                 x    t        dddddt        t        d      d	      }t        	 dd
| it        |fi |}|S )zK ViT-Huge model (ViT-H/14) CLIP image tower @ 378x378 w/ QuickGELU act
    r  r  r  r!  Tr   r  r7  r8  r  )#vit_huge_patch14_clip_quickgelu_378r  r  s       rO   rF  rF  Z  rD  rQ   c                 z    t        ddddddt        t        d      d	
      }t        	 dd| it        |fi |}|S )z0 ViT-bigG model (ViT-G/14) w/ QuickGELU act
    r  r  r  r  r!  Tr   r  r7  )r'  r.  rV   rF   r>   r3  rE   r[   r  )'vit_gigantic_patch14_clip_quickgelu_224r  r  s       rO   rH  rH  g  s[     bBY]9$/<J '1h>HhLPQ[Lf_eLfhELrQ   c           	      V    t        ddddd      }t        	 dd| it        |fi |}|S )	z ViT-Base (ViT-B/32+)
    r  r  r$  r  r   r'  r.  rF   r>   rY   r  )vit_base_patch32_plus_256r  r  s       rO   rK  rK  v  G     s"X\]J&#Z0:Z>B:>XQW>XZELrQ   c           	      V    t        ddddd      }t        	 dd| it        |fi |}|S )	z ViT-Base (ViT-B/16+)
    r!  r  r$  r  r   rJ  r  )vit_base_patch16_plus_240r  r  s       rO   rN  rN    rL  rQ   c                 f    t        dddddddt        d	      }t        	 d	d| it        |fi |}|S )
z/ ViT-Base (ViT-B/16) w/ residual post-norm
    r!  r#  r$  Fr   r  )	r'  r.  rF   r>   r?   rY   r/  rG  r*  r  )vit_base_patch16_rpn_224)rZ  r   r  r  s       rO   rP  rP    sV     B"uZ^LeEJ '"Y/9Y=A*=WPV=WYELrQ   c           	      V    t        ddddd      }t        	 dd| it        |fi |}|S )	a   ViT-Base w/ LayerScale + 36 x 1 (36 block serial) config. Experimental, may remove.
    Based on `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795
    Paper focuses on 24x2 + 48x1 for 'Small' width but those are extremely slow.
    r!  r  $   r  r   rJ  r  )vit_small_patch16_36x1_224r  r  s       rO   rS  rS    sG     s"W[\J&$[1;[?CJ?YRX?Y[ELrQ   c           	      `    t        dddddt              }t        	 dd| it        |fi |}|S )	a   ViT-Small w/ LayerScale + 18 x 2 (36 block parallel) config. Experimental, may remove.
    Based on `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795
    Paper focuses on 24x2 + 48x1 for 'Small' width but those are extremely slow.
    r!  r     r  r   r'  r.  rF   r>   rY   rG  r  )vit_small_patch16_18x2_224rZ  r  r  r  s       rO   rW  rW    sM     B!XkmJ&$[1;[?CJ?YRX?Y[ELrQ   c           	      `    t        dddddt              }t        	 dd| it        |fi |}|S )	z ViT-Base w/ LayerScale + 18 x 2 (36 block parallel) config. Experimental, may remove.
    Based on `Three things everyone should know about Vision Transformers` - https://arxiv.org/abs/2203.09795
    r!  r#  rU  r$  r   rV  r  )vit_base_patch16_18x2_224rX  r  s       rO   rZ  rZ    sM    
 B"$YlnJ&#Z0:Z>B:>XQW>XZELrQ   c           	      V    t        ddddd      }t        	 dd| it        |fi |}|S )	zG EVA-large model https://arxiv.org/abs/2211.07636 /via MAE MIM pretrainr  r  r  r!  r  r'  r.  rF   r>   r*  r  )eva_large_patch14_196r  r  s       rO   r]  r]    sF     t2Y^_J&V,6V:>z:TV:TVELrQ   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	zF EVA-large model https://arxiv.org/abs/2211.07636 via MAE MIM pretrainr  r  r  r!  r  r\  r  )eva_large_patch14_336r  r  s       rO   r_  r_    s;     t2Y^_J&t:tY]^hYslrYstELrQ   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	z FlexiViT-Small
    r!  r  r$  r  Tr'  r.  rF   r>   r1  r  )flexivit_smallr  r  s       rO   rb  rb    s;     s"Z^_J&mJmRVWaRlekRlmELrQ   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )z FlexiViT-Base
    r!  r#  r$  Tra  r  )flexivit_baser  r  s       rO   rd  rd    s;     s"[_`J&l:lQUV`QkdjQklELrQ   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )z FlexiViT-Large
    r!  r  r  Tra  r  )flexivit_larger  r  s       rO   rf  rf    s;     t2\`aJ&mJmRVWaRlekRlmELrQ   c                 p    t        ddddddt        t        dd
      }t        	 dd| it        |fi |}|S )	H ViT-Large model (ViT-L/14) w/ parallel blocks and qk norm enabled.
    r!  r#  r$  TF
r'  r.  rF   r>   r3  r1  rE   rG  r?   r@   r  )vit_base_patch16_xp_224rZ  r   r   r  r  s       rO   rj  rj    sX     B"t\`%9ESWJ '!X.8X<@<Vv<VXELrQ   c                 p    t        ddddddt        t        dd
      }t        	 d	d| it        |fi |}|S )
rh  r  r  r  r!  TFri  r  )vit_large_patch14_xp_224rk  r  s       rO   rm  rm    sY     R2]a%9ESWJ '"Y/9Y=A*=WPV=WYELrQ   c                 p    t        ddddddt        t        dd
      }t        	 d	d| it        |fi |}|S )
zG ViT-Huge model (ViT-H/14) w/ parallel blocks and qk norm enabled.
    r  r  r  r!  TFri  r  )vit_huge_patch14_xp_224rk  r  s       rO   ro  ro    sX     R2]a%9ESWJ '!X.8X<@<Vv<VXELrQ   c           	      V    t        ddddd      }t        	 dd| it        |fi |}|S )	z ViT-S/14 for DINOv2
    r  r  r$  r  r   rJ  r  )vit_small_patch14_dinov2r  r  s       rO   rq  rq    sG     s"W[\J&"Y/9Y=A*=WPV=WYELrQ   c           	      V    t        ddddd      }t        	 dd| it        |fi |}|S )z ViT-B/14 for DINOv2
    r  r#  r$  r   rJ  r  )vit_base_patch14_dinov2r  r  s       rO   rs  rs    sF     s"X\]J&!X.8X<@<Vv<VXELrQ   c           	      V    t        ddddd      }t        	 dd| it        |fi |}|S )	z ViT-L/14 for DINOv2
    r  r  r  r!  r   rJ  r  )vit_large_patch14_dinov2r  r  s       rO   ru  ru  "  sG     t2Y]^J&"Y/9Y=A*=WPV=WYELrQ   c           
          t        ddddddt        t        j                        }t	        	 d	d| it        |fi |}|S )
 ViT-G/14 for DINOv2
    r     r  r  r   h˹WU@)r'  r.  rF   r>   rY   rV   r\   r[   r  )vit_giant_patch14_dinov2rZ  r   rk   SiLUr  r  s       rO   rz  rz  ,  sV     R24J '"Y/9Y=A*=WPV=WYELrQ   c           	      Z    t        ddddddd      }t        	 d
d	| it        |fi |}|S )z( ViT-S/14 for DINOv2 w/ 4 registers
    r  r  r$  r  r   r  Tr'  r.  rF   r>   rY   r2  r1  r  )vit_small_patch14_reg4_dinov2r  r  s       rO   r  r  =  sP     B!TJ ''^4>^BFzB\U[B\^ELrQ   c           	      Z    t        ddddddd      }t        	 d	d| it        |fi |}|S )
z( ViT-B/14 for DINOv2 w/ 4 registers
    r  r#  r$  r   r  Tr~  r  )vit_base_patch14_reg4_dinov2r  r  s       rO   r  r  J  sP     B"$TJ '&]3=]AEjA[TZA[]ELrQ   c           	      Z    t        ddddddd      }t        	 d
d	| it        |fi |}|S )z( ViT-L/14 for DINOv2 w/ 4 registers
    r  r  r  r!  r   r  Tr~  r  )vit_large_patch14_reg4_dinov2r  r  s       rO   r  r  W  sP     R24TJ ''^4>^BFzB\U[B\^ELrQ   c                     t        ddddddt        t        j                  dd	
      }t	        	 dd
| it        |fi |}|S )rw  r  rx  r  r  r   ry  r  T
r'  r.  rF   r>   rY   rV   r\   r[   r2  r1  r  )vit_giant_patch14_reg4_dinov2r{  r  s       rO   r  r  d  s\     R24[f"''aPTJ ''^4>^BFzB\U[B\^ELrQ   c           	      Z    t        ddddddd      }t        	 d	d| it        |fi |}|S )
zR ViT-B/14 for TIPSv2 (DINOv2-style w/ 1 register token, LayerScale init=1.0).
    r  r#  r$  r  r.   Tr~  r  )vit_base_patch14_reg1_tipsv2r  r  s       rO   r  r  u  sP     B"#TJ '&]3=]AEjA[TZA[]ELrQ   c           	      Z    t        ddddddd      }t        	 d
d	| it        |fi |}|S )zR ViT-L/14 for TIPSv2 (DINOv2-style w/ 1 register token, LayerScale init=1.0).
    r  r  r  r!  r  r.   Tr~  r  )vit_large_patch14_reg1_tipsv2r  r  s       rO   r  r    sP     R23TJ ''^4>^BFzB\U[B\^ELrQ   c           
      \    t        dddddddd	      }t        	 dd
| it        |fi |}|S )zW SoViT-400M/14 for TIPSv2 (DINOv2-style w/ 1 register token, LayerScale init=1.0).
    r       r!  r  g98@r.   T)r'  r.  rF   r>   rY   rV   r2  r1  r  )vit_so400m_patch14_reg1_tipsv2r  r  s       rO   r  r    sR     R23!DJ '(_5?_CG
C]V\C]_ELrQ   c                     t        ddddddt        t        j                  dd	
      }t	        	 dd
| it        |fi |}|S )z^ ViT-G/14 for TIPSv2 (DINOv2-style w/ SwiGLU FFN, 1 register token, LayerScale init=1.0).
    r  rx  r  r  r  ry  r.   Tr  r  )vit_giant_patch14_reg1_tipsv2r{  r  s       rO   r  r    s[     R23TJ
 ''^4>^BFzB\U[B\^ELrQ   c           	      Z    t        ddddddd      }t        	 d	d| it        |fi |}|S )
Nr  r#  r$  Fr,  	gelu_tanhr'  r.  rF   r>   r/  r*  r[   r  )vit_base_patch32_siglip_256r  r  s       rO   r  r    sO    B"%]bJ '%\2<\@DZ@ZSY@Z\ELrQ   c           	      X    t        dddddd      }t        	 dd| it        |fi |}|S )	Nr!  r#  r$  Fr,  r'  r.  rF   r>   r/  r*  r  )vit_base_patch16_siglip_224r  r  s       rO   r  r    L    B"%]bJ '%\2<\@DZ@ZSY@Z\ELrQ   c           	      X    t        dddddd      }t        	 dd| it        |fi |}|S )	Nr!  r#  r$  Fr,  r  r  )vit_base_patch16_siglip_256r  r  s       rO   r  r    r  rQ   c           	      X    t        dddddd      }t        	 dd| it        |fi |}|S )	Nr!  r#  r$  Fr,  r  r  )vit_base_patch16_siglip_384r  r  s       rO   r  r    r  rQ   c           	      X    t        dddddd      }t        	 dd| it        |fi |}|S )	Nr!  r#  r$  Fr,  r  r  )vit_base_patch16_siglip_512r  r  s       rO   r  r    r  rQ   c           	      X    t        dddddd      }t        	 dd| it        |fi |}|S )	Nr!  r  r  Fr,  r  r  )vit_large_patch16_siglip_256r  r  s       rO   r  r    L    R25^cJ '&]3=]AEjA[TZA[]ELrQ   c           	      X    t        dddddd      }t        	 dd| it        |fi |}|S )	Nr!  r  r  Fr,  r  r  )vit_large_patch16_siglip_384r  r  s       rO   r  r    r  rQ   c           	      Z    t        ddddddd      }t        	 d	d| it        |fi |}|S )
Nr!  r  r  Fr,  r  r  r  )vit_large_patch16_siglip_512r  r  s       rO   r  r    sO    R25^cJ '&]3=]AEjA[TZA[]ELrQ   c           	      Z    t        ddddddd      }t        	 d
d	| it        |fi |}|S )Nr  r  r  r!  爅ZӼ@Fr,  r'  r.  rF   r>   rV   r/  r*  r  )vit_so400m_patch14_siglip_224r  r  s       rO   r  r    O    R2]bpuJ ''^4>^BFzB\U[B\^ELrQ   c           	      Z    t        ddddddd      }t        	 d
d	| it        |fi |}|S )Nr  r  r  r!  r  Fr,  r  r  )vit_so400m_patch14_siglip_378r  r  s       rO   r  r    sQ     R2]bpuJ ''^4>^BFzB\U[B\^ELrQ   c           	      Z    t        ddddddd      }t        	 d
d	| it        |fi |}|S )Nr  r  r  r!  r  Fr,  r  r  )vit_so400m_patch14_siglip_384r  r  s       rO   r  r    r  rQ   c           
      \    t        dddddddd      }t        	 d
d	| it        |fi |}|S )Nr!  r  r  r  Fr,  r  r'  r.  rF   r>   rV   r/  r*  r[   r  )vit_so400m_patch16_siglip_256r  r  s       rO   r  r    R    R2]bpuJ ''^4>^BFzB\U[B\^ELrQ   c           
      \    t        dddddddd      }t        	 d
d	| it        |fi |}|S )Nr!  r  r  r  Fr,  r  r  r  )vit_so400m_patch16_siglip_384r  r  s       rO   r  r  (  r  rQ   c           
      \    t        dddddddd      }t        	 d
d	| it        |fi |}|S )Nr!  r  r  r  Fr,  r  r  r  )vit_so400m_patch16_siglip_512r  r  s       rO   r  r  3  r  rQ   c           	      Z    t        ddddddd      }t        	 d	d| it        |fi |}|S )
Nr!  rx  r  Fr,  r  r  r  )vit_giantopt_patch16_siglip_256r  r  s       rO   r  r  >  O    R25^cJ ')`6@`DHD^W]D^`ELrQ   c           	      Z    t        ddddddd      }t        	 d	d| it        |fi |}|S )
Nr!  rx  r  Fr,  r  r  r  )vit_giantopt_patch16_siglip_384r  r  s       rO   r  r  I  r  rQ   c           
      \    t        dddddddd      }t        	 d	d| it        |fi |}|S )
Nr  r#  r$  Fr  r  r'  r.  rF   r>   r/  r*  r5  r[   r  )vit_base_patch32_siglip_gap_256r  r  s       rO   r  r  T  sR    B"%]blqJ ')`6@`DHD^W]D^`ELrQ   c           	      Z    t        ddddddd      }t        	 dd| it        |fi |}|S )	^ A SigLIP variant of ViT with global average pooling (GAP) instead of attention pooling (MAP).r!  r#  r$  Fr  r  r  )vit_base_patch16_siglip_gap_224r  r  s       rO   r  r  _  Q     B"%]blqJ ')`6@`DHD^W]D^`ELrQ   c           	      Z    t        ddddddd      }t        	 dd| it        |fi |}|S )	r  r!  r#  r$  Fr  r  r  )vit_base_patch16_siglip_gap_256r  r  s       rO   r  r  j  r  rQ   c           	      Z    t        ddddddd      }t        	 dd| it        |fi |}|S )	r  r!  r#  r$  Fr  r  r  )vit_base_patch16_siglip_gap_384r  r  s       rO   r  r  u  r  rQ   c           	      Z    t        ddddddd      }t        	 dd| it        |fi |}|S )	r  r!  r#  r$  Fr  r  r  )vit_base_patch16_siglip_gap_512r  r  s       rO   r  r    r  rQ   c           	      Z    t        ddddddd      }t        	 dd| it        |fi |}|S )	r  r!  r  r  Fr  r  r  ) vit_large_patch16_siglip_gap_256r  r  s       rO   r  r    Q     R25^cmrJ '*a7AaEI*E_X^E_aELrQ   c           	      Z    t        ddddddd      }t        	 dd| it        |fi |}|S )	r  r!  r  r  Fr  r  r  ) vit_large_patch16_siglip_gap_384r  r  s       rO   r  r    r  rQ   c           
      \    t        dddddddd      }t        	 d	d| it        |fi |}|S )
Nr!  r  r  Fr  r  r  r  ) vit_large_patch16_siglip_gap_512r  r  s       rO   r  r    sP    R255KJ '*a7AaEI*E_X^E_aELrQ   c           
      \    t        dddddddd      }t        	 d
d	| it        |fi |}|S )r  r  r  r  r!  r  Fr  r  r  )!vit_so400m_patch14_siglip_gap_224r  r  s       rO   r  r    R     R2ueJ '+b8BbFJ:F`Y_F`bELrQ   c           
      \    t        dddddddd      }t        	 d
d	| it        |fi |}|S )r  r  r  r  r!  r  Fr  r  r  )!vit_so400m_patch14_siglip_gap_378r  r  s       rO   r  r    r  rQ   c           
      \    t        dddddddd      }t        	 d
d	| it        |fi |}|S )r  r  r  r  r!  r  Fr  r  r  )!vit_so400m_patch14_siglip_gap_384r  r  s       rO   r  r    r  rQ   c           
      \    t        dddddddd      }t        	 d
d	| it        |fi |}|S )r  r  r  r  r!  r  Fr  r  r  )!vit_so400m_patch14_siglip_gap_448r  r  s       rO   r  r    r  rQ   c           
      \    t        dddddddd      }t        	 d
d	| it        |fi |}|S )r  r  r  r  r!  r  Fr  r  r  )!vit_so400m_patch14_siglip_gap_896r  r  s       rO   r  r    r  rQ   c                 ^    t        ddddddddd	      }t        	 d
d	| it        |fi |}|S )r  r!  r  r  r  Fr  r  	r'  r.  rF   r>   rV   r/  r*  r5  r[   r  )!vit_so400m_patch16_siglip_gap_256r  r  s       rO   r  r    sT     R2ue{J '+b8BbFJ:F`Y_F`bELrQ   c                 ^    t        ddddddddd	      }t        	 d
d	| it        |fi |}|S )Nr!  r  r  r  Fr  r  r  r  )!vit_so400m_patch16_siglip_gap_384r  r  s       rO   r  r    S    R2]b5KJ '+b8BbFJ:F`Y_F`bELrQ   c                 ^    t        ddddddddd	      }t        	 d
d	| it        |fi |}|S )Nr!  r  r  r  Fr  r  r  r  )!vit_so400m_patch16_siglip_gap_512r  r  s       rO   r  r    r  rQ   c           
      \    t        dddddddd      }t        	 d	d| it        |fi |}|S )
Nr!  rx  r  Fr  r  r  r  )#vit_giantopt_patch16_siglip_gap_256r  r  s       rO   r  r  
  P    R255KJ '-d:DdHLZHb[aHbdELrQ   c           
      \    t        dddddddd      }t        	 d	d| it        |fi |}|S )
Nr!  rx  r  Fr  r  r  r  )#vit_giantopt_patch16_siglip_gap_384r  r  s       rO   r  r    r  rQ   c                 `    t        ddddddddd	d

      }t        	 dd| it        |fi |}|S )Nr!  r  r  r  r      FTr.   r  
r'  r.  rF   r>   rY   rV   r/  r1  r2  r*  r  )vit_wee_patch16_reg1_gap_256r  r  s       rO   r  r     sU    B!YZ$1%J '&]3=]AEjA[TZA[]ELrQ   c                 b    t        ddddddddd	d
d      }t        	 dd| it        |fi |}|S )Nr!  r  r  r  r   r  FTr.   r  r;   r'  r.  rF   r>   rY   rV   r/  r1  r2  r*  r<   r  )vit_dwee_patch16_reg1_gap_256r  r  s       rO   r  r  +  sX    B!YZ$1%\bJ ''^4>^BFzB\U[B\^ELrQ   c                 j    t        dddddddddd	t        
      }t        	 dd| it        |fi |}|S )Nr!  r  r  r   r  FTr.   r  r'  r.  rF   r>   rY   rV   r/  r1  r2  r*  rG  r  )vit_pwee_patch16_reg1_gap_256)rZ  r   r  r  s       rO   r  r  6  sX    B!YZ$1%ZnJ ''^4>^BFzB\U[B\^ELrQ   c                 j    t        dddddddddd	t        
      }t        	 dd| it        |fi |}|S )Nr!  r  r  r   r  FTr.   r  r  r  )vit_dpwee_patch16_reg1_gap_256)rZ  r   r  r  s       rO   r  r  A  sX    B!YZ$1%ZrJ '(_5?_CG
C]V\C]_ELrQ   c                 `    t        ddddddddd	d

      }t        	 dd| it        |fi |}|S )Nr!  @  r  r  r   ffffff@FTr.   r  r  r  )vit_little_patch16_reg1_gap_256r  r  s       rO   r  r  L  U    B!Y\$1%J ')`6@`DHD^W]D^`ELrQ   c                 b    t        ddddddddd	d
d      }t        	 dd| it        |fi |}|S )Nr!  r  r  r  r   r  FTr.   r  r;   r  r  ) vit_dlittle_patch16_reg1_gap_256r  r  s       rO   r  r  W  sX    B!Y\$1%\bJ '*a7AaEI*E_X^E_aELrQ   c                 `    t        ddddddddd	d

      }t        	 dd| it        |fi |}|S )Nr!  r  r  r  r   r  FTr  r  r  r  )vit_little_patch16_reg4_gap_256r  r  s       rO   r  r  b  r  rQ   c                 ^    t        ddddddddd	
	      }t        	 dd| it        |fi |}|S )Nr!  r  r$  r  r   FTr.   r  	r'  r.  rF   r>   rY   r/  r1  r2  r*  r  )vit_medium_patch16_reg1_gap_256r  r  s       rO   r  r  m  R    B!$1%J ')`6@`DHD^W]D^`ELrQ   c                 ^    t        ddddddddd	
	      }t        	 dd| it        |fi |}|S )Nr!  r  r$  r  r   FTr  r  r  r  )vit_medium_patch16_reg4_gap_256r  r  s       rO   r  r  x  r   rQ   c                 ^    t        ddddddddd	
	      }t        	 dd| it        |fi |}|S )Nr!  r     r  r   FTr  r  r  r  ) vit_mediumd_patch16_reg4_gap_256r  r  s       rO   r  r    R    B!$1%J '*a7AaEI*E_X^E_aELrQ   c                 ^    t        ddddddddd	
	      }t        	 dd| it        |fi |}|S )Nr!  r  r  r  r   FTr  r  r  r  ) vit_mediumd_patch16_reg4_gap_384r  r  s       rO   r  r    r  rQ   c                 ^    t        ddddddddd	
	      }t        	 dd| it        |fi |}|S )Nr!  r  r$  r  r   FTr.   r  r  r  ) vit_betwixt_patch16_reg1_gap_256r  r  s       rO   r
  r
    R    B"$$1%J '*a7AaEI*E_X^E_aELrQ   c                 ^    t        ddddddddd	
	      }t        	 dd| it        |fi |}|S )Nr!  r  r$  r  r   FTr  r  r  r  ) vit_betwixt_patch16_reg4_gap_256r  r  s       rO   r  r    r  rQ   c                 ^    t        ddddddddd	
	      }t        	 dd| it        |fi |}|S )Nr!  r  r$  r  r   FTr  r  r  r  ) vit_betwixt_patch16_reg4_gap_384r  r  s       rO   r  r    r  rQ   c           
      \    t        dddddddd      }t        	 d
d	| it        |fi |}|S )Nr!  r#  r$  FTr  r  )r'  r.  rF   r>   r/  r1  r*  r2  r  )vit_base_patch16_reg4_gap_256r  r  s       rO   r  r    sP    B"%1J ''^4>^BFzB\U[B\^ELrQ   c           
      \    t        dddddddd	      }t        	 dd
| it        |fi |}|S )F SO150M (shape optimized, but diff than paper def, optimized for GPU) r!  r  rU  r  ~jt@Fr  r,  )r'  r.  rF   r>   rV   r/  r2  r*  r  )vit_so150m_patch16_reg4_map_256r  r  s       rO   r  r    sR     B"aUJ ')`6@`DHD^W]D^`ELrQ   c                 ^    t        ddddddddd		      }t        	 dd
| it        |fi |}|S )r  r!  r  rU  r  r  Fr  r  	r'  r.  rF   r>   rV   r/  r2  r*  r5  r  )vit_so150m_patch16_reg4_gap_256r  r  s       rO   r  r    T     B"aUEJ ')`6@`DHD^W]D^`ELrQ   c                 ^    t        ddddddddd		      }t        	 dd
| it        |fi |}|S )r  r!  r  rU  r  r  Fr  r  r  r  )vit_so150m_patch16_reg4_gap_384r  r  s       rO   r  r    r  rQ   c                 `    t        dddddddddd	

      }t        	 dd| it        |fi |}|S )I SO150M v2 (shape optimized, but diff than paper def, optimized for GPU) r!  @        NN@r   Fr.   r  
r'  r.  rF   r>   rV   rY   r?   r/  r2  r*  r  ) vit_so150m2_patch16_reg1_gap_256r  r  s       rO   r#  r#    W     B"[_EaUJ '*a7AaEI*E_X^E_aELrQ   c                 `    t        dddddddddd	

      }t        	 dd| it        |fi |}|S )r  r!  r  r  r   r!  r   Fr.   r  r"  r  ) vit_so150m2_patch16_reg1_gap_384r  r  s       rO   r&  r&    r$  rQ   c                 `    t        dddddddddd	

      }t        	 dd| it        |fi |}|S )r  r!  r  r  r   r!  r   Fr.   r  r"  r  ) vit_so150m2_patch16_reg1_gap_448r  r  s       rO   r(  r(    r$  rQ   c           	      Z    t        ddddddd      }t        	 d
d	| it        |fi |}|S )Nr  r  r  r!  r   FT)r'  r.  rF   r>   rY   r4  r  r  )vit_intern300m_patch14_448r  r  s       rO   r*  r*    sN    R2EDJ '$[1;[?CJ?YRX?Y[ELrQ   c                     t        dddddddddddt        t        d	
      t        t        d	
      t              }t	        	 dd| it        |fi |}|S ) ViT Large AIM-v2 model
    r  r  r  r  F      @r  silur   r  r'  r.  rF   r>   r/  r5  rV   r*  r?   rB   r[   rE   rF  r\   r  )aimv2_large_patch14_224rZ  r   r   r   r  r  s       rO   r0  r0    p     R1%Y^EEUV\7-UY@ZflJ
 '!X.8X<@<Vv<VXELrQ   c                     t        dddddddddddt        t        d	
      t        t        d	
      t              }t	        	 dd| it        |fi |}|S ) ViT Huge AIM-v2 model
    r  rx  r  r$  FAfU@r  r.  r   r  r/  r  )aimv2_huge_patch14_224r1  r  s       rO   r6  r6  &  sp    
 R25Z_eeuX^7-UY@ZflJ
 ' W-7W;?
;Uf;UWELrQ   c                     t        dddddddddddt        t        d	
      t        t        d	
      t              }t	        	 dd| it        |fi |}|S ) ViT 1B AIM-v2 model
    r     r  r!  Fr-  r  r.  r   r  r/  r  )aimv2_1b_patch14_224r1  r  s       rO   r:  r:  5  p     R25Z_EEUV\7-UY@ZflJ
 'U+5U9=j9SF9SUELrQ   c                     t        dddddddddddt        t        d	      t        t        d	      t        
      }t	        	 dd| it        |fi |}|S ) ViT 3B AIM-v2 model
    r     r  Fr5  r  r.  r   r  r/  r  )aimv2_3b_patch14_224r1  r  s       rO   r?  r?  C  p     R25Z_eeuX^7-UY@ZflJ
 'U+5U9=j9SF9SUELrQ   c                     t        dddddddddddt        t        d	
      t        t        d	
      t              }t	        	 dd| it        |fi |}|S )r,  r  r  r  r  Fr-  r  r.  r   r  r/  r  )aimv2_large_patch14_336r1  r  s       rO   rB  rB  Q  r2  rQ   c                     t        dddddddddddt        t        d	
      t        t        d	
      t              }t	        	 dd| it        |fi |}|S )r4  r  rx  r  r$  Fr5  r  r.  r   r  r/  r  )aimv2_huge_patch14_336r1  r  s       rO   rD  rD  _  p     R25Z_eeuX^7-UY@ZflJ
 ' W-7W;?
;Uf;UWELrQ   c                     t        dddddddddddt        t        d	
      t        t        d	
      t              }t	        	 dd| it        |fi |}|S )r8  r  r9  r  r!  Fr-  r  r.  r   r  r/  r  )aimv2_1b_patch14_336r1  r  s       rO   rG  rG  m  r;  rQ   c                     t        dddddddddddt        t        d	      t        t        d	      t        
      }t	        	 dd| it        |fi |}|S )r=  r  r>  r  Fr5  r  r.  r   r  r/  r  )aimv2_3b_patch14_336r1  r  s       rO   rI  rI  {  r@  rQ   c                     t        dddddddddddt        t        d	
      t        t        d	
      t              }t	        	 dd| it        |fi |}|S )r,  r  r  r  r  Fr-  r  r.  r   r  r/  r  )aimv2_large_patch14_448r1  r  s       rO   rK  rK    r2  rQ   c                     t        dddddddddddt        t        d	
      t        t        d	
      t              }t	        	 dd| it        |fi |}|S )r4  r  rx  r  r$  Fr5  r  r.  r   r  r/  r  )aimv2_huge_patch14_448r1  r  s       rO   rM  rM    rE  rQ   c                     t        dddddddddddt        t        d	
      t        t        d	
      t              }t	        	 dd| it        |fi |}|S )r8  r  r9  r  r!  Fr-  r  r.  r   r  r/  r  )aimv2_1b_patch14_448r1  r  s       rO   rO  rO    r;  rQ   c                     t        dddddddddddt        t        d	      t        t        d	      t        
      }t	        	 dd| it        |fi |}|S )r=  r  r>  r  Fr5  r  r.  r   r  r/  r  )aimv2_3b_patch14_448r1  r  s       rO   rQ  rQ    r@  rQ   c           	      V    t        dddddd      }t        d	d| it        |fi |}|S )
 ViT Test
    r!  @   r  r   r   T)r'  r.  rF   r>   rV   r  r  )test_vitr  r  s       rO   rU  rU    s=     raSTgklJ&gjgDQ[Lf_eLfgELrQ   c                 ^    t        ddddddddd	d

      }t        dd| it        |fi |}|S )rS  r!  rT  r  r   r   Fr.   r  r   T)
r'  r.  rF   r>   rV   r/  r2  r*  rY   r  r  )	test_vit2r  r  s       rO   rW  rW    sK     1QaU_ceJ 'hzhTR\Mg`fMghELrQ   c                 ^    t        ddddddddd	d

      }t        dd| it        |fi |}|S )rS  r!  `   	   r   r   Fr.   r,  Tr   )
r'  r.  rF   r>   rV   r/  r2  r*  r6  rY   r  )	test_vit3r  r  s       rO   r[  r[    sL     1QaUPTbfhJ 'hzhTR\Mg`fMghELrQ   c                 `    t        dddddddddd	d
      }t        dd| it        |fi |}|S )rS  r!  rY  rZ  r   Fr.   r  r   Trmsnorm)r'  r.  rF   r>   rV   r/  r2  r*  rY   r  rE   r  )	test_vit4r  r  s       rO   r^  r^    sM     1QaU_cJ
 'hzhTR\Mg`fMghELrQ   c                 |    t        dddddddddt        t        d      	
      }t        dd
| it        |fi |}|S )zu BEiT3 Base model (ViT-Base size) with patch size 16x16.
    Remapped to VisionTransformer with scale_norm=True.
    r!  r#  r$  r  Tr  r   r  
r'  r.  rF   r>   rV   rW   rX   r/  r*  rE   r  )beit3_base_patch16_224r  r  s       rO   ra  ra    sS    
 B"TtQV9$/J
 'uJuZ^_iZtmsZtuELrQ   c                 |    t        dddddddddt        t        d      	
      }t        dd
| it        |fi |}|S )zw BEiT3 Large model (ViT-Large size) with patch size 16x16.
    Remapped to VisionTransformer with scale_norm=True.
    r!  r  r  r  Tr  r   r  r`  r  )beit3_large_patch16_224r  r  s       rO   rc  rc    sS    
 R2TtQV9$/J
 'vZv[_`j[unt[uvELrQ   c                 |    t        dddddddddt        t        d	      

      }t        dd| it        |fi |}|S )zf BEiT3 Giant model with patch size 14x14.
    Remapped to VisionTransformer with scale_norm=True.
    r  r  r  r!  8mt@Tr  r   r  r`  r  )beit3_giant_patch14_224r  r  s       rO   rf  rf  	  sS    
 R2TtQV9$/J
 'vZv[_`j[unt[uvELrQ   c                 ~    t        ddddddddddt        t        d	
            }t        dd| it        |fi |}|S )z} BEiT3 Giant model with patch size 14x14 and image size 336x336.
    Remapped to VisionTransformer with scale_norm=True.
    r  r  r  r  r!  re  Tr  r   r  )r&  r'  r.  rF   r>   rV   rW   rX   r/  r*  rE   r  )beit3_giant_patch14_336r  r  s       rO   rh  rh    sV    
 t2W]TtQV9$/J
 'vZv[_`j[unt[uvELrQ   vit_tiny_patch16_224_in21kvit_small_patch32_224_in21kvit_small_patch16_224_in21kvit_base_patch32_224_in21kvit_base_patch16_224_in21kvit_base_patch8_224_in21kvit_large_patch32_224_in21kvit_large_patch16_224_in21kvit_huge_patch14_224_in21kvit_base_patch32_224_samzvit_base_patch32_224.samvit_base_patch16_224_samzvit_base_patch16_224.samvit_small_patch16_224_dinovit_small_patch8_224_dinovit_base_patch16_224_dinovit_base_patch8_224_dinovit_base_patch16_224_miil_in21k!vit_base_patch32_224_clip_laion2b)"vit_large_patch14_224_clip_laion2b!vit_huge_patch14_224_clip_laion2b"vit_giant_patch14_224_clip_laion2b)FFFTrU   rU   Nr   )r  r.   Fr  )r9   rU   T)rA  rU   T)r.   rg   ru  F)r9   F)zvisual.)Fru  Tr  )FNr  (  r   copyloggingr   oscollectionsr   	functoolsr   typingr   r   r   r   r	   r
   r   r   r   r   ImportErrortyping_extensionsr   torch.nnrk   torch.nn.functional
functionalr   	torch.jitr   	timm.datar   r   r   r   r   r   timm.layersr   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   _builderr/   	_featuresr0   _manipulater1   r2   r3   r4   	_registryr5   r6   r7   __all__	getLoggerr   _loggerrK   rp   r   r   r   rP   rS   r   r   r   r  r   rJ   r  r8   r  r  r  r  r  r  ro  r  rB  rL  rR  rZ  re  rr  r~  default_cfgsr;  rL   _quick_gelu_cfgsr  deepcopycr<  environlowerr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r  r  r  r  r   r"  r%  r'  r*  r,  r/  r2  r5  r9  r<  r>  rA  rC  rF  rH  rK  rN  rP  rS  rW  rZ  r]  r_  rb  rd  rf  rj  rm  ro  rq  rs  ru  rz  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r
  r  r  r  r  r  r  r#  r&  r(  r*  r0  r6  r:  r?  rB  rD  rG  rI  rK  rM  rO  rQ  rU  rW  r[  r^  ra  rc  rf  rh  )r  r  s   00rO   <module>r     sB  2    	 #  O O O*      
      4 + + R R Y Y
 '

H
% 	  04!!! ! 	!
 ! ! ! ! ! T"))_-! ! YY!HVBII VrH299 HV@299 @Fkryy k\V")) Vv !!"&+	<<   $	2O			 O	d"")) "3 "$ "Z^ "(  	"		"" " 	"
 
"@"")) "3 "$ "Z^ "."")) "3 "RV "bf "

Gs 
Gu 
GQU 
Gai 
G  "#"$&LL  c3h	
   \\2 WT* WTS WT# WTcg WTtx WT WTz  #ell*+# # # 
#u||
	#Lell*+  
#u||
.ell*+  
#u||
$9t 9~ #(&Qell*+Q Q  Q 	Q
 Q 
#u||
Qhc T#s(^ "S 1$3S
 02S 0 d2S 0 d]S2BS" 1$ e3#S* 1$ e]S3B+S2 1$ e33S: 1$ e]S3B;SB 0 f2CSJ 0 d]S2BKSR 0 e2SSZ 0 e]S2B[Sb / d1cSj 1$ e3kSr 1$ e]S3BsS~ .t}0SD .t} 300ESL /~ 310MSX ( d*YS` ( d]S*BaSh ' d)iSp ' d]S)BqSx ' d)yS@ ' d]S)BASJ &t|KSL %drlMSN &t|OSP )$2,QSV &t(WS^ &t(_Sf ')gSn ')oSv &t(wS@ ({e*-ASH )$|e+-ISP )$|e+-QSX (}e*-YS` (|e*-aSh '{e)-iSp )$|e+-qS| $THVZ&}SB $THVZ&CSL !$i"(<!#MMST  g"(<!"MUS\  e"(<!"M]Sd c"(<!!MeSp '\"(<! 3)0qS| &t\"(<! 3(0}SH '\"(<! 3)0IST '\"(<! 3)0USd ,Ta"(<! 3.0eSp +Da"(<! 3-0qS| ,Ta"(<! 3.0}SH ,Ta"(<! 3.0ISZ )$| 3	+0[Sd *4| 3	,0eSn +D| 3	-0oSx *4| 3	,0ySF &t L|e:ch(jGSN .t N|e:0WOSZ ' L)[S` *4 4U,DaSf 24 441gSl 24 484EmSr ,T 4.1sSv )$&wS| 6t?84}SB	 6t?S]8\C	SH	 6t?S]8\I	SN	 6t?T8CO	ST	 6t?(8DU	S\	 7$*@39P]	Sb	 7$*@(9Dc	Sj	 6t?S8Bk	Sp	 6t?(8Dq	Sz	 5d?74{	S@
 5d?-87EA
SH
 5d?T7CI
SN
 5d?-87EO
SV
 6t?S8BW
S\
 6t?(8D]
Sf
 0?24g
Sl
 0?S2Bm
Sr
 0?(2Ds
Sz
 1$$*@33P{
S@ 1$$*@(3DASH 0?S2BISN 0?(2DOSX /?14YS^ /?14_Sd /?(1DeSl 0?S2BmSt 1$?3GuSz 24$*@3\a4c{S@ 1$?Se3UASH 0?2GISN 1$?Se3UOSV (?*EWS\ (?Sc*S]Sb )$$*@3\_+acSh (?Sd*TiSn )$?Sd+ToSt ,T?Sd.TuS| .tW?	0E}SF .t?Sc0SGSN 3D? 3C	5AOSX /?Sc1SYSb +D?Sc-ScSh +D?C-AiSp +D?Sc-SqSv ,T?Sc.SwS~ &t?Sc(SSF ,T?Sc.SGSN 'W?Sc	)SOSX &tW?Sd	(TYSb &t?WD(BcSz 4TW?Sd	6T{SD 4T? 3(PT	6VESN 8?Sd:TOSV 8? 3(PT	:VWSb /W?Sc	1ScSl /W?Sc	1SmSv 0W?Sc	2SwS@ /W?Sd	1TASJ 4T?Sd6TKSR 3DW?Sd	5TSS\ .tW?Sc	0S]Sf .tW?Sc	0SgSp /W?Sc	1SqS| 'W?)E}SD 'W?)EESL (W?Sc*SMST (W?C	*AUS` 6t?S	8aSp *4B=[_+`qSr *4B=[_+`sSt +DRLuSv +DRLwSx *4B<yS@ 05? 3	20ASJ 05? 3(	2DKST *45? 3	,0US^ *45? 3(	,D_Sj !$Sae 4#1kSr  Ygk 4"1sSz  Ygk 4"1{SD  Sae 4"1ESL Ygk 4!1MST Ygk 4!1US\ !$[im 4U#D]Sd  Zhl 4U"DeSn !$Sae 4#1oSv  Ygk 4"1wS~  Ygk 4"1SH "4Wei 4U$DISP "4Wei 4U$DQSZ ("[S\ )$2,]S^ ("_Sb S"(<!	!McSl  T"(<!	"MmSv S"(<!	!MwSB *4M"(<!	,MCSL +DN"(<!	-MMSV *4S 3"(<!,MWSb ,TN"(<!	.McSn +D -oSv +D-wS| (*}SB +D -CSJ ( *KSR -d /SSZ +D -[Sb ( *cSj +D -kSr ( *sSz ,T .{SB )$ +CSJ ,T .KSR )$ +SSZ ,T .[Sb -d/cSh *4,iSn -d /oSv *4 ,wS~ *4 ,SF -d /GSN / 1OSV -d /WS^ -d /_Sf / 1gSn / 1oSx / 1yS@ /1ASF ,T.GSL / 1MST ,T .US\ 1$ 3]Sd / 1eSl ,T .mSt / 1uS| ,T .}SD 0 2ESL -d /MST 0 2US\ -d /]Sd 0 2eSl 1$3mSr .t0sSx 1$3yS~ 02SD 4T6ESJ 5d7KSZ 1$ 3[Sb .t 30cSj .t 30kSr 1$ 33sSz 0 32{SB 9$ 3;CSJ 4T 36KSR 4T 36SSZ 5d 37[Sn 7 39oSv 8 3:wS~ 0 32SF 9$ 3;GSN 4T 36OSV 4T 36WS^ 5d 37_Sr 1$ 3sSz 3D 5{SB 1$ 3CSJ 1$ 3KSR 3D 5SSZ 3D 5[Sd 24 3(4eSl 6t 3(8mSv 3D?5EwS~ 5d?7ESF 3D?5EGSN 6t?8EOSX ,T 4.1YS^ 6t 481_Sd -d 4/1eSj -d 4/1kSp 7 491qSv .t 401wS| 8 4:1}SB  0 421C SJ  / 411K SP  9$ 4;1Q SV  / 411W S\  8 4:1] Sb  / 411c Sh  0 421i Sp  ? 4A1q Sv  9$ 4;1w S|  7 491} SD! 1$ 431E!SL! ? 3A0M!SR! 0 421S!SX! ? 4A1Y!S^! 9$ 4;1_!Sd! 0 421e!Sj! 7 491k!Sr! 1$ 431s!Sz! ? 3A0{!S@" .t 0"A"SF" =d 4?1G"SL" 5d 471M"ST" =d 3?0U"SZ" 0 2"["S^" >t 3@0_"Sd" 6t 380e"Sl" >t 3@0m"Sr" >t 3(@Ds"Sz" +D"(< 3A	-{"SF# +D"(< 3A-G#SR# '?L!)%S#SZ# ,T?L!.%[#Sb# &t?L!(%c#Sj# $T?L!&%k#Sr# $T?L!&%s#Sz# '?L 3A)?{#SB$ ,T?L 3A.?C$SJ$ &t?L 3A(?K$SR$ $T?L 3A&?S$SZ$ $T?L 3A&?[$Sb$ '?L 3A)?c$Sj$ &t?L 3A(?k$Sr$ $T?L 3A&?s$Sz$ $T?L 3A&?{$SD% $ 41E%SJ% 4 41K%SP% 4 41Q%SV% 4 41W%S^% +D"(<s-L_%Sd% 4T"(<s6Le%Sj% ,T"(<s.Lk%Sp% 5d"(<s7Lq%Sv% (*0Ds*Tw%Sz% (=/DJ^il*n{%S~%  "(<s"%SH& )$"(<s+I&SR& !$"(<s#S&S\& *4"(<s,]&Sj& #/"4"4"6n$!Q!%%:LQ\`abi`jkl`mQmAn 	 >Al1o&A~  1+<=L8%789> %\2 jjnn%97CIIKvU  !%)### TN#
 k)*#L T @Q   T @Q   d AR   d AR   d AR   d AR   T @Q   T @Q   T @Q   T @Q   T @Q   D ?P   d AR   d AR   d AR   d AR   d AR   T @Q   d AR    DU   $ EV   4 FW   4 FW   4 FW   D GX    DU    DU    DU   $ EV   D GX   D GX   D GX   T HY   $ EV   $ EV   $ EV   $ EV   $ EV   $ EV   t J[   4 FW   4 FW   $ EV   $ EV   $ EV   
4 
FW 
 
 
d 
IZ 
 
 
d 
IZ 
 
 	D 	O` 	 	 	D 	O` 	 	 	T 	Pa 	 	 	T 	Pa 	 	 	D 	O` 	 	 	D 	O` 	 	 	 	Sd 	 	 $ EV   $ EV    DU   4 FW   	4 	FW 	 	 $ EV   d AR   d AR   t :K   d 9J   t :K   	 	CT 	 	 	 	DU 	 	 	 	CT 	 	  DU    CT    DU    DU    	d 	IZ 	 	 	T 	HY 	 	 	d 	IZ 	 	 d IZ    	T 	HY 	 	 	d 	IZ 	 	 	t 	J[ 	 	 d IZ   D GX   D GX   D GX   D GX   D GX   T HY   T HY   T HY   d IZ   d IZ   d IZ   d IZ   d IZ   d IZ    K\    K\    K\    K\    K\    K\    K\    L]    L]    L]   $ M^   $ M^   $ M^   $ M^   $ M^   $ M^   $ M^   $ M^   D O`   D O`   T HY   d IZ   d IZ   t J[    K\    L]    K\    K\    K\    L]    L]    L]    L]    L]   d IZ    K\    K\    K\    L]    L]    L]   4 FW   
 
CT 
 
 t BS   
T 
@Q 
 
 
T 
@Q 
 
 
 
CT 
 
 
t 
BS 
 
 
T 
@Q 
 
 
T 
@Q 
 
 
 
CT 
 
 
t 
BS 
 
 
T 
@Q 
 
 
T 
@Q 
 
  4E   $ 5F   $ 5F   	$ 	5F 	 	 
t 
BS 
 
 
 
CT 
 
 
 
CT 
 
 
 
CT 
 
 H ' "E'!#G' "#G' !"E	'
 !"E'  !C' "#E' "#G' !"C'  :'  :' !">'  !<'  !<'  :'  &'H!'" ()L#'$ +O)L*N)' EX  *))*T^ os.   BD UBDU;BDVBDDBDDBD