
    ^j                    J   d Z ddlZddlZddlmZ ddlmZmZmZm	Z	m
Z
mZmZmZ ddlZddlmZ ddlmc mZ ddlmZmZmZmZ ddlmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z. ddl/m0Z0 dd	l1m2Z2 dd
l3m4Z4 ddl5m6Z6m7Z7 dgZ8 G d dejr                        Z: G d dejr                        Z; G d dejr                        Z< G d dejr                        Z=	 ddee>ej~                  f   dejr                  de>dee>ej~                  f   fdZ@	 	 ddee>ej~                  f   dejr                  de>deAdee>ej~                  f   f
dZBdde>deAde=fdZCdde>dee>ef   fd ZDdde>dee>ef   fd!ZEdde>dee>ef   fd"ZFdde>dee>ef   fd#ZG e6i d$ eDd%&      d' eDd%d(d)d*+      d, eDd%eed(d)d*-      d. eDd%eed/d)d*-      d0 eDd%d1d)d*+      d2 eDd%d1d)d*+      d3 eDd%d1d)d*+      d4 eDd%d(d)5      d6 eDd%d(d)5      d7 eDd%d1d)5      d8 eDd%d1d)5      d9 eDd%d1d)5      d: eDd%d1d)d*d;<      d= eDd%d1d)d*d;<      d> eDd%d1d)d*d;<      d? eDd%d@      dA eDd%d@      i dB eDd%d@      dC eDd%d@      dD eDd%d@      dE eDd%dF@      dG eDd%dF@      dH eDd%dI@      dJ eDd%dK@      dL eDd%d(d)dKM      dN eDd%dF@      dO eDd%dF@      dP eDdQ      dR eDd%dSdTdUdUV      dW eDd%dSdTdUdUV      dX eDd%dSdT5      dY eDd%dSdTdUdUV      dZ eEd%d[dI\      d] eEd%d[dI\      i d^ eEd%d_dF\      d` eEd%d(dF\      da eEd%d1db\      dc eEd%d1d\      dd eEd%d1d\      de eEd%d1d\      df eEd%d1d\      dg eEd%dhd\      di eEd%dhd\      dj eEd%dhd\      dk eEd%d1d\      dl eEd%d1d\      dm eDd%eedno      dp eDd%eedno      dq eDd%eedno      dr eDd%eedno      ds eDd%eedno      i dt eDd%eedno      du eDd%eedno      dv eDd%eedno      dw eDd%eedno      dx eDd%eedno      dy eDd%eedno      dz eDd%eedno      d{ eFd%&      d| eFd%&      d} eFd%&      d~ eFd%&      d eFd%&      d eFd%&      d eGd%&      d eGd%&      d eGd%&      d eFd%&       eFd%&       eFd%dd       eFd%dd       eFd%&       eFd%&       eFd%&       eFd%dd      d      ZHe7ddeAde=fd       ZIe7ddeAde=fd       ZJe7ddeAde=fd       ZKe7ddeAde=fd       ZLe7ddeAde=fd       ZMe7ddeAde=fd       ZNe7ddeAde=fd       ZOe7ddeAde=fd       ZPe7ddeAde=fd       ZQe7ddeAde=fd       ZRe7ddeAde=fd       ZSe7ddeAde=fd       ZTe7ddeAde=fd       ZUe7ddeAde=fd       ZVe7ddeAde=fd       ZWe7ddeAde=fd       ZXe7ddeAde=fd       ZYe7ddeAde=fd       ZZe7ddeAde=fd       Z[e7ddeAde=fd       Z\e7ddeAde=fd       Z]e7ddeAde=fd       Z^e7ddeAde=fd       Z_e7ddeAde=fd       Z`e7ddeAde=fd       Zae7ddeAde=fd       Zbe7ddeAde=fd       Zce7ddeAde=fd       Zde7ddeAde=fd       Zee7ddeAde=fd       Zfe7ddeAde=fd       Zge7ddeAde=fd       Zhe7ddeAde=fd       Zie7ddeAde=fd       Zje7ddeAde=fd       Zke7ddeAde=fd       Zle7ddeAde=fd       Zme7ddeAde=fd       Zne7ddeAde=fd       Zoe7ddeAde=fd       Zpe7ddeAde=fd       Zqe7ddeAde=fd       Zre7ddeAde=fd       Zse7ddeAde=fd       Zte7ddeAde=fd       Zue7ddeAde=fd       Zve7ddeAde=fd       Zwe7ddeAde=fd       Zxe7ddeAde=fd       Zye7ddeAde=fd       Zze7ddeAde=fd       Z{e7ddeAde=fd       Z|e7ddeAde=fd       Z}e7ddeAde=fd       Z~e7ddeAde=fd       Ze7ddeAde=fd       Zy)aK   EVA

EVA ViT from https://github.com/baaivision/EVA , paper: https://arxiv.org/abs/2211.07636

This file contains a number of ViT variants the utilise ROPE position embeddings, SwiGLU and other additions:
 * EVA & EVA02 model implementations that evolved from BEiT, additional models in vision_transformer.py.
 * `timm` original SBB ViT w/ ROPE position embeddings
 * Perception Encoder (PE) ViT from Meta (https://arxiv.org/abs/2504.13181)
 * ROPE-ViT from Naver AI (https://arxiv.org/abs/2403.13298)
 * DINOv3 from META AI Research (https://arxiv.org/abs/2508.10104)

@article{EVA,
  title={EVA: Exploring the Limits of Masked Visual Representation Learning at Scale},
  author={Fang, Yuxin and Wang, Wen and Xie, Binhui and Sun, Quan and Wu, Ledell and Wang, Xinggang and Huang,
  Tiejun and Wang, Xinlong and Cao, Yue},
  journal={arXiv preprint arXiv:2211.07636},
  year={2022}
}

EVA-02: A Visual Representation for Neon Genesis - https://arxiv.org/abs/2303.11331
@article{EVA02,
  title={EVA-02: A Visual Representation for Neon Genesis},
  author={Fang, Yuxin and Sun, Quan and Wang, Xinggang and Huang, Tiejun and Wang, Xinlong and Cao, Yue},
  journal={arXiv preprint arXiv:2303.11331},
  year={2023}
}

@article{bolya2025perception,
  title={Perception encoder: The best visual embeddings are not at the output of the network},
  author={Bolya, Daniel and Huang, Po-Yao and Sun, Peize and Cho, Jang Hyun and Madotto, Andrea and Wei, Chen and Ma,
    Tengyu and Zhi, Jiale and Rajasegaran, Jathushan and Rasheed, Hanoona and others},
  journal={arXiv preprint arXiv:2504.13181},
  year={2025}
}

@inproceedings{heo2024rotary,
  title={Rotary position embedding for vision transformer},
  author={Heo, Byeongho and Park, Song and Han, Dongyoon and Yun, Sangdoo},
  booktitle={European Conference on Computer Vision},
  pages={289--305},
  year={2024},
  organization={Springer}
}

@article{simeoni2025dinov3,
  title={{DINOv3}},
  author={Sim{'e}oni, Oriane and Vo, Huy V. and Seitzer, Maximilian and Baldassarre, Federico and Oquab, Maxime
    and Jose, Cijo and Khalidov, Vasil and Szafraniec, Marc and Yi, Seungeun and Ramamonjisoa, Micha{"e}l
    and Massa, Francisco and Haziza, Daniel and Wehrstedt, Luca and Wang, Jianyuan and Darcet, Timoth{'e}e
    and Moutakanni, Th{'e}o and Sentana, Leonel and Roberts, Claire and Vedaldi, Andrea and Tolan, Jamie
    and Brandt, John and Couprie, Camille and Mairal, Julien and J{'e}gou, Herv{'e} and Labatut, Patrick
    and Bojanowski, Piotr},
  year={2025},
  eprint={2508.10104},
  url={https://arxiv.org/abs/2508.10104},
}

DINOv3 code was a modification of existing EVA model and support modules, so licensed under Apache-2.0 like timm.
Weights from META remain under DINOv3 License (https://ai.meta.com/resources/models-and-libraries/dinov3-license/).

Modifications by / Copyright 2023 Ross Wightman, original copyrights below
    N)partial)AnyCallableDictListOptionalSetTupleUnion)IMAGENET_DEFAULT_MEANIMAGENET_DEFAULT_STDOPENAI_CLIP_MEANOPENAI_CLIP_STD)
PatchEmbedMlpGluMlpSwiGLU	LayerNormDropPathcalculate_drop_path_ratesPatchDropoutWithIndicescreate_rope_embedapply_rot_embed_catapply_keep_indices_nlctrunc_normal_resample_patch_embedresample_abs_pos_embedglobal_pool_nlc	to_2tupleuse_fused_attnmaybe_add_maskresolve_self_attn_maskAttentionRopeAttentionPoolLatent   )build_model_with_cfg)feature_take_indices)
checkpoint)generate_default_cfgsregister_modelEvac                   *    e Zd ZU dZej
                  j                  e   ed<   	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dde	de	dededede	d	e
d
e
dee	   dee   dedededef fdZddZddZ	 	 	 ddeej                      deej                      defdZddZ xZS )EvaAttentionzG EVA Attention with ROPE, no k-bias, and fused/unfused qkv options
    
fused_attndim	num_headsqkv_bias	qkv_fusedqkv_bias_separatenum_prefix_tokens	attn_drop	proj_dropattn_head_dim
norm_layerqk_norm
scale_normrotate_halfgatedc                    ||d}t         |           |s|r	|
J d       || _        ||z  | _        |	|	| _        | j                  | j                  z  }| j                  dz  | _        || _        t               | _        || _        || _	        |rt        j                  ||dz  fddi|| _        dx| _        x| _        | _        |rt        j                   t#        j$                  |fi |      | _        | j)                  dt#        j$                  |fi |d	       t        j                   t#        j$                  |fi |      | _        ndx| _        x| _        | _        nvt        j                  ||fd|i|| _        t        j                  ||fddi|| _        t        j                  ||fd|i|| _        d| _        dx| _        x| _        | _        |r |
| j                  fi |nt        j.                         | _        |r |
| j                  fi |nt        j.                         | _        t        j4                  |      | _        |r	 |
|fi |nt        j.                         | _        |rt        j                  ||fd|i|nd| _        t        j                  ||fi || _        t        j4                  |      | _        | jA                          y)
au  
        Args:
            dim: Input dimension of the token embeddings
            num_heads: Number of attention heads
            qkv_bias: Whether to add a bias term to the query, key, and value projections
            qkv_fused: Whether qkv projections are fused into one projection or separate
            qkv_bias_separate: Whether to apply bias to qkv as a separate addition or part of F.linear() call
            num_prefix_tokens: Number of reg/cls tokens at the beginning of the sequence that
                should not have position embeddings applied
            attn_drop: Dropout rate for attention weights
            proj_drop: Dropout rate for the output projection
            attn_head_dim: Dimension of each attention head (if None, computed as dim // num_heads)
            norm_layer: Normalization layer constructor to use for QK and scale normalization
            qk_norm: Enable normalization of query (Q) and key (K) vectors with norm_layer
            scale_norm: Enable normalization (scaling) of attention output with norm_layer
            rotate_half: Use half rotation layout instead of interleaved
        devicedtypeNz<norm_layer must be provided if qk_norm or scale_norm is Trueg         biasFk_bias)
persistent)!super__init__r0   head_dimscaler4   r    r.   r3   r;   nnLinearqkvq_projk_projv_proj	Parametertorchemptyq_biasregister_bufferv_biasrC   Identityq_normk_normDropoutr5   normgateprojr6   reset_parameters)selfr/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r?   r@   ddattn_dim	__class__s                      Z/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/eva.pyrF   zEvaAttention.__init__n   su   H /)i+ii)"y($)DM==4>>1]]d*
!2(*!2&yyhlEE"EDH6::DK:$+ ll5;;x+F2+FG$$Xu{{8/Jr/JW\$] ll5;;x+F2+FG:>>>dkDK))CGGBGDK))CDDDDK))CGGBGDKDH6::DK:$+9@j5"5bkkm9@j5"5bkkmI.2<Jx.2."++-	EJBIIc8A(AbAPT	IIh2r2	I. 	    c                     | j                   Rt        j                  j                  | j                          t        j                  j                  | j                         | j                          y)z"Initialize parameters and buffers.N)rR   rI   initzeros_rT   _init_buffersr]   s    ra   r\   zEvaAttention.reset_parameters   s@    ;;"GGNN4;;'GGNN4;;'rb   c                 R    | j                   | j                   j                          yy)z.Compute and fill non-persistent buffer values.N)rC   zero_rg   s    ra   rf   zEvaAttention._init_buffers   s!    ;;"KK #rb   rope	attn_mask	is_causalc                    |j                   \  }}}| j                  | j                  |      j                         nd}| j                  | j                  | j                  |      }	nt        j                  | j                  | j                  | j                  f      }
| j                  r| j                  |      }	|	|
z  }	n,t        j                  || j                  j                  |
      }	|	j                  ||d| j                  d      j                  ddddd      }	|	j!                  d      \  }}}n| j#                  |      j                  ||| j                  d      j%                  dd      }| j'                  |      j                  ||| j                  d      j%                  dd      }| j)                  |      j                  ||| j                  d      j%                  dd      }| j+                  |      | j-                  |      }}|| j.                  }t1        | d	d
      }t        j                  |ddddd|ddf   t3        |dddd|dddf   ||      gd      j5                  |      }t        j                  |ddddd|ddf   t3        |dddd|dddf   ||      gd      j5                  |      }| j6                  r>t        j8                  ||||| j:                  r| j<                  j>                  nd|      }ng|| j@                  z  }||j%                  dd      z  }tC        ||||      }tE        ||      }|jG                  d      }| j=                  |      }||z  }|j%                  dd      j                  |||      }| jI                  |      }|||z  }| jK                  |      }| jM                  |      }|S )a  Forward pass for the attention module.

        Args:
            x: Input tensor of shape (batch_size, sequence_length, embedding_dim)
            rope: Rotary position embeddings tensor for position-aware attention
            attn_mask: Optional attention mask to apply during attention computation
            is_causal: If True, use causal (autoregressive) masking

        Returns:
            Tensor of shape (batch_size, sequence_length, embedding_dim)
        N)weightrB   rA      r   r%      r;   F)halfr/           )rk   	dropout_prl   )rl   )'shaperZ   sigmoidrK   rR   rP   catrC   rT   r3   Flinearrn   reshaper0   permuteunbindrL   	transposerM   rN   rV   rW   r4   getattrr   type_asr.   scaled_dot_product_attentiontrainingr5   prH   r"   r!   softmaxrY   r[   r6   )r]   xrj   rk   rl   BNCrZ   rK   r1   qkvnptrr   attn	attn_biass                     ra   forwardzEvaAttention.forward   sK   $ ''1a)-)>tyy|##%D88{{"hhqk 99dkk4;;%LM))((1+C8OC((1TXX__8LC++aAt~~r:BB1aAqQCjjmGAq!A&&q!T^^R@JJ1aPAA&&q!T^^R@JJ1aPAA&&q!T^^R@JJ1aPA{{1~t{{1~1((C46D		1Q4C4]+-@1aq=AQSW^b-cdjklttuvwA		1Q4C4]+-@1aq=AQSW^b-cdjklttuvwA??..1a#.2mm$..**#	A DJJAq{{2r**D.q$	YWI!$	2D<<B<'D>>$'DqAKK1%%aA.IIaLDAIIaLNN1rb   c                 $    | j                          y)z"Initialize non-persistent buffers.N)rf   rg   s    ra   init_non_persistent_buffersz(EvaAttention.init_non_persistent_buffers  s    rb   )   TTFr%   rt   rt   NNFTFFNNreturnNNNF)__name__
__module____qualname____doc__rP   jitFinalbool__annotations__intfloatr   r   rF   r\   rf   Tensorr   r   __classcell__r`   s   @ra   r-   r-   i   sF   		%%
 !"&+%&!!+/-1!# %#K K  K  	K 
 K   $K   #K  K  K  $C=K  !*K  K  K  K  K Z  ,004#D 5<<(D  -	D
 DLrb   r-   c            *       B    e Zd Zdddddddddddddddej                  edddfd	ed
ededededededededede	dededededede
e   dedede
e   f( fdZd%dZ	 	 	 d&d ej                  d!e
ej                     d"e
ej                     d#edej                  f
d$Z xZS )'EvaBlockT      @Fr   r%   evart   Nr/   r0   r1   r2   	mlp_ratio
swiglu_mlpswiglu_align_to	scale_mlpscale_attn_innerr4   	attn_typer;   
gated_attnr6   r5   	drop_pathinit_values	act_layerr8   r7   c                 @   ||d}t         |            ||fi || _        |dk(  rt        nt        } ||f||||
|||||	||d|| _        || _        |)t        j                  t        j                  |fi |      nd| _        |dkD  rt        |      nt        j                         | _         ||fi || _        t!        ||z        }|rL|s|rt#        d|||r|nd||d|| _        nGt'        d||dz  |r|ndt        j(                  d|d	|| _        nt+        d||||r|nd|d
|| _        |)t        j                  t        j                  |fi |      nd| _        |dkD  rt        |      nt        j                         | _        | j1                          y)a   Initialize the EVA transformer block.

        Args:
          dim: Input dimension of the token embeddings
            num_heads: Number of attention heads
            qkv_bias: Whether to use bias terms in query, key, value projections
            qkv_fused: Whether to use a single projection for query, key, value
            mlp_ratio: Ratio of MLP hidden dimension to input dimension
            swiglu_mlp: Whether to use SwiGLU activation in the MLP
            scale_mlp: Whether to use normalization in the MLP
            scale_attn_inner: Whether to use normalization within the attention mechanism
            num_prefix_tokens: Number of tokens at the beginning of the sequence (class tokens, etc.)
            attn_type: Type of attention module to use ('eva' or 'rope')
            proj_drop: Dropout rate for projection layers
            attn_drop: Dropout rate for attention matrix
            drop_path: Stochastic depth rate
            init_values: Initial value for LayerScale, None = no LayerScale
            act_layer: Activation layer constructor
            norm_layer: Normalization layer constructor
            attn_head_dim: Dimension of each attention head (if None, computed as dim // num_heads)
        r>   rj   r0   r1   r2   r4   r5   r6   r7   r8   r:   r;   r<   Nrt   in_featureshidden_featuresr8   dropalign_torp   Fr   r   r8   r   	gate_lastr   r   r   r   r8   r    )rE   rF   norm1r#   r-   r   r   rI   rO   rP   rQ   gamma_1r   rU   
drop_path1norm2r   r   mlpr   SiLUr   gamma_2
drop_path2r\   )r]   r/   r0   r1   r2   r   r   r   r   r   r4   r   r;   r   r6   r5   r   r   r   r8   r7   r?   r@   kwargsr^   attn_clsr   r`   s                              ra   rF   zEvaBlock.__init__  s   ^ /*r*
$-$7=\
/'!'#
 
	 '?J?Vr||EKK$:r$:;\`1:R(9-R[[]*r*
cIo.O!  #$3-6zD",  "  #$3a$7-6zD gg#"    /#)2: DH @K?Vr||EKK$:r$:;\`1:R(9-R[[] 	rb   r   c                     | j                   it        j                  j                  | j                   | j                         t        j                  j                  | j
                  | j                         yy)zInitialize parameters.N)r   rI   rd   	constant_r   r   rg   s    ra   r\   zEvaBlock.reset_parameters  sM    <<#GGdllD,<,<=GGdllD,<,<= $rb   r   rj   rk   rl   c           
         | j                   j|| j                  | j                  | j                  |      |||            z   }|| j	                  | j                  | j                  |                  z   }|S || j                  | j                   | j                  | j                  |      |||      z        z   }|| j	                  | j                  | j                  | j                  |            z        z   }|S Nrj   rk   rl   )r   r   r   r   r   r   r   r   r]   r   rj   rk   rl   s        ra   r   zEvaBlock.forward  s     <<DOODIIdjjm$R[gpI$qrrADOODHHTZZ]$;<<A  DOODLL499TZZ]QUajv9  4A  %A  B  BADOODLL488DJJqM3J$JKKArb   r   r   )r   r   r   rI   GELUr   r   r   r   strr   r   rF   r\   rP   r   r   r   r   s   @ra   r   r     s    ""!$#$#%*%&" %$!!!+/"$''#,+//l l  l  	l 
 l  l  l  !l  l  #l   #l  l  l  l  l   !l " #l $ "%%l &  'l ( !)l * $C=+l \> ,004#|| 5<<(  -	
  
rb   r   c            *       R    e Zd ZdZddddddddddddddd	ej
                  ej                  d	d	d	fd
edededede	de
dededededededede	de	de	dee	   dededee   f( fdZ	 	 	 d%dej                  d eej                     d!eej                     d"ed#ej                  f
d$Z xZS )&EvaBlockPostNormzF EVA block w/ post-norm and support for swiglu, MLP norm scale, ROPE. Tr   r   Fr   r%   rt   Nr/   r0   r1   r2   r   r   r;   r   r   r   r   r   r4   r6   r5   r   r   r   r8   r7   c                 R   ||d}t         |           |dk(  rt        nt        } ||f|||||||||||d|| _         ||fi || _        |dkD  rt        |      nt        j                         | _	        t        ||z        }|	rJ|rt        d|||r|nd||
d|| _        nGt        d||dz  |r|ndt        j                  d|d	|| _        nt        d||||r|nd|d
|| _         ||fi || _        |dkD  rt        |      | _        yt        j                         | _        y)a   Initialize the post-norm EVA transformer block.

        Args:
          dim: Input dimension of the token embeddings
            num_heads: Number of attention heads
            qkv_bias: Whether to use bias terms in query, key, value projections
            qkv_fused: Whether to use a single projection for query, key, value
            mlp_ratio: Ratio of MLP hidden dimension to input dimension
            swiglu_mlp: Whether to use SwiGLU activation in the MLP
            scale_mlp: Whether to use normalization in the MLP
            scale_attn_inner: Whether to use normalization within the attention mechanism
            num_prefix_tokens: Number of tokens at the beginning of the sequence (class tokens, etc.)
            attn_type: Type of attention module to use ('eva' or 'rope')
            proj_drop: Dropout rate for projection layers
            attn_drop: Dropout rate for attention matrix
            drop_path: Stochastic depth rate
            init_values: Initial value for LayerScale, None = no LayerScale (NOTE: ignored for post-norm block)
            act_layer: Activation layer constructor
            norm_layer: Normalization layer constructor
            attn_head_dim: Dimension of each attention head (if None, computed as dim // num_heads)
        r>   rj   r   rt   Nr   rp   Fr   r   r   )rE   rF   r#   r-   r   r   r   rI   rU   r   r   r   r   r   r   r   r   r   )r]   r/   r0   r1   r2   r   r   r;   r   r   r   r   r   r4   r6   r5   r   r   r   r8   r7   r?   r@   r   r^   r   r   r`   s                              ra   rF   zEvaBlockPostNorm.__init__  st   ^ /$-$7=\
/'!'#
 
	  *r*
1:R(9-R[[]cIo.!  #$3-6zD",  "  #$3a$7-6zD gg#"    /#)2: DH  *r*
1:R(9-R[[]rb   r   rj   rk   rl   r   c                     || j                  | j                  | j                  ||||                  z   }|| j                  | j	                  | j                  |                  z   }|S r   )r   r   r   r   r   r   r   s        ra   r   zEvaBlockPostNorm.forward  s[     

499QTYbk9+l mnn

488A; 788rb   r   )r   r   r   r   rI   r   r   r   r   r   r   r   r   rF   rP   r   r   r   r   s   @ra   r   r     s   P
 ""!" %$$#$#%*%&!!!+/"$''#%<<+//fSfS fS 	fS
 fS fS fS fS fS fS !fS fS #fS  #fS fS  !fS" #fS$ "%%fS&  'fS( !)fS* $C=+fSV ,004#	||	 5<<(	  -		
 	 
	rb   r   c            [           e Zd ZdZddddddddd	d	d
dddddddddddedd	ddd	ddddddddddddddddddf.deeeeef   f   deeeeef   f   dedededededede	de	de
de	d ed!e	d"e	d#ed$e
d%e
d&e
d'e
d(e
d)e
d*ed+ee
   d,e	d-ed.e	d/e	d0e	d1ee   d2e
d3ed4e
d5e	d6e	d7e	d8ee	   d9ee	   d:ee   d;ee
   d<e	d=e	d>eeeeef   ef      d?e
fX fd@ZdcdAe	fdBZdddDZdcdEej"                  dAe	dCdfdFZej(                  j*                  dCee   fdG       Zej(                  j*                  dcdHe	dCdfdI       Zej(                  j*                  dedJe	dCeeef   fdK       Zej(                  j*                  dCej"                  fdL       Zdfdedee   dCdfdMZ	 	 dgdeeeef      deeeef      dCdfdNZdCeej>                  eej>                     f   fdOZ 	 	 	 	 	 	 	 	 dhdPej>                  dQeeee!e   f      dRe	dSe	dTe	dUedVe	dWeej>                     dXe	dCee!ej>                     eej>                  e!ej>                     f   f   fdYZ"	 	 	 didQeee!e   f   dZe	d[e	fd\Z#dfdPej>                  d]ee   dCej>                  fd^Z$	 	 djdPej>                  dWeej>                     dXe	dCej>                  fd_Z%dedPej>                  d`e	dCej>                  fdaZ&	 	 djdPej>                  dWeej>                     dXe	dCej>                  fdbZ' xZ(S )kr+   a!   Eva Vision Transformer w/ Abs & Rotary Pos Embed

    This class implements the EVA and EVA02 models that were based on the BEiT ViT variant
      * EVA - abs pos embed, global avg pool
      * EVA02 - abs + rope pos embed, global avg pool, SwiGLU, scale Norm in MLP (ala normformer)
          rA     avg      Tr   Fr   r   rt   Nry   ijg     @gMbP?img_size
patch_sizein_chansnum_classesglobal_pool	embed_dimdepthr0   r1   r2   r   r   r   r   r   r   	drop_ratepos_drop_ratepatch_drop_rateproj_drop_rateattn_drop_ratedrop_path_rater8   r   class_tokennum_reg_tokensno_embed_classuse_abs_pos_embuse_rot_pos_emb	rope_typerope_grid_offsetrope_grid_indexingrope_temperaturerope_rotate_halfuse_post_normuse_pre_transformer_normuse_post_transformer_normuse_fc_normattn_pool_num_headsattn_pool_mlp_ratiodynamic_img_sizedynamic_img_padref_feat_shapehead_init_scalec/                 	   t         ;|           |-|.d}/|dv sJ || _        || _        || _        |x| _        x| _        | _        |rdnd|z   | _        || _	        |)| _
        d| _        |$}0|&|&}1n|dk(  }1|%|%}2n|1 }2i }3|)r|3j                  t        dd	             t        d+|||||*|$ d
|3|/| _        | j                  j                   }4t#        | j                  d      r| j                  j%                         n|}5|r+t'        j(                  t+        j,                  dd|fi |/      nd| _        |r+t'        j(                  t+        j,                  d||fi |/      nd| _        |xr | j0                  du | _        |r|4n|4| j                  z   }6|r+t'        j(                  t+        j,                  d|6|fi |/      nd| _        t'        j6                  |      | _        |dkD  rt;        || j                        | _        nd| _        d| _        |r|+tA        |+      nd}+t        d+|||)rdn| j                  jB                  |!| d|/}7|dk(  r#|7j                  t        |             d| _        n"|dk(  r|7j                  t        d||+             tE        d+d|i|7| _#        nd| _#        |0r	 ||fi |/nt'        jH                         | _%        tM        ||      }8|#rtN        ntP        }9t'        jR                  tU        |      D :cg c]J  }: |9d+i d|d|d|	d|
d|d|d|d|d|d|d|"d | j                  d!|d"|d#|8|:   d$|d%||/L c}:      | _+        tU        |      D :cg c]  }:t        d&|: ||5'       c}:| _,        |2r	 ||fi |/nt'        jH                         | _-        |d(k(  r7t]        | j                  f|'xs ||(xs ||t&        j^                  d)|/| _0        nd| _0        |1r	 ||fi |/nt'        jH                         | _1        t'        j6                  |      | _2        |dkD  rt'        jf                  ||fi |/nt'        jH                         | _4        |,| _5        | jm                  d*       yc c}:w c c}:w ),ay  Initialize the EVA Vision Transformer model.

        Args:
            img_size: Input image size (single int for square, or tuple for rectangular)
            patch_size: Patch size to divide image into tokens (single int for square, or tuple)
            in_chans: Number of input image channels
            num_classes: Number of classes (output dim) for classification head (final projection), 0 for pass-through
            global_pool: Type of global pooling for final sequence ('avg', 'token', 'map', etc.)
            embed_dim: Embedding dimension for tokens
            depth: Number of transformer blocks
            num_heads: Number of attention heads
            qkv_bias: Enable bias for query, key, value projections
            qkv_fused: Use a single projection for query, key, value
            mlp_ratio: Ratio of mlp hidden dim to embedding dim
            swiglu_mlp: Use SwiGLU activation in MLP
            scale_mlp: Apply scaling normalization in MLP (normformer style)
            scale_attn_inner: Apply scaling normalization inside attention
            attn_type: Type of attention module to use
            drop_rate: Dropout rate after final projection and pooling
            pos_drop_rate: Dropout rate for positional embeddings
            patch_drop_rate: Rate of dropping patches during training
            proj_drop_rate: Dropout rate for projections
            attn_drop_rate: Dropout rate for attention
            drop_path_rate: Stochastic depth rate
            norm_layer: Normalization layer constructor
            init_values: Initial layer-scale values
            class_token: Use class token
            num_reg_tokens: Number of additional learnable 'register' tokens to add to the sequence
            no_embed_class: Don't include position embeddings for class (or reg) tokens
            use_abs_pos_emb: Use absolute (learned) positional embeddings
            use_rot_pos_emb: Use rotary position embeddings
            rope_type: Type of RoPE to use ('cat', 'mixed', 'dinov3', etc.).
            rope_grid_offset: Offset for rotary position embedding grid
            rope_grid_indexing: Indexing mode for rotary position embeddings ('ij' or 'xy')
            rope_temperature: Temperature parameter for ROPE frequency computation
            rope_rotate_half: Use half rotation layout (rotate D/2 dims), else use interleaved rotation layout
            use_post_norm: Use post-norm transformer block type
            use_pre_transformer_norm: Use normalization layer before transformer blocks
            use_post_transformer_norm: Use normalization layer after transformer blocks
            use_fc_norm: Use normalization layer after pooling, before final classifier
            attn_pool_num_heads: Number of heads in attention pooling
            attn_pool_mlp_ratio: MLP ratio in attention pooling
            dynamic_img_size: Support dynamic image sizes in forward pass
            dynamic_img_pad: Apply dynamic padding for irregular image sizes
            ref_feat_shape: Reference feature shape for rotary position embedding scale
            head_init_scale: Initialization scale for classification head weights
        r>   ) r   avgmaxmaxtokenmapr%   r   FNr   NHWC)strict_img_size
output_fmt)r   r   r   r   r   rB   
feat_ratio)r   )r4   )r/   r0   
feat_shapetemperaturegrid_indexingmixed)r   Try   )	in_pixelsgrid_offsetr   r   r/   r0   r1   r2   r   r   r   r   r   r   r;   r4   r6   r5   r   r8   r   blocks.)modulenum_chs	reductionr  )r0   r   r8   r   needs_resetr   )7rE   rF   r   r   r   num_featureshead_hidden_sizer   r4   r   r   grad_checkpointingupdatedictr   patch_embednum_patcheshasattrr  rI   rO   rP   rQ   	cls_token	reg_token	cls_embed	pos_embedrX   pos_dropr   
patch_drop
rope_mixedr   	grid_sizer   rj   rU   norm_prer   r   r   
ModuleListrangeblocksfeature_inforY   r$   r   	attn_poolfc_norm	head_droprJ   headr   init_weights)<r]   r   r   r   r   r   r   r   r0   r1   r2   r   r   r   r   r   r   r   r   r   r   r   r   r8   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r?   r@   r^   activate_pre_normactivate_fc_normactivate_post_norm
embed_argsr  rnum_pos_tokensrope_kwargsdprblock_fnir`   s<                                                              ra   rF   zEva.__init__  s   @ 	/JJJJ& &ENNND1DN'2!^!K, 0"' 5"**e3$0!:%5!5
d5VLM% 	
!+--	
 	
 	
 &&22-4T5E5E|-TD'')ZdMXekk!Q	&HR&HI^bZhekk!^Y&URT&UVnr$?4)?(6K$J`J`<`Ziekk!^Y&URT&UVos

]3Q5oY]YoYopDO"DO:H:TY~6Z^N  ##349I9I9S9S,0 K G#""4e#45"&e#""4# 0#1$  *MIMMDIDI7H
933bkkm'>'4#(mm* 5\+%#* )  # " $	
 $ & !0 $ "2 $ - #'"8"8 ) ) a&  &!" (%%# $. QVV[P\^KLD'!yAF^ 4FJy/B/2;;=	%0-:-:%'' DN "DN6Fz)2r2BKKMI.?JQBIIi;;TVT_T_Ta	. 	e,Y%#,^s   AR/Rr  c                    | j                  t        | j                  |             | j                  t	        | j                  d       | j
                  t	        | j
                  d       | j                  t	        | j                  d       | j                          | j                  rt        | j                  t        j                        rt	        | j                  j                  d       t        j                         5  | j                  j                  j!                  | j                         | j                  j"                  j!                  | j                         d d d        y y y # 1 sw Y   y xY w)Nr  {Gz?std)applyr   _init_weightsr  r   r  r  fix_init_weightr   
isinstancer*  rI   rJ   rn   rP   no_gradmul_rB   )r]   r  s     ra   r+  zEva.init_weights  s   

74--;GH>>%$..c2>>%$..c2>>%$..c2Jtyy"))$D$))**4 :		  %%d&:&:;		##D$8$89: : %E: :s   AE//E8r   c                    t        j                         5  t        | j                        D ]~  \  }}t	        j
                  d|dz   z        }|j                  j                  j                  j                  |       |j                  j                  j                  j                  |        	 ddd       y# 1 sw Y   yxY w)z=Fix initialization weights by rescaling based on layer depth.g       @r%   N)rP   r>  	enumerater%  mathsqrtr   r[   rn   div_r   fc2)r]   layer_idlayerrH   s       ra   r<  zEva.fix_init_weight  s    ]]_ 	1#,T[[#9 1%		#A"67

&&++E2		$$))%01	1 	1 	1s   BB66B?mc                    t        |t        j                        rNt        |j                  d       |j
                  *t        j                  j                  |j
                         yy|r#t        |d      r|| ur|j                          yyyy)zInitialize weights for Linear layers and call reset_parameters on modules.

        Args:
            m: Module to initialize.
            needs_reset: Whether to call reset_parameters() on modules.
        r7  r8  Nr\   )
r=  rI   rJ   r   rn   rB   rd   re   r  r\   )r]   rH  r  s      ra   r;  zEva._init_weights  sn     a#!((,vv!qvv& "WQ(:;  AN;[rb   c                     h d}t        | dd      x}r0t        |d      r$||j                         D ch c]  }d| 	 c}z  S |S c c}w )z(Parameters to exclude from weight decay.>   r  r  r  rj   Nno_weight_decayzrope.)r   r  rK  )r]   nwdrj   r   s       ra   rK  zEva.no_weight_decay  sW     6D&$//D/WTCT5Ut/C/C/EF!E!+FFF
 Gs   Aenablec                     || _         y)z)Enable or disable gradient checkpointing.N)r  )r]   rM  s     ra   set_grad_checkpointingzEva.set_grad_checkpointing&  s     #)rb   coarsec                 $    t        dddg      }|S )z(Create layer groupings for optimization.z*^cls_token|reg_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)i )stemr%  )r  )r]   rP  matchers      ra   group_matcherzEva.group_matcher+  s!     >-/CD
 rb   c                     | j                   S N)r*  rg   s    ra   get_classifierzEva.get_classifier4  s    yyrb   c                     || _         ||| _        |dkD  r&t        j                  | j                  |      | _        yt        j
                         | _        y)zReset the classifier head.

        Args:
            num_classes: Number of output classes.
            global_pool: Global pooling type.
        Nr   )r   r   rI   rJ   r   rU   r*  )r]   r   r   s      ra   reset_classifierzEva.reset_classifier8  sF     '"*D>IAoBIIdnnk:	SUS^S^S`	rb   c           	      $   | j                   j                  }| j                   j                  ||       | j                  | j                  rdn| j
                  }| j                   j                  |z   }|| j                  j                  d   k7  rFt        j                  t        | j                  | j                   j                  ||d            | _        | j                  0| j                  j                  | j                   j                         yy)zUpdate the input image resolution and patch size.

        Args:
            img_size: New input resolution, if None current resolution is used.
            patch_size: New patch size, if None existing patch size is used.
        )r   r   Nr   r%   T)new_sizeold_sizer4   verbose)r  r!  set_input_sizer  r   r4   r  rw   rI   rO   r   rj   update_feat_shape)r]   r   r   prev_grid_sizer4   num_new_tokenss         ra   r^  zEva.set_input_sizeD  s     ))33''j'Q>>%%)%8%8d>T>T!--99<MMN!5!5a!88!#.DNN!--77+&7 / " 99 II''(8(8(B(BC !rb   c                    | j                   r|j                  \  }}}}| j                  J| j                  j                  }t        | j                  ||f|| j                  rdn| j                        }nd }|j                  |d|      }| j                  | j                  j                  ||f      nd }n4| j                  }| j                  | j                  j                         nd }g }	| j                  9|	j                  | j                  j                  |j                  d   dd             | j                  9|	j                  | j                  j                  |j                  d   dd             | j                  r%|||z   }|	r@t        j                   |	|gz   d      }n$|	rt        j                   |	|gz   d      }|||z   }| j#                  |      }| j$                  Y| j%                  |      \  }}
|C|
At'        |||
      }t)        | dd      r|j+                  dd      }||fS |j-                  d      }||fS )	Nr   )r[  r\  r4   ro   )rw   r%   rs   r   F)r   rw   r  r  r!  r   r   r4   viewrj   	get_embedr  appendexpandr  rP   ry   r  r  r   r   r   	unsqueeze)r]   r   r   HWr   r`  r  rot_pos_embedto_catkeep_indicess              ra   
_pos_embedzEva._pos_embeda  s     JAq!Q~~)!%!1!1!;!;2NNV++/+>+>aDDZDZ		 !	q"a AAEAVDII//q!f/=\`MI59YY5JDII//1PTM>>%MM$..//
BCD>>%MM$..//
BCD$	MIIfsl2 IIfsl2$	MMM! ??&"ooa0OA|(\-E 6q- V4u5$1$;$;Aq$AM
 - %2$;$;A$>M-rb   r   indicesreturn_prefix_tokensrY   
stop_earlyr  intermediates_onlyrk   rl   c
           	         |dv sJ d       |dk(  }
g }t        t        | j                        |      \  }}|j                  \  }}}}| j	                  |      }| j                  |      \  }}| j                  |      }t        j                  j                         s|s| j                  }n| j                  d|dz    }t        | dd      r|t        |      D ]{  \  }}| j                  r2t        j                  j                         st        ||||   ||	      }n ||||   ||	      }||v sX|j                  |r| j                  |      n|       } nt        |      D ]u  \  }}| j                  r/t        j                  j                         st        |||||	      }n |||||	      }||v sR|j                  |r| j                  |      n|       w | j                   rD|D cg c]  }|ddd	| j                   f    }}|D cg c]  }|dd| j                   df    }}|
ra| j                  j#                  ||f      \  }}|D cg c]6  }|j%                  |||d
      j'                  d	ddd      j)                         8 }}t        j                  j                         s|rt+        t-        |            }|r|S | j                  |      }||fS c c}w c c}w c c}w )a   Forward features that returns intermediates.
        Args:
            x: Input image tensor
            indices: Take last n blocks if an int, if is a sequence, select by matching indices
            return_prefix_tokens: Return both prefix and spatial intermediate tokens
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
            attn_mask: Optional attention mask for masked attention
            is_causal: If True, use causal (autoregressive) masking in attention
        )NCHWNLCz>Output format for EVA-ViT features must be one of NCHW or NLC.rs  Nr%   r   Fr   r   ro   rA   rp   )r'   lenr%  rw   r  rm  r"  rP   r   is_scriptingr   rA  r  r(   re  rY   r4   dynamic_feat_sizer|   r}   
contiguouslistzip)r]   r   rn  ro  rY   rp  r  rq  rk   rl   r|   intermediatestake_indices	max_indexr   _heightwidthrj  r%  r5  blkyprefix_tokensrh  ri  s                             ra   forward_intermediateszEva.forward_intermediates  s   0 _,n.nn,&"6s4;;7G"Qi  gg1feQ??1-=MM!99!!#:[[F[[)a-0F 4u--2K#F+ F3**5993I3I3K"3a0@IajkAAM!$4	U^_A$!((11EF $F+ F3**5993I3I3K"3^ghAAMYR[\A$!((11EF !!ERSQq!D$:$:"::;SMSDQRqQq$"8"8"99:RMR##55vuoFDAq^klYZQYYq!Q3;;Aq!QGRRTlMlyy%%',@ ]M!BCM  IIaL- TR ms   'K	K;K"
prune_norm
prune_headc                    t        t        | j                        |      \  }}| j                  d|dz    | _        |rt        j                         | _        |r2d| _        t        j                         | _        | j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr%   r   r   )	r'   ru  r%  rI   rU   rY   r'  r(  rY  )r]   rn  r  r  r|  r}  s         ra   prune_intermediate_layerszEva.prune_intermediate_layers  sq     #7s4;;7G"Qikk.9q=1DI!DN;;=DL!!!R(rb   	pool_typec                     | j                   | j                  |      }|S || j                  n|}t        ||| j                        }|S )N)r  r4   )r'  r   r   r4   )r]   r   r  s      ra   poolzEva.pool  sJ    >>%q!AH(1(9D$$y	AdF\F\]rb   c                 8   | j                  |      }| j                  |      \  }}| j                  |      }t        | dd      rm|kt	        | j
                        D ]R  \  }}| j                  r2t        j                  j                         st        ||||   ||      }D ||||   ||      }T nX| j
                  D ]I  }| j                  r/t        j                  j                         st        |||||      }> |||||      }K | j                  |      }|S )a(  Forward pass through feature extraction layers.

        Args:
            x: Input tensor.
            attn_mask: Optional attention mask for masked attention
            is_causal: If True, use causal (autoregressive) masking in attention.

        Returns:
            Feature tensor.
        r   Fr   )r  rm  r"  r   rA  r%  r  rP   r   rv  r(   rY   )r]   r   rk   rl   rj  r5  r  s          ra   forward_featureszEva.forward_features  s     Q??1-=MM!4u--2K $DKK0 `3**5993I3I3K"3a0@IajkAAM!$4	U^_A	` {{ ]**5993I3I3K"3^ghAAMYR[\A	] IIaLrb   
pre_logitsc                     | j                  |      }| j                  |      }| j                  |      }|r|S | j                  |      S )zForward pass through classifier head.

        Args:
            x: Feature tensor.
            pre_logits: Return pre-logits if True.

        Returns:
            Output tensor.
        )r  r(  r)  r*  )r]   r   r  s      ra   forward_headzEva.forward_head&  sA     IIaLLLONN1q0DIIaL0rb   c                 P    | j                  |||      }| j                  |      }|S )a  Forward pass.

        Args:
            x: Input tensor.
            attn_mask: Optional attention mask for masked attention
            is_causal: If True, use causal (autoregressive) masking in attention.

        Returns:
            Output tensor.
        )rk   rl   )r  r  )r]   r   rk   rl   s       ra   r   zEva.forward5  s0      !!!yI!Na rb   )Tr   FrV  )NN)NFFFrs  FNF)r%   FT)NF))r   r   r   r   r   r   r   r
   r   r   r   r   r   rF   r+  r<  rI   Moduler;  rP   r   ignorer	   rK  rO  r   r   rT  rW  rY  r^  r   rm  r   r  r  r  r  r  r   r   r   s   @ra   r+   r+     s<    5868#$ !"!$#$#%*"!#%%'$&$&$&#,+/ $"##($($)',&(&*&,%*"'-28<*.1537%*$)DH%*__-CsCx01_- c5c?23_- 	_-
 _- _- _- _- _- _- _- _- _- !_- _-  #!_-" #_-$ %_-& !'_-( #)_-* "+_-, "-_-. "/_-0 !1_-2 "%3_-4 5_-6  7_-8 !9_-: ";_-< "=_->  }?_-@ $A_-B !$C_-D $E_-F #G_-H  I_-J '+K_-L (0~M_-N "$O_-P "*#Q_-R "*%S_-T #U_-V "W_-X %U5c?C+?%@AY_-Z #[_-B: :"1!ryy !t !t ! YYS   YY)T )T ) ) YYD T#s(^   YY		  
aC 
ahsm 
aW[ 
a 3748DuS#X/D !sCx1D 
	D:5 uU\\8ELL3I%IJ 5 t 8<).$$',04#K ||K  eCcN34K  #'	K 
 K  K  K  !%K   -K  K  
tELL!5tELL7I)I#JJ	KK ^ ./$#	3S	>*  	$ell x}   15#	%||%  -% 	%
 
%N1ell 1 1 1$ 15#	||  - 	
 
rb   
state_dictmodelprefixr   c                    | j                  d|       } | j                         D ci c]  \  }}|j                  dd      | } }}i }g d}t        |      }| j                         D ]n  \  }}|r|j	                  |      s||d }|D ]  }|j                  |d   |d         } |j	                  d      r|j                  d	d      }|j                  d
d      }|j                  dd      }|j	                  d      r|j
                  d   dz  }	|j                  d      r|d|	 |d<   ||	d |d<   n!|j                  d      r|d|	 |d<   ||	d |d<   |dk(  r:d}|j                  dd      }t        j                  |j
                  d         |d<   n>|dk(  r#d}|j                  d      j                  d      }n|dk(  r|j                  d      }|||<   q |S c c}}w )zConvert Perception Encoder weights.

    Args:
        state_dict: State dictionary to convert.
        model: Target model instance.
        prefix: Prefix to strip from keys.

    Returns:
        Converted state dictionary.
    r  zmodule.r   ))conv1patch_embed.proj)positional_embeddingr  )ztransformer.resblocks.r  )ln_prer"  )ln_postrY   )ln_rY   )z
ls_1.gammar   )z
ls_2.gammar   )in_proj_zqkv.)out_projr[   )zmlp.c_fcmlp.fc1)z
mlp.c_projmlp.fc2Nr   r%   r'  zattn_pool.attnzattn_pool.layernormzattn_pool.normzattn_pool.probezattn_pool.latentzattn_pool.qkvrA   rn   zattn_pool.q.weightzattn_pool.kv.weightrB   zattn_pool.q.biaszattn_pool.kv.biasr[   zhead.weightz	head.biasclass_embeddingr  r  )getitemsreplaceru  
startswithrw   endswithr   rP   zerosrg  )
r  r  r  r   r   out_dictswaps
len_prefixspr/   s
             ra   _convert_per  J  s    4J:D:J:J:LM$!Q!))Ir*A-MJMHE VJ  " 1<<'*+A 	(B		"Q%A'A	( <<$		*K8A		/1ABA		+-?@A||O,ggajAo::h'56tWH1267gH23ZZ'34Tc7H/045cdGH01&[AAq!A$)KK
$;H[!##AA((+A+AA?B Og Ns   Ginterpolation	antialiasc           	         i }| j                  d|       } | j                  d|       } | j                  d|       } | j                  d|       } d| v rt        | |      S d| v rt        | |d      S d	| v rd
}n	d| v rd}nd}d| v }| xr |dz   | v }|dz   | v }t        |      }	| j                         D ]  \  }
}|r|
j	                  |      s|
|	d }
d|
v r|
dk(  s*|rt        dD cg c]  }|
j                  |       c}      rU|
j	                  d      rg|
j	                  d      ry|
j                  d      rV|
j                  dd      }	 |j                  |       |j                  dd      \  }}}|||<   |||
j                  dd      <   |
j                  dd      }
|
j                  dd      }
|
j                  dd       }
n%|r#|
d!v r|
d"k(  s|
d#k(  r|
j                  d$d%      }
n<d&|
v rf|j                  j                  j                  j                  \  }}}}|j                  d   |k7  s|j                  d'   |k7  rt!        |||f||d()      }no|
d*k(  rj|j                  d+   |j"                  j                  d+   k7  rAt%        |d,d-      rd.nt%        |d/d+      }t'        ||j                  j(                  |||d(0      }|
j                  d1d2      }
|
j                  d3d4      }
|
j                  d5d6      }
|
j                  d7d8      }
|
j                  d9d:      }
|
j                  d;d<      }
|r$|
j                  dd=      }
|
j                  dd>      }
|||
<    |S c c}w # t        $ r}t        |       Y d}~d}~ww xY w)?aZ  Convert patch embedding weight from manual patchify + linear proj to conv.

    Args:
        state_dict: Checkpoint state dictionary.
        model: Target model instance.
        interpolation: Interpolation method for resizing.
        antialias: Whether to use antialiasing when resizing.

    Returns:
        Filtered state dictionary.
    	model_emar  r  r  zvisual.conv1.weightzconv1.weightr   )r  zvisual.trunk.pos_embedzvisual.trunk.zvisual.pos_embedvisual.storage_tokens
mask_tokenzblocks.0.attn.q_proj.weightNrj   z
rope.freqs)z.periodsz
.bias_maskr  zprojectors.local_cls_normzqkv.biasrR   rA   ro   rs   rT   z	ls1.gammar   z	ls2.gammar   r  )r  zlm_head.weightzlm_head.biasnorm.weight	norm.biasr  r  rY   r(  zpatch_embed.proj.weightrv   T)r  r  r]  r  r%   r   Fr   r4   )r[  r4   r  r  r]  z
mlp.ffn_lnzmlp.normzattn.inner_attn_lnz	attn.normzmlp.w12r  zmlp.w1z	mlp.fc1_gzmlp.w2z	mlp.fc1_xzmlp.w3r  zq_proj.biaszv_proj.bias)r  r  ru  r  r  anyr  r  get_parameter	Exceptionprintchunkr  r[   rn   rw   r   r  r   r   r!  )r  r  r  r  r  r  dinov3_weightsmim_weightsno_qkvr  r   r   fq_bias_keqvkvvvr~  rh  ri  r4   s                         ra   checkpoint_filter_fnr    s   " HZ8J4J*5Jj9J 
*:u--	:	%:uR88  :- 	z	)%3N$$L,)>*)LK33zAFVJ  " O1<<'*+AQ;qL0+STaAJJqMTU||M*||,-zz*%99Z: ''1 WWQBW/
B%'"<>:x89		+y1A		+y1A		*K8AQ"nnM!Q+%5IIfi0 $)**//66<<JAq!Qwwr{a1772;!#3(F"/'  +!''!*0E0Ea0H"H%,U4De%LRYZ_atvwRx&**44"3+#A IIlJ/II*K8IIi+IIh,IIh,IIh	*		(M2A		(M2A_Ob OM U ! !Hs   $M 
M%%	N.M??Nvariant
pretrainedc                    |j                  dd      }t        j                  j                  dd      dk(  }||}|rddlm}  || |fi |S |j                  dd	      }t        t        | |ft        t        |d
      d|}|S )zCreate an EVA model.

    Args:
        variant: Model variant name.
        pretrained: Load pretrained weights.
        **kwargs: Additional model arguments.

    Returns:
        Instantiated Eva model.
    
use_naflexNTIMM_USE_NAFLEX01r%   )_create_naflexvit_from_evaout_indicesrA   getter)r  feature_cls)pretrained_filter_fnfeature_cfg)
poposenvironr  	naflexvitr  r&   r+   r  r  )r  r  r   r  _USE_NAFLEX_DEFAULTr  r  r  s           ra   _create_evar    s     L$/J**..):C@CG(
9)':HHH**]A.K Wj1[hG 	E Lrb   urlc                 4    | ddddddt         t        ddd	d
|S )zGenerate default configuration for EVA models.

    Args:
        url: Model weights URL.
        **kwargs: Additional configuration parameters.

    Returns:
        Model configuration dictionary.
    r   rA   r   r   Ng?bicubicTr  r*  mitr  r   
input_size	pool_sizecrop_pctr  fixed_input_sizemeanr9  
first_conv
classifierlicense)r   r   r  r   s     ra   _cfgr  .  s6     =t ( # rb   c                 $    | dddddddddd	d
d|S )zGenerate default configuration for Perception Encoder models.

    Args:
        url: Model weights URL.
        **kwargs: Additional configuration parameters.

    Returns:
        Model configuration dictionary.
    r   r  N      ?r  T      ?r  r  r  r*  
apache-2.0r  r   r  s     ra   _pe_cfgr  B  s6     D)( $* rb   c                 4    | ddddddt         t        ddd	d
|S )a  Generate default configuration for DINOv3 models.

    Note: Original DINOv3 uses CLS-token pooling for representations. timm defaults to avg
    pooling for the Eva architecture. Pass global_pool='token' at model creation to match
    upstream behavior, which may be preferred for tasks like retrieval and few-shot classification.

    Args:
        url: Model weights URL.
        **kwargs: Additional configuration parameters.

    Returns:
        Model configuration dictionary.
    r   rA      r  Nr  r  Tr  r*  zdinov3-licenser  r   r   r  s     ra   _dinov3_cfgr  V  s7     D)%.B(# (. rb   c                 4    | ddddddt         t        ddd	d
|S )z/Generate default configuration for EUPE models.r   r  Nr  r  Tr  r*  z#fair-noncommercial-research-licenser  r  r  s     ra   	_eupe_cfgr  n  s8     D)%.B(8 =C rb   z"eva_giant_patch14_224.clip_ft_in1kztimm/)	hf_hub_idz"eva_giant_patch14_336.clip_ft_in1k)rA   P  r  r  squash)r  r  r  	crop_modez(eva_giant_patch14_336.m30m_ft_in22k_in1k)r  r  r9  r  r  r  z(eva_giant_patch14_560.m30m_ft_in22k_in1k)rA   0  r  z.eva02_base_patch14_448.mim_in22k_ft_in22k_in1k)rA     r  z/eva02_large_patch14_448.mim_in22k_ft_in22k_in1kz.eva02_large_patch14_448.mim_m38m_ft_in22k_in1kz(eva02_tiny_patch14_336.mim_in22k_ft_in1k)r  r  r  z)eva02_small_patch14_336.mim_in22k_ft_in1kz(eva02_base_patch14_448.mim_in22k_ft_in1kz)eva02_large_patch14_448.mim_in22k_ft_in1kz(eva02_large_patch14_448.mim_m38m_ft_in1kz)eva02_base_patch14_448.mim_in22k_ft_in22kiQU  )r  r  r  r  r   z*eva02_large_patch14_448.mim_in22k_ft_in22kz)eva02_large_patch14_448.mim_m38m_ft_in22kz eva02_tiny_patch14_224.mim_in22k)r  r   z!eva02_small_patch14_224.mim_in22kz eva02_base_patch14_224.mim_in22kz!eva02_large_patch14_224.mim_in22kz eva02_large_patch14_224.mim_m38mz$eva_giant_patch14_clip_224.laion400m   z#eva_giant_patch14_clip_224.merged2bz$eva02_base_patch16_clip_224.merged2b   z%eva02_large_patch14_clip_224.merged2br   z%eva02_large_patch14_clip_336.merged2b)r  r  r  r   z'eva02_enormous_patch14_clip_224.laion2bz,eva02_enormous_patch14_clip_224.laion2b_plusz(eva02_enormous_patch14_clip_224.pretrain)r   z-vit_medium_patch16_rope_reg1_gap_256.sbb_in1kr  gffffff?r  )r  r  r  r  r9  z.vit_mediumd_patch16_rope_reg1_gap_256.sbb_in1kz.vit_betwixt_patch16_rope_reg4_gap_256.sbb_in1kz+vit_base_patch16_rope_reg1_gap_256.sbb_in1kzvit_pe_core_tiny_patch16_384.fb)rA     r   )r  r  r   z vit_pe_core_small_patch16_384.fbzvit_pe_core_base_patch16_224.fbr  z vit_pe_core_large_patch14_336.fbz#vit_pe_core_gigantic_patch14_448.fb   z vit_pe_lang_large_patch14_448.fbz'vit_pe_lang_large_patch14_448.fb_tilingz#vit_pe_lang_gigantic_patch14_448.fbz*vit_pe_lang_gigantic_patch14_448.fb_tilingz"vit_pe_spatial_tiny_patch16_512.fb)rA   r  r  z#vit_pe_spatial_small_patch16_512.fbz"vit_pe_spatial_base_patch16_512.fbz#vit_pe_spatial_large_patch14_448.fbz&vit_pe_spatial_gigantic_patch14_448.fbz%vit_small_patch16_rope_224.naver_in1kr  )r  r  r9  r  z$vit_base_patch16_rope_224.naver_in1kz%vit_large_patch16_rope_224.naver_in1kz+vit_small_patch16_rope_mixed_224.naver_in1kz*vit_base_patch16_rope_mixed_224.naver_in1kz+vit_large_patch16_rope_mixed_224.naver_in1kz)vit_small_patch16_rope_ape_224.naver_in1kz(vit_base_patch16_rope_ape_224.naver_in1kz)vit_large_patch16_rope_ape_224.naver_in1kz/vit_small_patch16_rope_mixed_ape_224.naver_in1kz.vit_base_patch16_rope_mixed_ape_224.naver_in1kz/vit_large_patch16_rope_mixed_ape_224.naver_in1kz!vit_small_patch16_dinov3.lvd1689mz&vit_small_patch16_dinov3_qkvb.lvd1689mz&vit_small_plus_patch16_dinov3.lvd1689mz+vit_small_plus_patch16_dinov3_qkvb.lvd1689mz vit_base_patch16_dinov3.lvd1689mz%vit_base_patch16_dinov3_qkvb.lvd1689mz*vit_tiny_patch16_dinov3_qkvb.eupe_lvd1689mz+vit_small_patch16_dinov3_qkvb.eupe_lvd1689mz*vit_base_patch16_dinov3_qkvb.eupe_lvd1689mz!vit_large_patch16_dinov3.lvd1689m)gQ?gM?gl?)g$C?g+?gM?)r  r  r9  )z&vit_large_patch16_dinov3_qkvb.lvd1689mz vit_large_patch16_dinov3.sat493mz%vit_large_patch16_dinov3_qkvb.sat493mz%vit_huge_plus_patch16_dinov3.lvd1689mz*vit_huge_plus_patch16_dinov3_qkvb.lvd1689mzvit_7b_patch16_dinov3.lvd1689mzvit_7b_patch16_dinov3.sat493mc           	      T    t        ddddd      }t        dd| it        |fi |}|S )	,EVA-g model https://arxiv.org/abs/2211.07636     (   r   tE]t@r   r   r   r0   r   r  )eva_giant_patch14_224r  r  r  r   
model_argsr  s       ra   r	  r	    9     t2WbcJeJe$zJd]cJdeELrb   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	r  r  r  r  r   r  r  r  )eva_giant_patch14_336r
  r  s       ra   r  r    r  rb   c           	      T    t        ddddd      }t        dd| it        |fi |}|S )	r  r  r  r  r   r  r  r  )eva_giant_patch14_560r
  r  s       ra   r  r  '  r  rb   c                 \    t        ddddddddd		      }t        dd
| it        |fi |}|S )+EVA02 Tiny https://arxiv.org/abs/2303.11331r   r     r   rA   UUUUUU@Tr   r   	r   r   r   r   r0   r   r   r   r   r  )eva02_tiny_patch14_224r
  r  s       ra   r  r  /  N     
J fZf4PZKe^dKefELrb   c                 \    t        ddddddddd		      }t        dd
| it        |fi |}|S ),EVA02 Small https://arxiv.org/abs/2303.11331r   r  r   r      r  Tr  r  r  )eva02_small_patch14_224r
  r  s       ra   r  r  A  N     
J gjgDQ[Lf_eLfgELrb   c                 `    t        ddddddddddd	      }t        dd
| it        |fi |}|S )+EVA02 Base https://arxiv.org/abs/2303.11331r   r  r   r   Fr  Tr  r   r   r   r   r0   r2   r   r   r   r   r   r  )eva02_base_patch14_224r
  r  s       ra   r"  r"  S  T     J fZf4PZKe^dKefELrb   c                 `    t        ddddddddddd	
      }t        dd| it        |fi |}|S ),EVA02 Large https://arxiv.org/abs/2303.11331r   r  r     r   r  FTr  r   r   r   r   r0   r   r2   r   r   r   r   r  )eva02_large_patch14_224r
  r  s       ra   r(  r(  g  T     J gjgDQ[Lf_eLfgELrb   c                 \    t        ddddddddd		      }t        dd
| it        |fi |}|S )r  r  r  r  r   rA   r  Tr  r  r  )eva02_tiny_patch14_336r
  r  s       ra   r+  r+  {  r  rb   c                 \    t        ddddddddd		      }t        dd
| it        |fi |}|S )r  r  r  r   r   r  r  Tr  r  r  )eva02_small_patch14_336r
  r  s       ra   r-  r-    r  rb   c                 `    t        ddddddddddd	      }t        dd
| it        |fi |}|S )r   r  r  r   r   Fr  Tr  r!  r  )eva02_base_patch14_448r
  r  s       ra   r/  r/    r#  rb   c                 `    t        ddddddddddd	
      }t        dd| it        |fi |}|S )r%  r  r  r  r&  r   r  FTr  r'  r  )eva02_large_patch14_448r
  r  s       ra   r1  r1    r)  rb   c                 v    t        ddddd|j                  dd            }t        d
d	| it        |fi |}|S )z?EVA-g CLIP model (only difference from non-CLIP is the pooling)r  r  r  r   r  r   r   )r   r   r   r0   r   r   r  )eva_giant_patch14_clip_224r  r  r  r  s       ra   r3  r3    sL     R2JJ}g68J jjtT^OibhOijELrb   c                     t        dddddddddddd|j                  d	d
            }t        dd| it        |fi |}|S )zUAn EVA-CLIP specific variant that adds additional attn scale layer-norm to eva02_baser   r   r   r   Fr  Tr  r   r   )r   r   r   r   r0   r2   r   r   r   r   r   r   r   r  )eva02_base_patch16_clip_224r4  r  s       ra   r6  r6    sf     JJ}g6J k*kPTU_PjciPjkELrb   c                     t        dddddddddddd	|j                  d
d            }t        dd| it        |fi |}|S )VAn EVA-CLIP specific variant that adds additional attn scale layer-norm to eva02_larger   r  r  r&  r   r  FTr  r   r   r   r   r   r   r0   r   r2   r   r   r   r   r   r   r  )eva02_large_patch14_clip_224r4  r  s       ra   r:  r:    f     JJ}g6J l:lQUV`QkdjQklELrb   c                     t        dddddddddddd	|j                  d
d            }t        dd| it        |fi |}|S )r8  r  r  r  r&  r   r  FTr  r   r   r9  r  )eva02_large_patch14_clip_336r4  r  s       ra   r=  r=    r;  rb   c                 z    t        ddddddd|j                  dd	      
      }t        dd| it        |fi |}|S )zCAn EVA-CLIP specific variant that uses residual post-norm in blocksr   r  i   @   r   gI$I$!@Tr   r   )r   r   r   r   r0   r   r   r   r  )eva02_enormous_patch14_clip_224r4  r  s       ra   r@  r@    sW     JJ}g6	J ojoTXYcTngmTnoELrb   c                 d    t        dddddddddd	ddd
      }t        dd| it        |fi |}|S )timm SBB ViT with ROPEr  r   r  r   r   Th㈵>Fr%   r  r   r   r   r   r0   r2   r1   r   r   r   r   r   r   r  )$vit_medium_patch16_rope_reg1_gap_256r
  r  s       ra   rE  rE  $  s[     J t:tY]^hYslrYstELrb   c                 d    t        dddddddddd	ddd
      }t        dd| it        |fi |}|S )rB  r  r   r     r   TFrC  r%   r  rD  r  )%vit_mediumd_patch16_rope_reg1_gap_256r
  r  s       ra   rH  rH  :  s[     J uJuZ^_iZtmsZtuELrb   c                 d    t        dddddddddd	ddd
      }t        dd| it        |fi |}|S )rB  r  r   i  r   
   TrC  Frq   r  rD  r  )%vit_betwixt_patch16_rope_reg4_gap_256r
  r  s       ra   rK  rK  P  s[     J uJuZ^_iZtmsZtuELrb   c                 d    t        ddddddddddddd	
      }t        dd| it        |fi |}|S )rB  r  r   r   r   TrC  Fr%   r  rD  r  )"vit_base_patch16_rope_reg1_gap_256r
  r  s       ra   rM  rM  f  s[     J rrW[\fWqjpWqrELrb   c                     t        dddddddddd	d
dddt        t        d            }t        dd| it        |fi |S )HPerception Encoder (PE) ViT from Meta (https://arxiv.org/abs/2504.13181)r   r  r   rA   r   r  rj   Tr&  r&  r  xyr   rC  epsr   r   r   r0   r   r   r   r   r   r   r   r   r   r   r8   r  )vit_pe_core_tiny_patch16_384r  r   r   r  r  r   r  s      ra   rU  rU  |  sd     !%9$/J$ k*kPTU_PjciPjkkrb   c                     t        dddddddddd	d
dddt        t        d            }t        dd| it        |fi |S )rO  r   r   r   r  r   r  rj   TrP  r  rQ  r   rC  rR  rT  r  )vit_pe_core_small_patch16_384rV  rW  s      ra   rY  rY    sd     !%9$/J$ l:lQUV`QkdjQkllrb   c                     t        ddddddddddd	d
ddt        t        d            }t        dd| it        |fi |S )rO  r   r   r   r   r  rj   T)r  r  r  rQ  r   rC  rR  rT  r  )vit_pe_core_base_patch16_224rV  rW  s      ra   r[  r[    sd     !%9$/J$ k*kPTU_PjciPjkkrb   c                     t        dddddddddd	d
dddt        t        d            }t        dd| it        |fi |S )rO  r  r  r&  r   r   r  rj   TrP  r  rQ  r   rC  rR  rT  r  )vit_pe_core_large_patch14_336rV  rW  s      ra   r]  r]    sd     !%9$/J$ l:lQUV`QkdjQkllrb   c                     t        ddddddddd	d	d
dddt        t        d            }t        dd| it        |fi |S )rO  r     2   r   UUUUUU@r  rj   FT    rc  rQ  r   r   rC  rR  )r   r   r   r0   r   r   r   r   r   r   r   r   r   r   r8   r  ) vit_pe_core_gigantic_patch14_448rV  rW  s      ra   rd  rd    sd     !%9$/J$ ojoTXYcTngmTnoorb   c           	          t        d i ddddddddd	d
dddddddddddddddddddddt        t        d      }t        d!d| it        |fi |S )"rO  r   r  r   r  r      r0   r   r   r   r   rj   r   Tr   r   rb  r   r  r   rQ  r   r   Fr   r   皙?r8   rC  rR  r  r   )vit_pe_lang_large_patch14_448rV  rW  s      ra   rh  rh    s        	
          "& #(    9$/!J& l:lQUV`QkdjQkllrb   c                     t        ddddddddd	d
ddddt        t        d            }t        dd| it        |fi |S )rO  r  r_  /   r   ra  rj   FTrb  rQ  rg  rC  rR  r   r   r   r0   r   r   r   r   r   r   r   r   r   r   r8   r  ) vit_pe_lang_gigantic_patch14_448rV  rW  s      ra   rl  rl  	  sd     !%"'9$/J$ ojoTXYcTngmTnoorb   c                     t        ddddddddddd	d
dt        t        d            }t        dd| it        |fi |S )rO  r   r  r   rA   r   rj   TFrb  r  rQ  rC  rR  r   r   r   r0   r   r   r   r   r   r   r   r   r   r8   r  )vit_pe_spatial_tiny_patch16_512rV  rW  s      ra   ro  ro  &	  sa     !%"'9$/J" nZnSWXbSmflSmnnrb   c                     t        ddddddddddd	d
dt        t        d            }t        dd| it        |fi |S )rO  r   r   r   r  r   rj   TFrb  r  rQ  rC  rR  rn  r  ) vit_pe_spatial_small_patch16_512rV  rW  s      ra   rq  rq  =	  sa     !%"'9$/J" ojoTXYcTngmTnoorb   c                     t        dddddddddddd	d
t        t        d            }t        dd| it        |fi |S )rO  r   r   r   r   rj   TFrb  r  rQ  rC  rR  rn  r  )vit_pe_spatial_base_patch16_512rV  rW  s      ra   rs  rs  T	  sa     !%"'9$/J" nZnSWXbSmflSmnnrb   c                     t        ddddddddddd	d
dt        t        d            }t        dd| it        |fi |S )rO  r  r  r&  r   r   rj   TFrb  r  rQ  rC  rR  rn  r  ) vit_pe_spatial_large_patch14_448rV  rW  s      ra   ru  ru  k	  sa     !%"'9$/J" ojoTXYcTngmTnoorb   c                     t        ddddddddd	d
ddddt        t        d            }t        dd| it        |fi |S )rO  r  r_  r`  r   ra  rj   FTrb  rQ  rg  rC  rR  rk  r  )#vit_pe_spatial_gigantic_patch14_448rV  rW  s      ra   rw  rw  	  sd     !%"'9$/J$ rrW[\fWqjpWqrrrb   c                 f    t        dddddddddd	d
ddd      }t        dd| it        |fi |}|S )z=RoPE-Axial ViT-S/16 from https://github.com/naver-ai/rope-vitr   r   r   r  rq   rj   TrC  r   FrQ        Y@r   r   r   r0   r   r   r1   r   r   r   r   r   r   r   r  )vit_small_patch16_rope_224r
  r  s       ra   r{  r{  	  s]     J  jjtT^OibhOijELrb   c                 h    t        ddddddddddd	ddd
d      }t        dd| it        |fi |}|S )z=RoPE-Axial ViT-B/16 from https://github.com/naver-ai/rope-vitr   r   r   rq   rj   FTrC  r   rQ  ry  )r   r   r   r0   r   r   r   r1   r   r   r   r   r   r   r   r  )vit_base_patch16_rope_224r
  r  s       ra   r}  r}  	  s`     J" i
idS]NhagNhiELrb   c                 f    t        ddddddddddd	dd
d      }t        dd| it        |fi |}|S )z=RoPE-Axial ViT-L/16 from https://github.com/naver-ai/rope-vitr   r  r&  rq   rj   TrC  r   FrQ  ry  rz  r  )vit_large_patch16_rope_224r
  r  s       ra   r  r  	  s]     J  jjtT^OibhOijELrb   c                 h    t        dddddddddd	d
dddd      }t        dd| it        |fi |}|S )z=RoPE-Mixed ViT-S/16 from https://github.com/naver-ai/rope-vitr   r   r   r  rq   rj   TrC  r   FrQ        $@r	  r   r   r   r0   r   r   r1   r   r   r   r   r   r   r   r   r  ) vit_small_patch16_rope_mixed_224r
  r  s       ra   r  r  	  sa     J" pzpUYZdUohnUopELrb   c                 h    t        ddddddddddd	dd
dd      }t        dd| it        |fi |}|S )z=RoPE-Mixed ViT-B/16 from https://github.com/naver-ai/rope-vitr   r   r   rq   Trj   rC  r   FrQ  r  r	  )r   r   r   r0   r   r1   r   r   r   r   r   r   r   r   r   r  )vit_base_patch16_rope_mixed_224r
  r  s       ra   r  r  	  sa     J" ojoTXYcTngmTnoELrb   c                 h    t        ddddddddddd	dd
dd      }t        dd| it        |fi |}|S )z=RoPE-Mixed ViT-L/16 from https://github.com/naver-ai/rope-vitr   r  r&  rq   rj   TrC  r   FrQ  r  r	  r  r  ) vit_large_patch16_rope_mixed_224r
  r  s       ra   r  r  
  sa     J" pzpUYZdUohnUopELrb   c                 h    t        dddddddddd	dddd
d      }t        dd| it        |fi |}|S )zCRoPE-Axial + APE ViT-S/16 from https://github.com/naver-ai/rope-vitr   r   r   r  rq   rj   TrC  r   rQ  ry  r   r   r   r0   r   r   r1   r   r   r   r   r   r   r   r   r  )vit_small_patch16_rope_ape_224r
  r  s       ra   r  r  *
  sa     J" nZnSWXbSmflSmnELrb   c                 h    t        dddddddddddddd	d
      }t        dd| it        |fi |}|S )zCRoPE-Axial + APE ViT-B/16 from https://github.com/naver-ai/rope-vitr   r   r   rq   rj   TrC  r   rQ  ry  r  r  )vit_base_patch16_rope_ape_224r
  r  s       ra   r  r  B
  sa     J$ mJmRVWaRlekRlmELrb   c                 h    t        dddddddddddddd	d
      }t        dd| it        |fi |}|S )zCRoPE-Axial + APE ViT-L/16 from https://github.com/naver-ai/rope-vitr   r  r&  rq   rj   TrC  r   rQ  ry  r  r  )vit_large_patch16_rope_ape_224r
  r  s       ra   r  r  [
  sa     J$ nZnSWXbSmflSmnELrb   c           	          t        di ddddddddd	d
dddddddddddddddddddddd}t        dd| it        |fi |}|S ) zCRoPE-Mixed + APE ViT-S/16 from https://github.com/naver-ai/rope-vitr   r   r   r   r   r   r0   r  r   rq   r   rj   r1   Tr   rC  r   r   r   r   r   r   r   rQ  r   r  r   r	  r  r   )$vit_small_patch16_rope_mixed_ape_224r
  r  s       ra   r  r  t
  s        	
              !J& t:tY]^hYslrYstELrb   c           	          t        di dddddddddd	d
ddddddddddddddddddddd}t        dd| it        |fi |}|S )zCRoPE-Mixed + APE ViT-B/16 from https://github.com/naver-ai/rope-vitr   r   r   r   r   r   r0   r   rq   r   rj   r1   Tr   rC  r   r   r   r   r   r   r   rQ  r   r  r   r	  r  r   )#vit_base_patch16_rope_mixed_ape_224r
  r  s       ra   r  r  
  s        	
              !J$ s*sX\]gXrkqXrsELrb   c           	          t        di dddddddddd	d
ddddddddddddddddddddd}t        dd| it        |fi |}|S )zCRoPE-Mixed + APE ViT-L/16 from https://github.com/naver-ai/rope-vitr   r   r   r  r   r&  r0   r   rq   r   rj   r1   Tr   rC  r   r   r   r   r   r   r   rQ  r   r  r   r	  r  r   )$vit_large_patch16_rope_mixed_ape_224r
  r  s       ra   r  r  
  s        	
              !J$ t:tY]^hYslrYstELrb   c                     t        dddddddddddd	d
d	t        t        d            }t        dd| it        |fi |}|S )z&DINOv3-style T/16 w/ QKV bias enabled.r   Tr  r   rA   rC  dinov3d   Frq   rR  r   r   r   r   r0   r1   r   r   r   r   r   r   r   r   r8   r  )vit_tiny_patch16_dinov3_qkvbrV  r  s       ra   r  r  
  sh     9$/#J& l:lQUV`QkdjQklELrb   c                     t        ddddddddd	dddd
dt        t        d            }t        dd| it        |fi |}|S )zDINOv3 S/16 https://arxiv.org/abs/2508.10104
    NOTE: Pass global_pool='token' to use CLS-token pooling (matches upstream DINOv3).
    r   Tr   r   r  FrC  r  r  rq   rR  r  r  )vit_small_patch16_dinov3rV  r  s       ra   r  r  
  sg    
 9$/#J& hzhTR\Mg`fMghELrb   c                     t        dddddddddddd	d
d	t        t        d            }t        dd| it        |fi |}|S )zDINOv3 S/16 w/ QKV bias enabled (but zero) https://arxiv.org/abs/2508.10104
    NOTE: Pass global_pool='token' to use CLS-token pooling (matches upstream DINOv3).
    r   Tr   r   r  rC  r  r  Frq   rR  r  r  )vit_small_patch16_dinov3_qkvbrV  r  s       ra   r  r  
  sh    
 9$/#J& mJmRVWaRlekRlmELrb   c           	          t        di ddddddddd	d
dddddddddddddddddddddddt        t        d      }t        d d| it        |fi |}|S )!zDINOv3 S/16 Plus https://arxiv.org/abs/2508.10104
    NOTE: Pass global_pool='token' to use CLS-token pooling (matches upstream DINOv3).
    r   r   r   Tr   r   r   r   r0   r  r1   Fr   rC  r   r  r   r  r   r   r   r   r   r   r   rq   r   r8   rR  r  r   )vit_small_plus_patch16_dinov3rV  r  s       ra   r  r    s    
    	
           !" #$ %& 9$/'J* mJmRVWaRlekRlmELrb   c           	          t        di ddddddddd	d
dddddddddddddddddddddddt        t        d      }t        d d| it        |fi |}|S )!zDINOv3 S/16 Plus w/ QKV bias enabled (but 0) https://arxiv.org/abs/2508.10104
    NOTE: Pass global_pool='token' to use CLS-token pooling (matches upstream DINOv3).
    r   r   r   Tr   r   r   r   r0   r  r1   r   rC  r   r  r   r  r   r   r   Fr   r   r   r   rq   r   r8   rR  r  r   )"vit_small_plus_patch16_dinov3_qkvbrV  r  s       ra   r  r  0  s    
    	
           !" #$ %& 9$/'J* rrW[\fWqjpWqrELrb   c                     t        ddddddddddddd	dt        t        d
            }t        dd| it        |fi |}|S )zDINOv3 B/16 https://arxiv.org/abs/2508.10104
    NOTE: Pass global_pool='token' to use CLS-token pooling (matches upstream DINOv3).
    r   Tr   r   FrC  r  r  rq   rR  r  r  )vit_base_patch16_dinov3rV  r  s       ra   r  r  N  sg    
 9$/#J& gjgDQ[Lf_eLfgELrb   c                     t        ddddddddddddd	dt        t        d
            }t        dd| it        |fi |}|S )zDINOv3 B/16 w/ QKV bias enabled (but zero) https://arxiv.org/abs/2508.10104
    NOTE: Pass global_pool='token' to use CLS-token pooling (matches upstream DINOv3).
    r   Tr   r   rC  r  r  Frq   rR  r  r  )vit_base_patch16_dinov3_qkvbrV  r  s       ra   r  r  j  sh    
 9$/#J& l:lQUV`QkdjQklELrb   c                     t        ddddddddddddd	dt        t        d
            }t        dd| it        |fi |}|S )zDINOv3 L/16 https://arxiv.org/abs/2508.10104
    NOTE: Pass global_pool='token' to use CLS-token pooling (matches upstream DINOv3).
    r   Tr  r&  FrC  r  r  rq   rR  r   r   r   r   r0   r1   r   r   r   r   r   r   r   r   r8   r  )vit_large_patch16_dinov3rV  r  s       ra   r  r    sg    
 9$/#J& hzhTR\Mg`fMghELrb   c                     t        ddddddddddddd	dt        t        d
            }t        dd| it        |fi |}|S )zDINOv3 w/ QKV bias enabled (but zero) https://arxiv.org/abs/2508.10104
    NOTE: Pass global_pool='token' to use CLS-token pooling (matches upstream DINOv3).
    r   Tr  r&  rC  r  r  Frq   rR  r  r  )vit_large_patch16_dinov3_qkvbrV  r  s       ra   r  r    sh    
 9$/#J& mJmRVWaRlekRlmELrb   c           	          t        di ddddddddd	d
dddddddddddddddddddddddt        t        d      }t        d d| it        |fi |}|S )!zDINOv3 H/16 Plus https://arxiv.org/abs/2508.10104
    NOTE: Pass global_pool='token' to use CLS-token pooling (matches upstream DINOv3).
    r   r   r   Tr   r  r   rc  r0   rG  r1   Fr   rC  r   r  r   r  r   r   r   r   r   r   r   rq   r   r8   rR  r  r   )vit_huge_plus_patch16_dinov3rV  r  s       ra   r  r    s    
    	
          " #$ %& 9$/'J, l:lQUV`QkdjQklELrb   c           	          t        di ddddddddd	d
dddddddddddddddddddddddt        t        d      }t        d d| it        |fi |}|S )!zDINOv3 H/16 Plus w/ QKV bias enabled (but zero) https://arxiv.org/abs/2508.10104
    NOTE: Pass global_pool='token' to use CLS-token pooling (matches upstream DINOv3).
    r   r   r   Tr   r  r   rc  r0   rG  r1   r   rC  r   r  r   r  r   r   Fr   r   r   r   r   rq   r   r8   rR  r  r   )!vit_huge_plus_patch16_dinov3_qkvbrV  r  s       ra   r  r    s    
    	
          " #$ %& 9$/'J, q
qVZ[eVpioVpqELrb   c           	          t        d!i ddddddddd	d
dddddddddddddddddddddddddt        t        d      }t        d"d | it        |fi |}|S )#zDINOv3 7B/16 https://arxiv.org/abs/2508.10104
    NOTE: Pass global_pool='token' to use CLS-token pooling (matches upstream DINOv3).
    r   r   r   Tr   i   r   r  r0   rc  r1   Fr   rp   r   rC  r   r  r   r  r   r   r   r   r   r?  r   rq   r   r8   rR  r  r   )vit_7b_patch16_dinov3rV  r  s       ra   r  r    s    
    	
            !$ %& '( 9$/)J. eJe$zJd]cJdeELrb   )r  )r  Tr  )r   )r   rB  r  	functoolsr   typingr   r   r   r   r   r	   r
   r   rP   torch.nnrI   torch.nn.functional
functionalrz   	timm.datar   r   r   r   timm.layersr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   _builderr&   	_featuresr'   _manipulater(   	_registryr)   r*   __all__r  r-   r   r   r+   r   r   r  r   r  r  r  r  r  r  default_cfgsr	  r  r  r  r  r"  r(  r+  r-  r/  r1  r3  r6  r:  r=  r@  rE  rH  rK  rM  rU  rY  r[  r]  rd  rh  rl  ro  rq  rs  ru  rw  r{  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   rb   ra   <module>r     s  =@  	  I I I     d d     , + + # <'f299 fRCryy CLsryy sly")) y~ CS%,,&'C99C C 
#u||
	CR '	|ell*+|yy| | 	|
 
#u||
|~ $ S >c T#s(^ ( S#X (S DcN 0	3 	$sCx. 	 % [& )$+[& )$ 3(+D[& /"(< 3(	1D[&$ /"(< 3(	1D%[&2 5d 3(73[&< 6t 3(8=[&F 5d 3(7G[&T / 31U[&^ 0 32_[&h / 31i[&r 0 32s[&| / 31}[&J 0 3(PU2K[&T 1$ 3(PU3U[&^ 0 3(PU2_[&l ')m[&v (*w[&@ ')A[&J (*K[&T ')U[&b +D -c[&p *4 ,q[&~ +D -[&L ,T .M[&Z ,T  3.[[&j .t 0k[&x 3D 5y[&F /1G[&P 4T 4/6Q[&Z 5d 4/7[[&d 5d 47e[&l 24 4/4m[&z &w !({[&H ' !)I[&V &w !(W[&d ' !)e[&r *7 !,s[&B ' !)C[&P .w !0Q[&^ *7 !,_[&l 1' !3m[&| )' !+}[&J *7 !,K[&X )' !+Y[&f *7 !,g[&t -g !/u[&F	 ,T"(<.G	[&P	 +D"(<-Q	[&Z	 ,T"(<.[	[&d	 24"(<4e	[&n	 1$"(<3o	[&x	 24"(<4y	[&B
 0"(<2C
[&L
 /"(<1M
[&V
 0"(<2W
[&`
 6t"(<8a
[&j
 5d"(<7k
[&t
 6t"(<8u
[&H (*I[&N -k/O[&T -k/U[&Z 2;4[[&` ')a[&f ,[.g[&l 1)3m[&r 294s[&x 1)3y[&~ (*[&D /:/ )4"(=) .9"(=. .9. 3>3 '2' &1"(=&m[& [| d    d    d    t #  "  3  " t #  &  3  & t #  "  3  " t #  &  3  & 4 c   D s  * T   * T   *  3    T PS  * d QT  * d QT  * 4 c  * lT l l l0 md m m m. lT l l l. md m m m. p pC p p. md m m m0 p pC p p. o o3 o o, p pC p p, o o3 o o, p pC p p, sD ss s s0 4 c  , $ S  . 4 c  ,  C  .  3  .  C  0 t #  . d   0 t #  0 T PS  2 D s  0 T PS  0 T   2  C  6 d   6 d   : 4 c  :  3  6 T   6  C  6 d   6 T   < $ S  : d   rb   