
    ^jv                        d Z ddlZddlZddlmZ ddlmZmZmZm	Z	m
Z
 	 ddlmZ ddlZddlmZ ddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZ d	d
l m!Z! d	dl"m#Z# d	dl$m%Z%m&Z& d	dl'm(Z(m)Z) d	dl*m+Z+ dgZ, ejZ                  e.      Z/ G d dej`                        Z1 G d dej`                        Z2 G d dej`                        Z3 G d dej`                        Z4d8dZ5d9dZ6 e( e6ddd       e6dd       e6d d!       e6d"d!       e6d#d!       e6d$d!       e6d%d!       e6d&d!       e6        e6d'd!       e6        e6d(d!       e6       d)      Z7e)d8d*e4fd+       Z8e)d8d*e4fd,       Z9e)d8d*e4fd-       Z:e)d8d*e4fd.       Z;e)d8d*e4fd/       Z<e)d8d*e4fd0       Z=e)d8d*e4fd1       Z>e)d8d*e4fd2       Z?e)d8d*e4fd3       Z@e)d8d*e4fd4       ZAe)d8d*e4fd5       ZBe)d8d*e4fd6       ZCe)d8d*e4fd7       ZDy# e$ r
 ddlmZ Y w xY w):z Relative Position Vision Transformer (ViT) in PyTorch

NOTE: these models are experimental / WIP, expect changes

Hacked together by / Copyright 2022, Ross Wightman
    N)partial)ListOptionalTupleTypeUnion)Literal)FinalIMAGENET_INCEPTION_MEANIMAGENET_INCEPTION_STD)	
PatchEmbedMlp
LayerScaleDropPathcalculate_drop_path_rates	RelPosMlp
RelPosBiasuse_fused_attn	LayerType   )build_model_with_cfg)feature_take_indices)named_apply
checkpoint)generate_default_cfgsregister_model)get_init_weights_vitVisionTransformerRelPosc                        e Zd ZU ee   ed<   ddddddej                  ddf	dededed	ed
e	e
ej                        dedede
ej                     f fdZdde	ej                     fdZ xZS )RelPosAttention
fused_attn   FN        dim	num_headsqkv_biasqk_normrel_pos_cls	attn_drop	proj_drop
norm_layerc                 r   |	|
d}t         |           ||z  dk(  sJ d       || _        ||z  | _        | j                  dz  | _        t               | _        t        j                  ||dz  fd|i|| _	        |r || j                  fi |nt        j                         | _        |r || j                  fi |nt        j                         | _        |r
 |dd|i|nd | _        t        j                  |      | _        t        j                  ||fi || _        t        j                  |      | _        y )	Ndevicedtyper   z$dim should be divisible by num_headsg         biasr&    )super__init__r&   head_dimscaler   r"   nnLinearqkvIdentityq_normk_normrel_posDropoutr*   projr+   )selfr%   r&   r'   r(   r)   r*   r+   r,   r/   r0   dd	__class__s               p/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/vision_transformer_relpos.pyr5   zRelPosAttention.__init__/   s    /Y!#K%KK#"y(]]d*
(*99S#'??B?9@j5"5bkkm9@j5"5bkkmAL{=Y="=RVI.IIc3-"-	I.    shared_rel_posc                    |j                   \  }}}| j                  |      j                  ||d| j                  | j                        j                  ddddd      }|j                  d      \  }}}	| j                  |      }| j                  |      }| j                  r| j                  | j                  j                         }
n||}
nd }
t        j                  j                  j                  |||	|
| j                   r| j"                  j$                  nd      }ns|| j&                  z  }||j)                  dd	      z  }| j                  | j                  ||
      }n|||z   }|j+                  d	      }| j#                  |      }||	z  }|j)                  dd      j                  |||      }| j-                  |      }| j/                  |      }|S )Nr1      r   r      r$   )	attn_mask	dropout_prF   r%   )shaper:   reshaper&   r6   permuteunbindr<   r=   r"   r>   get_biastorchr8   
functionalscaled_dot_product_attentiontrainingr*   pr7   	transposesoftmaxr@   r+   )rA   xrF   BNCr:   qkv	attn_biasattns               rD   forwardzRelPosAttention.forwardL   s   ''1ahhqk!!!Q4>>4==IQQRSUVXY[\^_`**Q-1aKKNKKN??||' LL113	+*	 	##@@1a#.2mm$..** A A DJJAq{{2r**D||'||D|H+n,<<B<'D>>$'DqAKK1%%aA.IIaLNN1rE   N)__name__
__module____qualname__r
   bool__annotations__r8   	LayerNormintr   r   Modulefloatr5   rU   Tensorre   __classcell__rC   s   @rD   r!   r!   ,   s    d
 "!59!!*,,,// / 	/
 / "$ryy/2/ / / RYY/:"%,,)? "rE   r!   c                       e Zd Zddddddddej                  ej
                  ddfdedededed	ed
e	e
ej                        de	e   dededede
ej                     de
ej                     f fdZdde	ej                     fdZ xZS )RelPosBlock      @FNr$   r%   r&   	mlp_ratior'   r(   r)   init_valuesr+   r*   	drop_path	act_layerr,   c           
      ,   ||d}t         |            ||fi || _        t        ||f||||	||d|| _        |rt        |fd|i|nt        j                         | _        |
dkD  rt        |
      nt        j                         | _
         ||fi || _        t        d|t        ||z        ||d|| _        |rt        |fd|i|nt        j                         | _        |
dkD  rt        |
      | _        y t        j                         | _        y )Nr.   r'   r(   r)   r*   r+   r,   rw   r$   in_featureshidden_featuresry   dropr3   )r4   r5   norm1r!   rd   r   r8   r;   ls1r   
drop_path1norm2r   rm   mlpls2
drop_path2rA   r%   r&   rv   r'   r(   r)   rw   r+   r*   rx   ry   r,   r/   r0   rB   rC   s                   rD   r5   zRelPosBlock.__init__s   s)   " /*r*
#

 #!

 

	 FQ:cA{AbAVXVaVaVc1:R(9-R[[]*r*
 
i0	

 
 FQ:cA{AbAVXVaVaVc1:R(9-R[[]rE   rF   c           
         || j                  | j                  | j                  | j                  |      |                  z   }|| j	                  | j                  | j                  | j                  |                        z   }|S NrN   )r   r   rd   r   r   r   r   r   rA   r\   rF   s      rD   re   zRelPosBlock.forward   sf    4::a=Q_)` abb$**Q-)@ ABBrE   rf   )rg   rh   ri   r8   GELUrl   rm   ro   rj   r   r   rn   r5   rU   rp   re   rq   rr   s   @rD   rt   rt   q   s      ""!59+/!!!)+*,,,,S,S ,S 	,S
 ,S ,S "$ryy/2,S "%,S ,S ,S ,S BII,S RYY,S\%,,)? rE   rt   c                       e Zd Zddddddddej                  ej
                  ddfdedededed	ed
e	e
ej                        de	e   dededede
ej                     de
ej                     f fdZd Zdde	ej                     fdZ xZS )ResPostRelPosBlockru   FNr$   r%   r&   rv   r'   r(   r)   rw   r+   r*   rx   ry   r,   c           
         ||d}t         |           || _        t        ||f||||	||d|| _         ||fi || _        |
dkD  rt        |
      nt        j                         | _	        t        d|t        ||z        ||d|| _         ||fi || _        |
dkD  rt        |
      nt        j                         | _        | j                          y )Nr.   r{   r$   r|   r3   )r4   r5   rw   r!   rd   r   r   r8   r;   r   r   rm   r   r   r   init_weightsr   s                   rD   r5   zResPostRelPosBlock.__init__   s    " /&#

 #!

 

	  *r*
1:R(9-R[[] 
i0	

 
  *r*
1:R(9-R[[]rE   c                    | j                   }t        j                  j                  | j                  j
                  | j                          t        j                  j                  | j                  j
                  | j                          y y rf   )rw   r8   init	constant_r   weightr   rA   s    rD   r   zResPostRelPosBlock.init_weights   s[    'GGdjj//1A1ABGGdjj//1A1AB (rE   rF   c           	          || j                  | j                  | j                  ||                  z   }|| j                  | j	                  | j                  |                  z   }|S r   )r   r   rd   r   r   r   r   s      rD   re   zResPostRelPosBlock.forward   sT    

499Q~9+V WXX

488A; 788rE   rf   )rg   rh   ri   r8   r   rl   rm   ro   rj   r   r   rn   r5   r   rU   rp   re   rq   rr   s   @rD   r   r      s      ""!59+/!!!)+*,,,-- - 	-
 - - "$ryy/2- "%- - - - BII- RYY-^C%,,)? rE   r   c            8           e Zd ZdZddddddddd	d
dddddddddddddeddeddfdeeeeef   f   deeeeef   f   dedede	d   dededede
dededee
   deded ed!ee   d"ed#e
d$e
d%e
d&e
d'e	d(   d)ed*eej                      d+ee   d,ee   d-eej                      f6 fd.ZdHd0ZdId1ed2ed/dfd3Zej,                  j.                  d4        Zej,                  j.                  dJd5       Zej,                  j.                  dKd6       Zej,                  j.                  d/ej                   fd7       ZdLdedee   fd8Z	 	 	 	 	 	 dMd9ej:                  d:eeeee   f      d;ed<ed=ed>ed?ed/eeej:                     eej:                  eej:                     f   f   fd@Z	 	 	 dNd:eeee   f   dAedBefdCZ dD Z!dJdEefdFZ"dG Z# xZ$S )Or   ah   Vision Transformer w/ Relative Position Bias

    Differing from classic vit, this impl
      * uses relative position index (swin v1 / beit) or relative log coord + mlp (swin v2) pos embed
      * defaults to no class token (can be enabled)
      * defaults to global avg pool for head (can be changed)
      * layer-scale (residual branch gain) enabled
          r1     avg      ru   TFư>r   Nr$   resetimg_size
patch_sizein_chansnum_classesglobal_pool) r   tokenmap	embed_dimdepthr&   rv   r'   r(   rw   class_tokenfc_normrel_pos_typerel_pos_dimrF   	drop_rateproj_drop_rateattn_drop_ratedrop_path_rateweight_init)skipr   jaxmocor   fix_initembed_layerr,   ry   block_fnc                    t         %|           ||d}|dv sJ |s|dk7  sJ |xs t        t        j                  d      }|xs t        j
                  }|| _        || _        || _        |x| _	        x| _
        | _        |rdnd| _        d| _         |d||||d	|| _        | j                  j                  }t!        | j                  d
      r| j                  j#                         n|} t%        || j                        }!|j'                  d      r!|r||!d<   d|v rd|!d<   t        t(        fi |!}"nt        t*        fi |!}"d| _        |r |"dd|i|| _        d}"|r5t        j.                  t1        j2                  d| j                  |fi |      nd| _        t7        ||      }#t        j8                  t;        |      D $cg c]  }$ |d|||	|
||"||||#|$   ||d| c}$      | _        t;        |      D $cg c]  }$t%        d|$ ||        c}$| _        |s	 ||fi |nt        j@                         | _!        |r	 ||fi |nt        j@                         | _"        t        jF                  |      | _$        |dkD  r!t        jJ                  | j                  |fi |nt        j@                         | _&        |dk(  rdn|| _'        || _(        |dk7  r| jS                  d       yyc c}$w c c}$w )aE  
        Args:
            img_size: input image size
            patch_size: patch size
            in_chans: number of input channels
            num_classes: number of classes for classification head
            global_pool: type of global pooling for final sequence (default: 'avg')
            embed_dim: embedding dimension
            depth: depth of transformer
            num_heads: number of attention heads
            mlp_ratio: ratio of mlp hidden dim to embedding dim
            qkv_bias: enable bias for qkv if True
            qk_norm: Enable normalization of query and key in attention
            init_values: layer-scale init values
            class_token: use class token (default: False)
            fc_norm: use pre classifier norm instead of pre-pool
            rel_pos_type: type of relative position
            shared_rel_pos: share relative pos across all blocks
            drop_rate: dropout rate
            proj_drop_rate: projection dropout rate
            attn_drop_rate: attention dropout rate
            drop_path_rate: stochastic depth rate
            weight_init: weight init scheme
            fix_init: apply weight initialization fix (scaling w/ layer index)
            embed_layer: patch embedding layer
            norm_layer: normalization layer
            act_layer: MLP activation layer
        r.   r   r   r   r   r   )epsr   r   F)r   r   r   r   
feat_ratio)window_sizeprefix_tokensr   
hidden_dimswinmodeNr&   )r%   r&   rv   r'   r(   r)   rw   r+   r*   rx   r,   ry   zblocks.)modulenum_chs	reductionr   r   needs_resetr3   )*r4   r5   r   r8   rl   r   r   r   r   num_featureshead_hidden_sizer   num_prefix_tokensgrad_checkpointingpatch_embed	grid_sizehasattrr   dict
startswithr   r   rF   	ParameterrU   zeros	cls_tokenr   
ModuleListrangeblocksfeature_infor;   normr   r?   	head_dropr9   headweight_init_moder   r   )&rA   r   r   r   r   r   r   r   r&   rv   r'   r(   rw   r   r   r   r   rF   r   r   r   r   r   r   r   r,   ry   r   r/   r0   rB   	feat_sizerrel_pos_argsr)   dprirC   s&                                        rD   r5   z VisionTransformerRelPos.__init__   s   x 	/2222kW444B72<<T#B
(	& &ENNND1DN&1q"'& 
!	

 
 $$..	-4T5E5E|-TD'')Zd	AWAWX""5)-8\*%'-V$!)<|<K!*==K""-"H	"HR"HDKbmekk!T5K5KY&]Z\&]^sw'>mm  5\!%#    ##!''((a&%# %# $$ QVV[P\^KLD'!yAF^7>Jy/B/BKKM	 7>z)2r22;;=I.DORSOBIIdnnk@R@Y[YdYdYf	+6&+@k & %0 !;%#"^s   K->K2returnc                    t        j                         5  t        | j                        D ]~  \  }}t	        j
                  d|dz   z        }|j                  j                  j                  j                  |       |j                  j                  j                  j                  |        	 ddd       y# 1 sw Y   yxY w)z9Apply weight initialization fix (scaling w/ layer index).g       @r   N)rU   no_grad	enumerater   mathsqrtrd   r@   r   div_r   fc2)rA   layer_idlayerr7   s       rD   fix_init_weightz'VisionTransformerRelPos.fix_init_weightt  s    ]]_ 	1#,T[[#9 1%		#A"67

&&++E2		$$))%01	1 	1 	1s   BB66B?r   r   c                 T   |xs | j                   }|dv sJ d|v r t        j                  | j                         nd}| j                  +t
        j                  j                  | j                  d       t        t        |||      |        | j                  r| j                          yy)aP  Initialize model weights.

        Args:
            mode: Weight initialization mode ('jax', 'jax_nlhb', 'moco', or '').
            needs_reset: If True, call reset_parameters() on modules (default for after to_empty()).
                If False, skip reset_parameters() (for __init__ where modules already self-initialized).
        )r   jax_nlhbr   r   r   nlhbr$   Nr   )stdr   )r   r   logr   r   r8   r   normal_r   r   r   r   )rA   r   r   	head_biass       rD   r   z$VisionTransformerRelPos.init_weights|  s     ,t,,????39T>TXXd..//r	>>%GGOODNNO5(ykRTXY==  " rE   c                     dhS )Nr   r3   r   s    rD   no_weight_decayz'VisionTransformerRelPos.no_weight_decay  s
    }rE   c                      t        dddg      S )Nz^cls_token|patch_embed)z^blocks\.(\d+)N)z^norm)i )stemr   )r   )rA   coarses     rD   group_matcherz%VisionTransformerRelPos.group_matcher  s    *-/CD
 	
rE   c                     || _         y rf   )r   )rA   enables     rD   set_grad_checkpointingz.VisionTransformerRelPos.set_grad_checkpointing  s
    "(rE   c                     | j                   S rf   )r   r   s    rD   get_classifierz&VisionTransformerRelPos.get_classifier  s    yyrE   c                     ||d}|| _         ||dv sJ || _        |dkD  r't        j                  | j                  |fi || _        y t        j
                         | _        y )Nr.   r   r   )r   r   r8   r9   r   r;   r   )rA   r   r   r/   r0   rB   s         rD   reset_classifierz(VisionTransformerRelPos.reset_classifier  sc    /&""6666*DDORSOBIIdnnk@R@	Y[YdYdYf	rE   r\   indicesreturn_prefix_tokensr   
stop_early
output_fmtintermediates_onlyc           	      4   |dv sJ d       |dk(  }g }	t        t        | j                        |      \  }
}|j                  \  }}}}| j	                  |      }| j
                  At        j                  | j
                  j                  |j                  d   dd      |fd      }| j                  | j                  j                         nd}t        j                  j                         s|s| j                  }n| j                  d|dz    }t        |      D ]q  \  }}| j                  r-t        j                  j                         st        |||	      }n
 |||	      }||
v sN|	j!                  |r| j#                  |      n|       s | j$                  rD|	D cg c]  }|ddd| j$                  f    }}|	D cg c]  }|dd| j$                  df    }	}|ra| j                  j'                  ||f      \  }}|	D cg c]6  }|j)                  |||d      j+                  dd
dd      j-                         8 }	}t        j                  j                         s|rt/        t1        |	            }	|r|	S | j#                  |      }||	fS c c}w c c}w c c}w )a=   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            return_prefix_tokens: Return both prefix and spatial intermediate tokens
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )NCHWNLCz)Output format must be one of NCHW or NLC.r  Nr   rM   r   rO   rN   r1   rH   )r   lenr   rP   r   r   rU   catexpandrF   rT   jitis_scriptingr   r   r   appendr   r   dynamic_feat_sizerQ   rR   
contiguouslistzip)rA   r\   r   r   r   r   r  r  rQ   intermediatestake_indices	max_indexr]   _heightwidthrF   r   r   blkyr   HWs                           rD   forward_intermediatesz-VisionTransformerRelPos.forward_intermediates  si   . _,Y.YY,&"6s4;;7G"Qi  gg1feQ>>%		4>>00RDaHaPA;?;N;N;Z,,557`d99!!#:[[F[[)a-0F' 	BFAs&&uyy/E/E/GsAnE.9L $$TTYYq\qA	B !!ERSQq!D$:$:"::;SMSDQRqQq$"8"8"99:RMR##55vuoFDAq^klYZQYYq!Q3;;Aq!QGRRTlMlyy%%',@ ]M!BCM  IIaL- TR ms   J<J ;J
prune_norm
prune_headc                    t        t        | j                        |      \  }}| j                  d|dz    | _        |rt        j                         | _        |r+t        j                         | _        | j                  dd       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r   )r   r  r   r8   r;   r   r   r   )rA   r   r  r  r  r  s         rD   prune_intermediate_layersz1VisionTransformerRelPos.prune_intermediate_layers  sj     #7s4;;7G"Qikk.9q=1DI;;=DL!!!R(rE   c                    | j                  |      }| j                  At        j                  | j                  j	                  |j
                  d   dd      |fd      }| j                  | j                  j                         nd }| j                  D ]E  }| j                  r-t        j                  j                         st        |||      }< |||      }G | j                  |      }|S )Nr   rM   r   rO   rN   )r   r   rU   r  r  rP   rF   rT   r   r   r	  r
  r   r   )rA   r\   rF   r  s       rD   forward_featuresz(VisionTransformerRelPos.forward_features  s    Q>>%		4>>00RDaHaPA;?;N;N;Z,,557`d;; 	:C&&uyy/E/E/GsAnE.9		:
 IIaLrE   
pre_logitsc                    | j                   r=| j                   dk(  r%|d d | j                  d f   j                  d      n|d d df   }| j                  |      }| j	                  |      }|r|S | j                  |      S )Nr   r   rO   r   )r   r   meanr   r   r   )rA   r\   r!  s      rD   forward_headz$VisionTransformerRelPos.forward_head  s~    =A=M=MQV=V!T++,,-22q29\]^_ab^b\cALLONN1q0DIIaL0rE   c                 J    | j                  |      }| j                  |      }|S rf   )r   r$  )rA   r\   s     rD   re   zVisionTransformerRelPos.forward  s'    !!!$a rE   )r   N)r   TF)T)NNN)NFFFr  F)r   FT)%rg   rh   ri   __doc__r   rt   r   rm   r   r	   ro   rj   r   strr   r8   rn   r   r5   r   r   rU   r	  ignorer   r   r   r   r   rp   r   r  r  r   r$  re   rq   rr   s   @rD   r   r      s    5868#>C !!!+/ %! %)-#(!$&$&$&GN"+5.2-1(3=D1CsCx01D1 c5c?23D1 	D1
 D1 !!:;D1 D1 D1 D1 D1 D1 D1 "%D1 D1 D1  !D1" "##D1$ !%D1& 'D1( ")D1* "+D1, "-D1. !!CD/D10 1D12 bii3D14 !+5D16  	*7D18 299o9D1L1# # # #& YY  YY
 
 YY) ) YY		  gC ghsm g 8<).$$',B ||B  eCcN34B  #'	B 
 B  B  B  !%B  
tELL!5tELL7I)I#JJ	KB L ./$#	3S	>*  	"1$ 1rE   c                 h    |j                  dd      }t        t        | |fdt        |d      i|}|S )Nout_indicesr1   feature_cfggetter)r+  feature_cls)popr   r   r   )variant
pretrainedkwargsr+  models        rD   !_create_vision_transformer_relposr4    sC    **]A.K *[hG E
 LrE   r   c                 4    | ddd dddt         t        dddd	|S )
Nr   )r1   r   r   g?bicubicTzpatch_embed.projr   z
apache-2.0)urlr   
input_size	pool_sizecrop_pctinterpolationfixed_input_sizer#  r   
first_conv
classifierlicenser   )r7  r2  s     rD   _cfgr@  $  s5    =t'0F(  rE   zhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_replos_base_patch32_plus_rpn_256-sw-dd486f51.pthztimm/)r1      rA  )r7  	hf_hub_idr8  )r1      rC  )r7  r8  zhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_relpos_small_patch16_224-sw-ec2778b4.pth)r7  rB  zhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_relpos_medium_patch16_224-sw-11c174af.pthzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_relpos_base_patch16_224-sw-49049aed.pthzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_srelpos_small_patch16_224-sw-6cdb8849.pthzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_srelpos_medium_patch16_224-sw-ad702b8c.pthzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_relpos_medium_patch16_cls_224-sw-cfe8e259.pthzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_relpos_base_patch16_gapcls_224-sw-1a341d6c.pthzhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_relpos_medium_patch16_rpn_224-sw-5d2befd8.pth)z,vit_relpos_base_patch32_plus_rpn_256.sw_in1kz*vit_relpos_base_patch16_plus_240.untrainedz$vit_relpos_small_patch16_224.sw_in1kz%vit_relpos_medium_patch16_224.sw_in1kz#vit_relpos_base_patch16_224.sw_in1kz%vit_srelpos_small_patch16_224.sw_in1kz&vit_srelpos_medium_patch16_224.sw_in1kz)vit_relpos_medium_patch16_cls_224.sw_in1kz)vit_relpos_base_patch16_cls_224.untrainedz*vit_relpos_base_patch16_clsgap_224.sw_in1kz*vit_relpos_small_patch16_rpn_224.untrainedz)vit_relpos_medium_patch16_rpn_224.sw_in1kz)vit_relpos_base_patch16_rpn_224.untrainedr   c           	      ^    t        ddddt              }t        	 dd| it        |fi |}|S )z` ViT-Base (ViT-B/32+) w/ relative log-coord position and residual post-norm, no class token
          r      )r   r   r   r&   r   r1  )$vit_relpos_base_patch32_plus_rpn_256r   r   r4  r1  r2  
model_argsr3  s       rD   rH  rH  X  sG     s"UghJ-.e;EeIMjIc\bIceELrE   c           	      T    t        dddd      }t        	 dd| it        |fi |}|S )zI ViT-Base (ViT-B/16+) w/ relative log-coord position, no class token
    r   rF  r   rG  )r   r   r   r&   r1  ) vit_relpos_base_patch16_plus_240r   r4  rJ  s       rD   rM  rM  b  sD     s"KJ-*a7AaEI*E_X^E_aELrE   c           	      X    t        dddddd      }t        	 d	d| it        |fi |}|S )
H ViT-Base (ViT-B/16) w/ relative log-coord position, no class token
    r     r      FTr   r   r   r&   r'   r   r1  )vit_relpos_small_patch16_224rN  rJ  s       rD   rT  rT  l  sJ     s"TYcghJ-&]3=]AEjA[TZA[]ELrE   c           	      X    t        dddddd      }t        	 d	d| it        |fi |}|S )
rP  r      r   r#   FTrS  r1  )vit_relpos_medium_patch16_224rN  rJ  s       rD   rW  rW  v  sM     B!eUY[J-'^4>^BFzB\U[B\^ELrE   c           	      X    t        dddddd      }t        	 dd| it        |fi |}|S )	rP  r   r   r   FTrS  r1  )vit_relpos_base_patch16_224rN  rJ  s       rD   rY  rY    sM     B"uVZ\J-%\2<\@DZ@ZSY@Z\ELrE   c           
      \    t        dddddddd      }t        	 d	d| it        |fi |}|S )
O ViT-Base (ViT-B/16) w/ shared relative log-coord position, no class token
    r   rQ  r   rR  FTr   r   r   r&   r'   r   r   rF   r1  )vit_srelpos_small_patch16_224rN  rJ  s       rD   r]  r]    sS     B!eUZ.J .'^4>^BFzB\U[B\^ELrE   c           
      \    t        dddddddd      }t        	 d	d| it        |fi |}|S )
r[  r   rV  r   r#   FTr\  r1  )vit_srelpos_medium_patch16_224rN  rJ  s       rD   r_  r_    sS     B!eUZ.J .(_5?_CG
C]V\C]_ELrE   c                 ^    t        ddddddddd		      }t        	 dd
| it        |fi |}|S )zM ViT-Base (ViT-M/16) w/ relative log-coord position, class token present
    r   rV  r   r#   FrA  Tr   )	r   r   r   r&   r'   r   r   r   r   r1  )!vit_relpos_medium_patch16_cls_224rN  rJ  s       rD   ra  ra    sV     B!eUZTw@J .+b8BbFJ:F`Y_F`bELrE   c           	      Z    t        ddddddd      }t        	 d	d| it        |fi |}|S )
zM ViT-Base (ViT-B/16) w/ relative log-coord position, class token present
    r   r   r   FTr   )r   r   r   r&   r'   r   r   r1  )vit_relpos_base_patch16_cls_224rN  rJ  s       rD   rc  rc    sP     B"uZ^lsuJ-)`6@`DHD^W]D^`ELrE   c           	      Z    t        ddddddd      }t        	 dd| it        |fi |}|S )	a   ViT-Base (ViT-B/16) w/ relative log-coord position, class token present
    NOTE this config is a bit of a mistake, class token was enabled but global avg-pool w/ fc-norm was not disabled
    Leaving here for comparisons w/ a future re-train as it performs quite well.
    r   r   r   FT)r   r   r   r&   r'   r   r   r1  )"vit_relpos_base_patch16_clsgap_224rN  rJ  s       rD   re  re    sP     B"uVZhlnJ-,c9CcGKJGaZ`GacELrE   c           	      `    t        dddddt              }t        	 dd| it        |fi |}|S )	_ ViT-Base (ViT-B/16) w/ relative log-coord position and residual post-norm, no class token
    r   rQ  r   rR  Fr   r   r   r&   r'   r   r1  ) vit_relpos_small_patch16_rpn_224rI  rJ  s       rD   ri  ri    sM     B!eVhjJ-*a7AaEI*E_X^E_aELrE   c           	      `    t        dddddt              }t        	 dd| it        |fi |}|S )	rg  r   rV  r   r#   Frh  r1  )!vit_relpos_medium_patch16_rpn_224rI  rJ  s       rD   rk  rk    sM     B!eVhjJ-+b8BbFJ:F`Y_F`bELrE   c           	      `    t        dddddt              }t        	 dd| it        |fi |}|S )rg  r   r   r   Frh  r1  )vit_relpos_base_patch16_rpn_224rI  rJ  s       rD   rm  rm    sM     B"uWikJ-)`6@`DHD^W]D^`ELrE   r&  )r   )Er'  loggingr   	functoolsr   typingr   r   r   r   r   r	   ImportErrortyping_extensionsrU   torch.nnr8   	torch.jitr
   	timm.datar   r   timm.layersr   r   r   r   r   r   r   r   r   _builderr   	_featuresr   _manipulater   r   	_registryr   r   vision_transformerr   __all__	getLoggerrg   _loggerrn   r!   rt   r   r   r4  r@  default_cfgsrH  rM  rT  rW  rY  r]  r_  ra  rc  re  ri  rk  rm  r3   rE   rD   <module>r     s      5 5*    E
 
 
 + + 0 < 4$
%
'

H
%Bbii BJ3")) 3l: :zsbii sl		 %48 X 5" 372-2X,0 P- .2 Q. ,0 O, .2 Q. /3 R/ 26 U2 2626 V3 37&15 U2 26I%& %P H_   D[   @W   AX   ?V   AX   BY   E\   CZ   	F] 	 	 D[   E\   CZ  c  *))*s   G: :H	H	