Ë
    éÓ^jãv  ã                   ó  — d Z ddlZddlZddlmZ ddlmZmZmZm	Z	m
Z
 	 ddlmZ ddlZddlmZ ddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZ d	d
l m!Z! d	dl"m#Z# d	dl$m%Z%m&Z& d	dl'm(Z(m)Z) d	dl*m+Z+ dgZ, ejZ                  e.«      Z/ G d„ dej`                  «      Z1 G d„ dej`                  «      Z2 G d„ dej`                  «      Z3 G d„ dej`                  «      Z4d8d„Z5d9d„Z6 e( e6ddd¬«       e6dd¬«       e6d d¬!«       e6d"d¬!«       e6d#d¬!«       e6d$d¬!«       e6d%d¬!«       e6d&d¬!«       e6«        e6d'd¬!«       e6«        e6d(d¬!«       e6«       d)œ«      Z7e)d8d*e4fd+„«       Z8e)d8d*e4fd,„«       Z9e)d8d*e4fd-„«       Z:e)d8d*e4fd.„«       Z;e)d8d*e4fd/„«       Z<e)d8d*e4fd0„«       Z=e)d8d*e4fd1„«       Z>e)d8d*e4fd2„«       Z?e)d8d*e4fd3„«       Z@e)d8d*e4fd4„«       ZAe)d8d*e4fd5„«       ZBe)d8d*e4fd6„«       ZCe)d8d*e4fd7„«       ZDy# e$ r
 ddlmZ Y �Œâw xY w):z¦ Relative Position Vision Transformer (ViT) in PyTorch

NOTE: these models are experimental / WIP, expect changes

Hacked together by / Copyright 2022, Ross Wightman
é    N)Úpartial)ÚListÚOptionalÚTupleÚTypeÚUnion)ÚLiteral)ÚFinal©ÚIMAGENET_INCEPTION_MEANÚIMAGENET_INCEPTION_STD)	Ú
PatchEmbedÚMlpÚ
LayerScaleÚDropPathÚcalculate_drop_path_ratesÚ	RelPosMlpÚ
RelPosBiasÚuse_fused_attnÚ	LayerTypeé   )Úbuild_model_with_cfg)Úfeature_take_indices)Únamed_applyÚ
checkpoint)Úgenerate_default_cfgsÚregister_model)Úget_init_weights_vitÚVisionTransformerRelPosc                   óÜ   ‡ — e Zd ZU ee   ed<   ddddddej                  ddf	dededed	ed
e	e
ej                        dedede
ej                     fˆ fd„Zdde	ej                     fd„Zˆ xZS )ÚRelPosAttentionÚ
fused_attné   FNç        ÚdimÚ	num_headsÚqkv_biasÚqk_normÚrel_pos_clsÚ	attn_dropÚ	proj_dropÚ
norm_layerc                 ór  •— |	|
dœ}t         ‰| �  «        ||z  dk(  sJ d«       ‚|| _        ||z  | _        | j                  dz  | _        t        «       | _        t        j                  ||dz  fd|i|¤Ž| _	        |r || j                  fi |¤Žnt        j                  «       | _        |r || j                  fi |¤Žnt        j                  «       | _        |r
 |dd|i|¤Žnd | _        t        j                  |«      | _        t        j                  ||fi |¤Ž| _        t        j                  |«      | _        y )	N©ÚdeviceÚdtyper   z$dim should be divisible by num_headsg      à¿é   Úbiasr&   © )ÚsuperÚ__init__r&   Úhead_dimÚscaler   r"   ÚnnÚLinearÚqkvÚIdentityÚq_normÚk_normÚrel_posÚDropoutr*   Úprojr+   )Úselfr%   r&   r'   r(   r)   r*   r+   r,   r/   r0   ÚddÚ	__class__s               €úp/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/vision_transformer_relpos.pyr5   zRelPosAttention.__init__/   s  ø€ ð ¨Ñ/ˆÜ‰ÑÔØ�Y‰ !Ò#ÐKÐ%KÓKÐ#Ø"ˆŒØ˜yÑ(ˆŒØ—]‘] dÑ*ˆŒ
Ü(Ó*ˆŒä—9‘9˜S #¨¡'Ñ?°Ð?¸BÑ?ˆŒÙ9@‘j §¡Ñ5°"Ò5ÄbÇkÁkÃmˆŒÙ9@‘j §¡Ñ5°"Ò5ÄbÇkÁkÃmˆŒÙAL‘{Ñ=¨YÐ=¸"Ò=ÐRVˆŒÜŸ™ IÓ.ˆŒÜ—I‘I˜c 3Ñ-¨"Ñ-ˆŒ	ÜŸ™ IÓ.ˆ�ó    Úshared_rel_posc                 ó°  — |j                   \  }}}| j                  |«      j                  ||d| j                  | j                  «      j                  ddddd«      }|j                  d«      \  }}}	| j                  |«      }| j                  |«      }| j                  r| j                  �| j                  j                  «       }
n|�|}
nd }
t        j                  j                  j                  |||	|
| j                   r| j"                  j$                  nd¬«      }ns|| j&                  z  }||j)                  dd	«      z  }| j                  �| j                  ||¬
«      }n|�||z   }|j+                  d	¬«      }| j#                  |«      }||	z  }|j)                  dd«      j                  |||«      }| j-                  |«      }| j/                  |«      }|S )Nr1   é   r   r   é   r$   )Ú	attn_maskÚ	dropout_péþÿÿÿéÿÿÿÿ©rF   ©r%   )Úshaper:   Úreshaper&   r6   ÚpermuteÚunbindr<   r=   r"   r>   Úget_biasÚtorchr8   Ú
functionalÚscaled_dot_product_attentionÚtrainingr*   Úpr7   Ú	transposeÚsoftmaxr@   r+   )rA   ÚxrF   ÚBÚNÚCr:   ÚqÚkÚvÚ	attn_biasÚattns               rD   ÚforwardzRelPosAttention.forwardL   s¦  € Ø—'‘'‰ˆˆ1ˆaØ�h‰h�q‹k×!Ñ! ! Q¨¨4¯>©>¸4¿=¹=ÓI×QÑQÐRSÐUVÐXYÐ[\Ð^_Ó`ˆØ—*‘*˜Q“-‰ˆˆ1ˆaØ�K‰K˜‹NˆØ�K‰K˜‹Nˆà�?Š?Ø�|‰|Ð'Ø ŸL™L×1Ñ1Ó3‘	ØÐ+Ø*‘	à �	ä—‘×#Ñ#×@Ñ@Ø�1�aØ#Ø.2¯mªm˜$Ÿ.™.×*Ò*Àð Aó ‰Að �D—J‘J‘ˆAØ�q—{‘{ 2 rÓ*Ñ*ˆDØ�|‰|Ð'Ø—|‘| D¸�|ÓH‘ØÐ+Ø˜nÑ,�Ø—<‘< B�<Ó'ˆDØ—>‘> $Ó'ˆDØ�q‘ˆAà�K‰K˜˜1Ó×%Ñ% a¨¨AÓ.ˆØ�I‰I�a‹LˆØ�N‰N˜1ÓˆØˆrE   ©N)Ú__name__Ú
__module__Ú__qualname__r
   ÚboolÚ__annotations__r8   Ú	LayerNormÚintr   r   ÚModuleÚfloatr5   rU   ÚTensorre   Ú__classcell__©rC   s   @rD   r!   r!   ,   s«   ø… Ø�d‘Óð
 Ø"Ø!Ø59Ø!Ø!Ø*,¯,©,ØØñ/àð/ð ð/ð ð	/ð
 ð/ð " $ r§y¡y¡/Ñ2ð/ð ð/ð ð/ð ˜RŸY™Y™õ/ñ:"¨°%·,±,Ñ)?÷ "rE   r!   c                   ó  ‡ — e Zd Zddddddddej                  ej
                  ddfdedededed	ed
e	e
ej                        de	e   dededede
ej                     de
ej                     fˆ fd„Zdde	ej                     fd„Zˆ xZS )ÚRelPosBlockç      @FNr$   r%   r&   Ú	mlp_ratior'   r(   r)   Úinit_valuesr+   r*   Ú	drop_pathÚ	act_layerr,   c           
      ó,  •— ||dœ}t         ‰| �  «         ||fi |¤Ž| _        t        ||f||||	||dœ|¤Ž| _        |rt        |fd|i|¤Žnt        j                  «       | _        |
dkD  rt        |
«      nt        j                  «       | _
         ||fi |¤Ž| _        t        d|t        ||z  «      ||dœ|¤Ž| _        |rt        |fd|i|¤Žnt        j                  «       | _        |
dkD  rt        |
«      | _        y t        j                  «       | _        y )Nr.   ©r'   r(   r)   r*   r+   r,   rw   r$   ©Úin_featuresÚhidden_featuresry   Údropr3   )r4   r5   Únorm1r!   rd   r   r8   r;   Úls1r   Ú
drop_path1Únorm2r   rm   ÚmlpÚls2Ú
drop_path2©rA   r%   r&   rv   r'   r(   r)   rw   r+   r*   rx   ry   r,   r/   r0   rB   rC   s                   €rD   r5   zRelPosBlock.__init__s   s)  ø€ ð" ¨Ñ/ˆÜ‰ÑÔÙ Ñ* rÑ*ˆŒ
Ü#ØØð

ð ØØ#ØØØ!ñ

ð ñ

ˆŒ	ñ FQ”:˜cÑA¨{ÐA¸bÒAÔVX×VaÑVaÓVcˆŒà1:¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒá Ñ* rÑ*ˆŒ
Üð 
ØÜ  i¡Ó0ØØñ	
ð
 ñ
ˆŒñ FQ”:˜cÑA¨{ÐA¸bÒAÔVX×VaÑVaÓVcˆŒØ1:¸R²œ( 9Ó-ˆ�ÄRÇ[Á[Ã]ˆ�rE   rF   c           
      ó  — || j                  | j                  | j                  | j                  |«      |¬«      «      «      z   }|| j	                  | j                  | j                  | j                  |«      «      «      «      z   }|S ©NrN   )r‚   r�   rd   r€   r†   r…   r„   rƒ   ©rA   r\   rF   s      rD   re   zRelPosBlock.forward¡   sf   € Ø�—‘ §¡¨¯©°4·:±:¸a³=ÐQ_¨Ó)`Ó aÓbÑbˆØ�—‘ §¡¨¯©°$·*±*¸Q³-Ó)@Ó AÓBÑBˆØˆrE   rf   )rg   rh   ri   r8   ÚGELUrl   rm   ro   rj   r   r   rn   r5   rU   rp   re   rq   rr   s   @rD   rt   rt   q   sð   ø„ ð  "Ø"Ø!Ø59Ø+/Ø!Ø!Ø!Ø)+¯©Ø*,¯,©,ØØñ,Sàð,Sð ð,Sð ð	,Sð
 ð,Sð ð,Sð " $ r§y¡y¡/Ñ2ð,Sð " %™ð,Sð ð,Sð ð,Sð ð,Sð ˜BŸI™I‘ð,Sð ˜RŸY™Y™õ,Sñ\¨°%·,±,Ñ)?÷ rE   rt   c                   ó  ‡ — e Zd Zddddddddej                  ej
                  ddfdedededed	ed
e	e
ej                        de	e   dededede
ej                     de
ej                     fˆ fd„Zd„ Zdde	ej                     fd„Zˆ xZS )ÚResPostRelPosBlockru   FNr$   r%   r&   rv   r'   r(   r)   rw   r+   r*   rx   ry   r,   c           
      óª  •— ||dœ}t         ‰| �  «        || _        t        ||f||||	||dœ|¤Ž| _         ||fi |¤Ž| _        |
dkD  rt        |
«      nt        j                  «       | _	        t        d|t        ||z  «      ||dœ|¤Ž| _         ||fi |¤Ž| _        |
dkD  rt        |
«      nt        j                  «       | _        | j                  «        y )Nr.   r{   r$   r|   r3   )r4   r5   rw   r!   rd   r€   r   r8   r;   r‚   r   rm   r„   rƒ   r†   Úinit_weightsr‡   s                   €rD   r5   zResPostRelPosBlock.__init__©   sí   ø€ ð" ¨Ñ/ˆÜ‰ÑÔØ&ˆÔä#ØØð

ð ØØ#ØØØ!ñ

ð ñ

ˆŒ	ñ   Ñ* rÑ*ˆŒ
Ø1:¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒäð 
ØÜ  i¡Ó0ØØñ	
ð
 ñ
ˆŒñ   Ñ* rÑ*ˆŒ
Ø1:¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒà×ÑÕrE   c                 ó  — | j                   �}t        j                  j                  | j                  j
                  | j                   «       t        j                  j                  | j                  j
                  | j                   «       y y rf   )rw   r8   ÚinitÚ	constant_r€   Úweightrƒ   ©rA   s    rD   r�   zResPostRelPosBlock.init_weightsØ   s[   € à×ÑÐ'Ü�G‰G×Ñ˜dŸj™j×/Ñ/°×1AÑ1AÔBÜ�G‰G×Ñ˜dŸj™j×/Ñ/°×1AÑ1AÕBð (rE   rF   c           	      óÒ   — || j                  | j                  | j                  ||¬«      «      «      z   }|| j                  | j	                  | j                  |«      «      «      z   }|S r‰   )r‚   r€   rd   r†   rƒ   r„   rŠ   s      rD   re   zResPostRelPosBlock.forwardÞ   sT   € Ø�—‘ §
¡
¨4¯9©9°QÀ~¨9Ó+VÓ WÓXÑXˆØ�—‘ §
¡
¨4¯8©8°A«;Ó 7Ó8Ñ8ˆØˆrE   rf   )rg   rh   ri   r8   r‹   rl   rm   ro   rj   r   r   rn   r5   r�   rU   rp   re   rq   rr   s   @rD   r�   r�   §   sé   ø„ ð  "Ø"Ø!Ø59Ø+/Ø!Ø!Ø!Ø)+¯©Ø*,¯,©,ØØñ-àð-ð ð-ð ð	-ð
 ð-ð ð-ð " $ r§y¡y¡/Ñ2ð-ð " %™ð-ð ð-ð ð-ð ð-ð ˜BŸI™I‘ð-ð ˜RŸY™Y™õ-ò^Cñ¨°%·,±,Ñ)?÷ rE   r�   c            8       ó‚  ‡ — e Zd ZdZddddddddd	d
dddddddddddddeddeddfdeeeeef   f   deeeeef   f   dedede	d   dededede
dededee
   deded ed!ee   d"ed#e
d$e
d%e
d&e
d'e	d(   d)ed*eej                      d+ee   d,ee   d-eej                      f6ˆ fd.„ZdHd0„ZdId1ed2ed/dfd3„Zej,                  j.                  d4„ «       Zej,                  j.                  dJd5„«       Zej,                  j.                  dKd6„«       Zej,                  j.                  d/ej                   fd7„«       ZdLdedee   fd8„Z	 	 	 	 	 	 dMd9ej:                  d:eeeee   f      d;ed<ed=ed>ed?ed/eeej:                     eej:                  eej:                     f   f   fd@„Z	 	 	 dNd:eeee   f   dAedBefdC„Z dD„ Z!dJdEefdF„Z"dG„ Z#ˆ xZ$S )Or   ah   Vision Transformer w/ Relative Position Bias

    Differing from classic vit, this impl
      * uses relative position index (swin v1 / beit) or relative log coord + mlp (swin v2) pos embed
      * defaults to no class token (can be enabled)
      * defaults to global avg pool for head (can be changed)
      * layer-scale (residual branch gain) enabled
    éà   é   r1   éè  Úavgé   é   ru   TFç�íµ ÷Æ°>r„   Nr$   ÚresetÚimg_sizeÚ
patch_sizeÚin_chansÚnum_classesÚglobal_pool)Ú rš   ÚtokenÚmapÚ	embed_dimÚdepthr&   rv   r'   r(   rw   Úclass_tokenÚfc_normÚrel_pos_typeÚrel_pos_dimrF   Ú	drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_rateÚweight_init)Úskiprž   ÚjaxÚmocor¤   Úfix_initÚembed_layerr,   ry   Úblock_fnc                 óî  •— t         ‰%| �  «        ||dœ}|dv sJ ‚|s|dk7  sJ ‚|xs t        t        j                  d¬«      }|xs t        j
                  }|| _        || _        || _        |x| _	        x| _
        | _        |rdnd| _        d| _         |d||||d	œ|¤Ž| _        | j                  j                  }t!        | j                  d
«      r| j                  j#                  «       n|} t%        || j                  ¬«      }!|j'                  d«      r!|r||!d<   d|v rd|!d<   t        t(        fi |!¤Ž}"nt        t*        fi |!¤Ž}"d| _        |r |"dd|i|¤Ž| _        d}"|r5t        j.                  t1        j2                  d| j                  |fi |¤Ž«      nd| _        t7        ||«      }#t        j8                  t;        |«      D �$cg c]  }$ |d|||	|
||"||||#|$   ||dœ|¤Ž‘Œ c}$«      | _        t;        |«      D �$cg c]  }$t%        d|$› �|| ¬«      ‘Œ c}$| _        |s	 ||fi |¤Žnt        j@                  «       | _!        |r	 ||fi |¤Žnt        j@                  «       | _"        t        jF                  |«      | _$        |dkD  r!t        jJ                  | j                  |fi |¤Žnt        j@                  «       | _&        |dk(  rdn|| _'        || _(        |dk7  r| jS                  d¬«       yyc c}$w c c}$w )aE  
        Args:
            img_size: input image size
            patch_size: patch size
            in_chans: number of input channels
            num_classes: number of classes for classification head
            global_pool: type of global pooling for final sequence (default: 'avg')
            embed_dim: embedding dimension
            depth: depth of transformer
            num_heads: number of attention heads
            mlp_ratio: ratio of mlp hidden dim to embedding dim
            qkv_bias: enable bias for qkv if True
            qk_norm: Enable normalization of query and key in attention
            init_values: layer-scale init values
            class_token: use class token (default: False)
            fc_norm: use pre classifier norm instead of pre-pool
            rel_pos_type: type of relative position
            shared_rel_pos: share relative pos across all blocks
            drop_rate: dropout rate
            proj_drop_rate: projection dropout rate
            attn_drop_rate: attention dropout rate
            drop_path_rate: stochastic depth rate
            weight_init: weight init scheme
            fix_init: apply weight initialization fix (scaling w/ layer index)
            embed_layer: patch embedding layer
            norm_layer: normalization layer
            act_layer: MLP activation layer
        r.   ©r¤   rš   r¥   r¥   r�   )Úepsr   r   F)rŸ   r    r¡   r§   Ú
feat_ratio)Úwindow_sizeÚprefix_tokensr„   Ú
hidden_dimÚswinÚmodeNr&   )r%   r&   rv   r'   r(   r)   rw   r+   r*   rx   r,   ry   zblocks.)ÚmoduleÚnum_chsÚ	reductionr²   rž   ©Úneeds_resetr3   )*r4   r5   r   r8   rl   r‹   r¢   r¡   r£   Únum_featuresÚhead_hidden_sizer§   Únum_prefix_tokensÚgrad_checkpointingÚpatch_embedÚ	grid_sizeÚhasattrr»   ÚdictÚ
startswithr   r   rF   Ú	ParameterrU   ÚzerosÚ	cls_tokenr   Ú
ModuleListÚrangeÚblocksÚfeature_infor;   Únormrª   r?   Ú	head_dropr9   ÚheadÚweight_init_moderµ   r�   )&rA   rŸ   r    r¡   r¢   r£   r§   r¨   r&   rv   r'   r(   rw   r©   rª   r«   r¬   rF   r­   r®   r¯   r°   r±   rµ   r¶   r,   ry   r·   r/   r0   rB   Ú	feat_sizeÚrÚrel_pos_argsr)   ÚdprÚirC   s&                                        €rD   r5   z VisionTransformerRelPos.__init__î   s  ø€ ôx 	‰ÑÔØ¨Ñ/ˆØÐ2Ñ2Ð2Ð2Ù˜k¨WÒ4Ð4Ð4ØÒB¤7¬2¯<©<¸TÔ#Bˆ
ØÒ(¤§¡ˆ	à&ˆÔØ ˆŒØ&ˆÔØENÐNˆÔÐN˜DÔ1°D´NÙ&1¡°qˆÔØ"'ˆÔá&ð 
ØØ!ØØñ	
ð
 ñ
ˆÔð ×$Ñ$×.Ñ.ˆ	Ü-4°T×5EÑ5EÀ|Ô-TˆD×Ñ×'Ñ'Ô)ÐZdˆä¨	À×AWÑAWÔXˆØ×"Ñ" 5Ô)ÙØ-8�˜\Ñ*Ø˜Ñ%Ø'-�˜VÑ$Ü!¤)Ñ<¨|Ñ<‰Kä!¤*Ñ=°Ñ=ˆKØ"ˆÔÙÙ"-Ñ"H¸	Ð"HÀRÑ"HˆDÔàˆKábmœŸ™¤e§k¡k°!°T×5KÑ5KÈYÑ&]ÐZ\Ñ&]Ô^ÐswˆŒä'¨¸Ó>ˆÜ—m‘mô  ˜5“\ö!%#ð  ñ ð ØØ#Ø#Ø!ØØ'Ø'Ø(Ø(Ø˜a™&Ø%Ø#ñð óò%#ó $ˆŒô$ QVÐV[ÓP\ö^ØKLŒD˜' ! ˜¨yÀAÖFò^ˆÔá7>‘J˜yÑ/¨BÒ/ÄBÇKÁKÃMˆŒ	ñ 7>‘z )Ñ2¨rÒ2Ä2Ç;Á;Ã=ˆŒÜŸ™ IÓ.ˆŒØDOÐRSÂO”B—I‘I˜dŸn™n¨kÑ@¸RÒ@ÔY[×YdÑYdÓYfˆŒ	à+6¸&Ò+@¡ÀkˆÔØ ˆŒà˜&Ò Ø×Ñ¨%ÐÕ0ð !ùò;%#ùò"^s   ÇK-Ç>K2Úreturnc                 ó„  — t        j                  «       5  t        | j                  «      D ]~  \  }}t	        j
                  d|dz   z  «      }|j                  j                  j                  j                  |«       |j                  j                  j                  j                  |«       Œ€ 	 ddd«       y# 1 sw Y   yxY w)z9Apply weight initialization fix (scaling w/ layer index).g       @r   N)rU   Úno_gradÚ	enumeraterÔ   ÚmathÚsqrtrd   r@   r“   Údiv_r„   Úfc2)rA   Úlayer_idÚlayerr7   s       rD   Úfix_init_weightz'VisionTransformerRelPos.fix_init_weightt  s‹   € ä�]‰]‹_ñ 	1Ü#,¨T¯[©[Ó#9ò 1‘�˜%ÜŸ	™	 #¨°A©Ñ"6Ó7�Ø—
‘
—‘×&Ñ&×+Ñ+¨EÔ2Ø—	‘	—‘×$Ñ$×)Ñ)¨%Õ0ñ1÷	1÷ 	1ñ 	1ús   •BB6Â6B?rÀ   rÅ   c                 óT  — |xs | j                   }|dv sJ ‚d|v r t        j                  | j                  «       nd}| j                  �+t
        j                  j                  | j                  d¬«       t        t        |||¬«      | «       | j                  r| j                  «        yy)aP  Initialize model weights.

        Args:
            mode: Weight initialization mode ('jax', 'jax_nlhb', 'moco', or '').
            needs_reset: If True, call reset_parameters() on modules (default for after to_empty()).
                If False, skip reset_parameters() (for __init__ where modules already self-initialized).
        )r³   Újax_nlhbr´   rž   r¤   Únlhbr$   Nr�   )ÚstdrÄ   )rÙ   rã   Úlogr¢   rÑ   r8   r‘   Únormal_r   r   rµ   ré   )rA   rÀ   rÅ   Ú	head_biass       rD   r�   z$VisionTransformerRelPos.init_weights|  s‘   € ð Ò,�t×,Ñ,ˆØÐ?Ñ?Ð?Ð?Ø39¸T±>”T—X‘X˜d×.Ñ.Ó/Ñ/Àrˆ	Ø�>‰>Ð%Ü�G‰G�O‰O˜DŸN™N°ˆOÔ5äÔ(¨¨yÀkÔRÐTXÔYà�=Š=Ø× Ñ Õ"ð rE   c                 ó   — dhS )NrÑ   r3   r”   s    rD   Úno_weight_decayz'VisionTransformerRelPos.no_weight_decay�  s
   € àˆ}ÐrE   c                 ó    — t        dddg¬«      S )Nz^cls_token|patch_embed)z^blocks\.(\d+)N)z^norm)iŸ† )ÚstemrÔ   )rÍ   )rA   Úcoarses     rD   Úgroup_matcherz%VisionTransformerRelPos.group_matcher“  s   € äØ*Ø-Ð/CÐDô
ð 	
rE   c                 ó   — || _         y rf   )rÉ   )rA   Úenables     rD   Úset_grad_checkpointingz.VisionTransformerRelPos.set_grad_checkpointingš  s
   € à"(ˆÕrE   c                 ó   — | j                   S rf   )rØ   r”   s    rD   Úget_classifierz&VisionTransformerRelPos.get_classifierž  s   € à�y‰yÐrE   c                 óÄ   — ||dœ}|| _         |�|dv sJ ‚|| _        |dkD  r't        j                  | j                  |fi |¤Ž| _        y t        j
                  «       | _        y )Nr.   r¹   r   )r¢   r£   r8   r9   r§   r;   rØ   )rA   r¢   r£   r/   r0   rB   s         rD   Úreset_classifierz(VisionTransformerRelPos.reset_classifier¢  sc   € Ø¨Ñ/ˆØ&ˆÔØÐ"ØÐ"6Ñ6Ð6Ð6Ø*ˆDÔØDOÐRSÂO”B—I‘I˜dŸn™n¨kÑ@¸RÑ@ˆ�	ÔY[×YdÑYdÓYfˆ�	rE   r\   ÚindicesÚreturn_prefix_tokensrÖ   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc           	      ó4  — |dv sJ d«       ‚|dk(  }g }	t        t        | j                  «      |«      \  }
}|j                  \  }}}}| j	                  |«      }| j
                  �At        j                  | j
                  j                  |j                  d   dd«      |fd¬«      }| j                  �| j                  j                  «       nd}t        j                  j                  «       s|s| j                  }n| j                  d|dz    }t        |«      D ]q  \  }}| j                  r-t        j                  j                  «       st        |||¬	«      }n
 |||¬	«      }||
v sŒN|	j!                  |r| j#                  |«      n|«       Œs | j$                  rD|	D �cg c]  }|dd…d| j$                  …f   ‘Œ }}|	D �cg c]  }|dd…| j$                  d…f   ‘Œ }	}|ra| j                  j'                  ||f«      \  }}|	D �cg c]6  }|j)                  |||d«      j+                  dd
dd«      j-                  «       ‘Œ8 }	}t        j                  j                  «       s|rt/        t1        |	«      «      }	|r|	S | j#                  |«      }||	fS c c}w c c}w c c}w )a=   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            return_prefix_tokens: Return both prefix and spatial intermediate tokens
            norm: Apply norm layer to all intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )ÚNCHWÚNLCz)Output format must be one of NCHW or NLC.r  Nr   rM   r   rO   rN   r1   rH   )r   ÚlenrÔ   rP   rÊ   rÑ   rU   ÚcatÚexpandrF   rT   ÚjitÚis_scriptingrâ   rÉ   r   ÚappendrÖ   rÈ   Údynamic_feat_sizerQ   rR   Ú
contiguousÚlistÚzip)rA   r\   rþ   rÿ   rÖ   r   r  r  rQ   ÚintermediatesÚtake_indicesÚ	max_indexr]   Ú_ÚheightÚwidthrF   rÔ   rÞ   ÚblkÚyr½   ÚHÚWs                           rD   Úforward_intermediatesz-VisionTransformerRelPos.forward_intermediatesª  si  € ð. ˜_Ñ,ÐYÐ.YÓYÐ,Ø Ñ&ˆØˆÜ"6´s¸4¿;¹;Ó7GÈÓ"QÑˆ�ið  Ÿg™gÑˆˆ1ˆf�eØ×Ñ˜QÓˆØ�>‰>Ð%Ü—	‘	˜4Ÿ>™>×0Ñ0°·±¸±¸RÀÓDÀaÐHÈaÔPˆAà;?×;NÑ;NÐ;Z˜×,Ñ,×5Ñ5Ô7Ð`dˆÜ�9‰9×!Ñ!Ô#©:Ø—[‘[‰Fà—[‘[  )¨a¡-Ð0ˆFÜ Ó'ò 	B‰FˆAˆsØ×&Ò&¬u¯y©y×/EÑ/EÔ/GÜ˜s A°nÔE‘á˜¨.Ô9�Ø�LÒ à×$Ñ$±T T§Y¡Y¨q¤\¸qÕAð	Bð ×!Ò!àERÖSÀ˜Qšq ! D×$:Ñ$:Ð":Ð:Ó;ÐSˆMÐSØDQÖR¸q˜Qšq $×"8Ñ"8Ñ"9Ð9Ó:ÐRˆMÐRÙà×#Ñ#×5Ñ5°v¸u°oÓF‰DˆAˆqØ^kÖlÐYZ˜QŸY™Y q¨!¨Q°Ó3×;Ñ;¸A¸qÀ!ÀQÓG×RÑRÕTÐlˆMÐlÜ�y‰y×%Ñ%Ô'Ñ,@ä ¤ ]°MÓ!BÓCˆMáØ Ð à�I‰I�a‹Lˆà�-ÐÐùò TùÚRùò ms   ÆJÆ<JÈ ;JÚ
prune_normÚ
prune_headc                 ó  — t        t        | j                  «      |«      \  }}| j                  d|dz    | _        |rt        j                  «       | _        |r+t        j                  «       | _        | j                  dd«       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r¤   )r   r  rÔ   r8   r;   rÖ   rª   rý   )rA   rþ   r  r  r  r  s         rD   Úprune_intermediate_layersz1VisionTransformerRelPos.prune_intermediate_layersî  sj   € ô #7´s¸4¿;¹;Ó7GÈÓ"QÑˆ�iØ—k‘k . 9¨q¡=Ð1ˆŒÙÜŸ™›ˆDŒIÙÜŸ;™;›=ˆDŒLØ×!Ñ! ! RÔ(ØÐrE   c                 óÜ  — | j                  |«      }| j                  �At        j                  | j                  j	                  |j
                  d   dd«      |fd¬«      }| j                  �| j                  j                  «       nd }| j                  D ]E  }| j                  r-t        j                  j                  «       st        |||¬«      }Œ< |||¬«      }ŒG | j                  |«      }|S )Nr   rM   r   rO   rN   )rÊ   rÑ   rU   r  r  rP   rF   rT   rÔ   rÉ   r	  r
  r   rÖ   )rA   r\   rF   r  s       rD   Úforward_featuresz(VisionTransformerRelPos.forward_featuresÿ  sÇ   € Ø×Ñ˜QÓˆØ�>‰>Ð%Ü—	‘	˜4Ÿ>™>×0Ñ0°·±¸±¸RÀÓDÀaÐHÈaÔPˆAà;?×;NÑ;NÐ;Z˜×,Ñ,×5Ñ5Ô7Ð`dˆØ—;‘;ò 	:ˆCØ×&Ò&¬u¯y©y×/EÑ/EÔ/GÜ˜s A°nÔE‘á˜¨.Ô9‘ð		:ð
 �I‰I�a‹LˆØˆrE   Ú
pre_logitsc                 ó  — | j                   r=| j                   dk(  r%|d d …| j                  d …f   j                  d¬«      n|d d …df   }| j                  |«      }| j	                  |«      }|r|S | j                  |«      S )Nrš   r   rO   r   )r£   rÈ   Úmeanrª   r×   rØ   )rA   r\   r!  s      rD   Úforward_headz$VisionTransformerRelPos.forward_head  s~   € Ø×ÒØ=A×=MÑ=MÐQVÒ=V�’!�T×+Ñ+Ñ,Ð,Ñ-×2Ñ2°qÐ2Ô9Ð\]Ò^_ÐabÐ^bÑ\cˆAØ�L‰L˜‹OˆØ�N‰N˜1ÓˆÙˆqÐ0 D§I¡I¨a£LÐ0rE   c                 óJ   — | j                  |«      }| j                  |«      }|S rf   )r   r$  )rA   r\   s     rD   re   zVisionTransformerRelPos.forward  s'   € Ø×!Ñ! !Ó$ˆØ×Ñ˜aÓ ˆØˆrE   )rß   N)r¤   T©F)T)NNN)NFFFr  F)r   FT)%rg   rh   ri   Ú__doc__r   rt   r   rm   r   r	   ro   rj   r   Ústrr   r8   rn   r   r5   ré   r�   rU   r	  Úignorerò   rö   rù   rû   rý   rp   r   r  r  r   r$  re   rq   rr   s   @rD   r   r   ä   s�  ø„ ñð 58Ø68ØØ#Ø>CØ ØØØ!Ø!Ø!Ø+/Ø %Ø!Ø %Ø)-Ø#(Ø!Ø$&Ø$&Ø$&ØGNØ"Ø+5Ø.2Ø-1Ø(3ØØñ=D1à˜C  s¨C x¡Ð0Ñ1ðD1ð ˜c 5¨¨c¨¡?Ð2Ñ3ðD1ð ð	D1ð
 ðD1ð !Ð!:Ñ;ðD1ð ðD1ð ðD1ð ðD1ð ðD1ð ðD1ð ðD1ð " %™ðD1ð ðD1ð ðD1ð  ð!D1ð" " #™ð#D1ð$ !ð%D1ð& ð'D1ð( "ð)D1ð* "ð+D1ð, "ð-D1ð. !Ð!CÑDð/D1ð0 ð1D1ð2 ˜bŸi™i™ð3D1ð4 ! Ñ+ð5D1ð6   	Ñ*ð7D1ð8 ˜2Ÿ9™9‘oõ9D1óL1ñ# ð #¸ð #Èó #ð& ‡Y�Y×Ññó ðð ‡Y�Y×Ñò
ó ð
ð ‡Y�Y×Ñò)ó ð)ð ‡Y�Y×Ñð §	¡	ò ó ðñg¨Cð g¸hÀs¹mó gð 8<Ø).ØØ$Ø$Ø',ñB à�|‰|ðB ð ˜e C¨¨c© NÑ3Ñ4ðB ð #'ð	B ð
 ðB ð ðB ð ðB ð !%ðB ð 
ˆt�E—L‘LÑ! 5¨¯©°t¸E¿L¹LÑ7IÐ)IÑ#JÐJÑ	KóB ðL ./Ø$Ø#ñ	à˜3  S¡	˜>Ñ*ðð ðð ó	ò"ñ1¨$ó 1örE   c                 óh   — |j                  dd«      }t        t        | |fdt        |d¬«      i|¤Ž}|S )NÚout_indicesr1   Úfeature_cfgÚgetter)r+  Úfeature_cls)Úpopr   r   rÍ   )ÚvariantÚ
pretrainedÚkwargsr+  Úmodels        rD   Ú!_create_vision_transformer_relposr4    sC   € Ø—*‘*˜]¨AÓ.€KÜ Ü ¨*ñä [¸hÔGðð ñ€Eð
 €LrE   r¤   c                 ó4   — | ddd dddt         t        dddd	œ|¥S )
Nr™   )r1   r—   r—   gÍÌÌÌÌÌì?ÚbicubicTzpatch_embed.projrØ   z
apache-2.0)Úurlr¢   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚfixed_input_sizer#  rí   Ú
first_convÚ
classifierÚlicenser   )r7  r2  s     rD   Ú_cfgr@  $  s5   € àØ¨=ÀtØ¨ÈÜ'Ô0FØ(¸Øñð ðð rE   z‰https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_replos_base_patch32_plus_rpn_256-sw-dd486f51.pthztimm/)r1   é   rA  )r7  Ú	hf_hub_idr8  )r1   éð   rC  )r7  r8  z�https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_relpos_small_patch16_224-sw-ec2778b4.pth)r7  rB  z‚https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_relpos_medium_patch16_224-sw-11c174af.pthz€https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_relpos_base_patch16_224-sw-49049aed.pthz‚https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_srelpos_small_patch16_224-sw-6cdb8849.pthzƒhttps://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_srelpos_medium_patch16_224-sw-ad702b8c.pthz†https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_relpos_medium_patch16_cls_224-sw-cfe8e259.pthz‡https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_relpos_base_patch16_gapcls_224-sw-1a341d6c.pthz†https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-tpu-weights/vit_relpos_medium_patch16_rpn_224-sw-5d2befd8.pth)z,vit_relpos_base_patch32_plus_rpn_256.sw_in1kz*vit_relpos_base_patch16_plus_240.untrainedz$vit_relpos_small_patch16_224.sw_in1kz%vit_relpos_medium_patch16_224.sw_in1kz#vit_relpos_base_patch16_224.sw_in1kz%vit_srelpos_small_patch16_224.sw_in1kz&vit_srelpos_medium_patch16_224.sw_in1kz)vit_relpos_medium_patch16_cls_224.sw_in1kz)vit_relpos_base_patch16_cls_224.untrainedz*vit_relpos_base_patch16_clsgap_224.sw_in1kz*vit_relpos_small_patch16_rpn_224.untrainedz)vit_relpos_medium_patch16_rpn_224.sw_in1kz)vit_relpos_base_patch16_rpn_224.untrainedrß   c           	      ó^   — t        ddddt        ¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )z` ViT-Base (ViT-B/32+) w/ relative log-coord position and residual post-norm, no class token
    é    é€  rœ   é   )r    r§   r¨   r&   r·   r1  )Ú$vit_relpos_base_patch32_plus_rpn_256©rÍ   r�   r4  ©r1  r2  Ú
model_argsr3  s       rD   rH  rH  X  sG   € ô  ¨s¸"ÈÔUgÔh€JÜ-Ø.ñeØ;EðeÜIMÈjÑIcÐ\bÑIcñe€Eà€LrE   c           	      óT   — t        dddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )zI ViT-Base (ViT-B/16+) w/ relative log-coord position, no class token
    r˜   rF  rœ   rG  )r    r§   r¨   r&   r1  )Ú vit_relpos_base_patch16_plus_240©rÍ   r4  rJ  s       rD   rM  rM  b  sD   € ô  ¨s¸"ÈÔK€JÜ-Ø*ñaØ7AðaÜEIÈ*ÑE_ÐX^ÑE_ña€Eà€LrE   c           	      óX   — t        dddddd¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
úH ViT-Base (ViT-B/16) w/ relative log-coord position, no class token
    r˜   é€  rœ   é   FT©r    r§   r¨   r&   r'   rª   r1  )Úvit_relpos_small_patch16_224rN  rJ  s       rD   rT  rT  l  sJ   € ô  ¨s¸"ÈÐTYÐcgÔh€JÜ-Ø&ñ]Ø3=ð]ÜAEÀjÑA[ÐTZÑA[ñ]€Eà€LrE   c           	      óX   — t        dddddd¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
rP  r˜   é   rœ   r#   FTrS  r1  )Úvit_relpos_medium_patch16_224rN  rJ  s       rD   rW  rW  v  sM   € ô Ø ¨B¸!ÀeÐUYô[€Jä-Ø'ñ^Ø4>ð^ÜBFÀzÑB\ÐU[ÑB\ñ^€Eà€LrE   c           	      óX   — t        dddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	rP  r˜   r›   rœ   FTrS  r1  )Úvit_relpos_base_patch16_224rN  rJ  s       rD   rY  rY  �  sM   € ô Ø ¨B¸"ÀuÐVZô\€Jä-Ø%ñ\Ø2<ð\Ü@DÀZÑ@ZÐSYÑ@Zñ\€Eà€LrE   c           
      ó\   — t        dddddddd¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
úO ViT-Base (ViT-B/16) w/ shared relative log-coord position, no class token
    r˜   rQ  rœ   rR  FT©r    r§   r¨   r&   r'   rª   r¬   rF   r1  )Úvit_srelpos_small_patch16_224rN  rJ  s       rD   r]  r]  Œ  sS   € ô Ø ¨B¸!ÀeÐUZØ¨ô.€Jô .Ø'ñ^Ø4>ð^ÜBFÀzÑB\ÐU[ÑB\ñ^€Eà€LrE   c           
      ó\   — t        dddddddd¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
r[  r˜   rV  rœ   r#   FTr\  r1  )Úvit_srelpos_medium_patch16_224rN  rJ  s       rD   r_  r_  ˜  sS   € ô Ø ¨B¸!ÀeÐUZØ¨ô.€Jô .Ø(ñ_Ø5?ð_ÜCGÈ
ÑC]ÐV\ÑC]ñ_€Eà€LrE   c                 ó^   — t        ddddddddd¬	«	      }t        	 dd
| it        |fi |¤Ž¤Ž}|S )zM ViT-Base (ViT-M/16) w/ relative log-coord position, class token present
    r˜   rV  rœ   r#   FrA  Tr¥   )	r    r§   r¨   r&   r'   rª   r¬   r©   r£   r1  )Ú!vit_relpos_medium_patch16_cls_224rN  rJ  s       rD   ra  ra  ¤  sV   € ô Ø ¨B¸!ÀeÐUZØ T°wô@€Jô .Ø+ñbØ8BðbÜFJÈ:ÑF`ÐY_ÑF`ñb€Eà€LrE   c           	      óZ   — t        ddddddd¬«      }t        	 d	d| it        |fi |¤Ž¤Ž}|S )
zM ViT-Base (ViT-B/16) w/ relative log-coord position, class token present
    r˜   r›   rœ   FTr¥   )r    r§   r¨   r&   r'   r©   r£   r1  )Úvit_relpos_base_patch16_cls_224rN  rJ  s       rD   rc  rc  °  sP   € ô Ø ¨B¸"ÀuÐZ^Ðlsôu€Jä-Ø)ñ`Ø6@ð`ÜDHÈÑD^ÐW]ÑD^ñ`€Eà€LrE   c           	      óZ   — t        ddddddd¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	a   ViT-Base (ViT-B/16) w/ relative log-coord position, class token present
    NOTE this config is a bit of a mistake, class token was enabled but global avg-pool w/ fc-norm was not disabled
    Leaving here for comparisons w/ a future re-train as it performs quite well.
    r˜   r›   rœ   FT)r    r§   r¨   r&   r'   rª   r©   r1  )Ú"vit_relpos_base_patch16_clsgap_224rN  rJ  s       rD   re  re  »  sP   € ô Ø ¨B¸"ÀuÐVZÐhlôn€Jä-Ø,ñcØ9CðcÜGKÈJÑGaÐZ`ÑGañc€Eà€LrE   c           	      ó`   — t        dddddt        ¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	ú_ ViT-Base (ViT-B/16) w/ relative log-coord position and residual post-norm, no class token
    r˜   rQ  rœ   rR  F©r    r§   r¨   r&   r'   r·   r1  )Ú vit_relpos_small_patch16_rpn_224rI  rJ  s       rD   ri  ri  È  sM   € ô Ø ¨B¸!ÀeÔVhôj€Jä-Ø*ñaØ7AðaÜEIÈ*ÑE_ÐX^ÑE_ña€Eà€LrE   c           	      ó`   — t        dddddt        ¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )	rg  r˜   rV  rœ   r#   Frh  r1  )Ú!vit_relpos_medium_patch16_rpn_224rI  rJ  s       rD   rk  rk  Ó  sM   € ô Ø ¨B¸!ÀeÔVhôj€Jä-Ø+ñbØ8BðbÜFJÈ:ÑF`ÐY_ÑF`ñb€Eà€LrE   c           	      ó`   — t        dddddt        ¬«      }t        	 dd| it        |fi |¤Ž¤Ž}|S )rg  r˜   r›   rœ   Frh  r1  )Úvit_relpos_base_patch16_rpn_224rI  rJ  s       rD   rm  rm  Þ  sM   € ô Ø ¨B¸"ÀuÔWiôk€Jä-Ø)ñ`Ø6@ð`ÜDHÈÑD^ÐW]ÑD^ñ`€Eà€LrE   r&  )r¤   )Er'  Úloggingrã   Ú	functoolsr   Útypingr   r   r   r   r   r	   ÚImportErrorÚtyping_extensionsrU   Útorch.nnr8   Ú	torch.jitr
   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_manipulater   r   Ú	_registryr   r   Úvision_transformerr   Ú__all__Ú	getLoggerrg   Ú_loggerrn   r!   rt   r�   r   r4  r@  Údefault_cfgsrH  rM  rT  rW  rY  r]  r_  ra  rc  re  ri  rk  rm  r3   rE   rD   ú<module>r€     s  ðñó Û Ý ß 5Õ 5ð*Ýó Ý Ý ç E÷
÷ 
õ 
õ +Ý +ß 0ß <Ý 4à$Ð
%€à
ˆ'×
Ñ
˜HÓ
%€ôB�b—i‘iô BôJ3�"—)‘)ô 3ôl:˜Ÿ™ô :ôzs˜bŸi™iô sól	ó	ñ %Ù48ð XØØ ô5"ñ 37¸2È-Ô2Xá,0ð PØô-ñ .2ð QØô.ñ ,0ð OØô,ñ .2ð QØô.ñ /3ð RØô/ñ 26ð UØô2ñ 26³Ù26ð VØô3ñ 37³&Ù15ð UØô2ñ 26³ñI%&ó %€ðP ñÐH_ò ó ðð ñÐD[ò ó ðð ñÐ@Wò ó ðð ñÐAXò ó ðð ñÐ?Vò ó ðð ñÐAXò ó ðð ñÐBYò ó ðð ñÐE\ò ó ðð ñÐCZò ó ðð ñ	ÐF]ò 	ó ð	ð ñÐD[ò ó ðð ñÐE\ò ó ðð ñÐCZò ó ñøðc ò *ß)Ð)ð*ús    G: Ç:H	ÈH	