Ë
    éÓ^j$a  ã            	       ód  — d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
 ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZmZmZmZmZmZmZmZmZ ddlm Z  dd	l!m"Z" dd
l#m$Z$ ddl%m&Z&m'Z' ddl(m)Z)m*Z* dgZ+ G d„ dejX                  «      Z- G d„ dejX                  «      Z. G d„ dejX                  «      Z/ G d„ dejX                  «      Z0 G d„ dejX                  «      Z1dee2e2f   fd„Z3e$dee2e2f   dee2e2f   fd„«       Z4 G d„ dejX                  «      Z5 G d„ dejX                  «      Z6 G d „ dejX                  «      Z7d0d!„Z8d1d"„Z9 e* e9d#¬$«       e9d%¬$«       e9d&¬$«       e9d'¬$«       e9d(¬$«      d)œ«      Z:e)d0d*e7fd+„«       Z;e)d0d*e7fd,„«       Z<e)d0d*e7fd-„«       Z=e)d0d*e7fd.„«       Z>e)d0d*e7fd/„«       Z?y)2a)   Global Context ViT

From scratch implementation of GCViT in the style of timm swin_transformer_v2_cr.py

Global Context Vision Transformers -https://arxiv.org/abs/2206.09959

@article{hatamizadeh2022global,
  title={Global Context Vision Transformers},
  author={Hatamizadeh, Ali and Yin, Hongxu and Kautz, Jan and Molchanov, Pavlo},
  journal={arXiv preprint arXiv:2206.09959},
  year={2022}
}

Free of any code related to NVIDIA GCVit impl at https://github.com/NVlabs/GCVit.
The license for this code release is Apache 2.0 with no commercial restrictions.

However, weight files adapted from NVIDIA GCVit impl ARE under a non-commercial share-alike license
(https://creativecommons.org/licenses/by-nc-sa/4.0/) until I have a chance to train new ones...

Hacked together by / Copyright 2022, Ross Wightman
é    N)Úpartial)ÚCallableÚListÚOptionalÚTupleÚTypeÚUnion©ÚIMAGENET_DEFAULT_MEANÚIMAGENET_DEFAULT_STD)ÚDropPathÚcalculate_drop_path_ratesÚ	to_2tupleÚ	to_ntupleÚMlpÚClassifierHeadÚLayerNorm2dÚ
LayerScaleÚget_attnÚget_act_layerÚget_norm_layerÚ
RelPosBiasÚ_asserté   )Úbuild_model_with_cfg)Úfeature_take_indices)Úregister_notrace_function)Únamed_applyÚ
checkpoint)Úregister_modelÚgenerate_default_cfgsÚGlobalContextVitc                   ó†   ‡ — e Zd ZdZddddej
                  ddfdedee   ded	e	d
e
deej                     fˆ fd„Zd„ Zˆ xZS )ÚMbConvBlockzR A depthwise separable / fused mbconv style residual block with SE, `no norm.
    Ng      ð?ÚseFÚin_chsÚout_chsÚexpand_ratioÚ
attn_layerÚbiasÚ	act_layerc	                 ó†  •— ||dœ}	t         ‰| �  «        t        dd|i|	¤Ž}
t        |t        «      r|dk(  s|dk(  r
d|
d<   d|
d<   t        |«      }|xs |}t        ||z  «      }t        j                  ||d	d
d
f||dœ|	¤Ž| _	         |«       | _
         ||fi |
¤Ž| _        t        j                  ||d
d
dfd|i|	¤Ž| _        y )N©ÚdeviceÚdtyper+   r%   Úecag      Ð?Úrd_ratioFr*   é   r   )Úgroupsr*   r   © )ÚsuperÚ__init__ÚdictÚ
isinstanceÚstrr   ÚintÚnnÚConv2dÚconv_dwÚactr%   Úconv_pw)Úselfr&   r'   r(   r)   r*   r+   r.   r/   ÚddÚattn_kwargsÚmid_chsÚ	__class__s               €ú\/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/gcvit.pyr6   zMbConvBlock.__init__9   sÜ   ø€ ð ¨Ñ/ˆÜ‰ÑÔÜÑ5 YÐ5°"Ñ5ˆÜ�j¤#Ô&¨:¸Ò+=ÀÈuÒATØ&*ˆK˜
Ñ#Ø"'ˆK˜ÑÜ˜jÓ)ˆ
ØÒ#˜VˆÜ�l VÑ+Ó,ˆä—y‘y ¨°!°Q¸ÐZÀ&ÈtÑZÐWYÑZˆŒÙ“;ˆŒÙ˜WÑ4¨Ñ4ˆŒÜ—y‘y ¨'°1°a¸ÑLÀÐLÈÑLˆ�ó    c                 óœ   — |}| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }||z   }|S ©N)r=   r>   r%   r?   )r@   ÚxÚshortcuts      rE   ÚforwardzMbConvBlock.forwardS   sI   € ØˆØ�L‰L˜‹OˆØ�H‰H�Q‹KˆØ�G‰G�A‹JˆØ�L‰L˜‹OˆØ�‰LˆØˆrF   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r;   ÚGELUr:   r   Úfloatr9   Úboolr   ÚModuler6   rK   Ú__classcell__©rD   s   @rE   r$   r$   6   sz   ø„ ñð
 &*Ø"%Ø"ØØ)+¯©ØØñMàðMð ˜c‘]ðMð  ð	Mð
 ðMð ðMð ˜BŸI™I‘õMö4rF   r$   c                   ó–   ‡ — e Zd Zddej                  eddfdedee   dede	ej                     de	ej                     f
ˆ fd„Zd	„ Zˆ xZS )
ÚDownsample2dNÚconvÚdimÚdim_outÚ	reductionr+   Ú
norm_layerc                 óì  •— ||dœ}t         ‰	| �  «        |xs |}|�	 ||fi |¤Žnt        j                  «       | _        t        |fd|i|¤Ž| _        |dv sJ ‚|dk(  r"t        j                  ||dddfdd	i|¤Ž| _        nL|d
k(  r%||k(  sJ ‚t        j                  ddd¬«      | _        n"||k(  sJ ‚t        j                  d¬«      | _        |� ||fi |¤Ž| _        y t        j                  «       | _        y )Nr-   r+   )rX   ÚmaxÚavgrX   r2   é   r   r*   Fr^   ©Úkernel_sizeÚstrideÚpadding©rb   )r5   r6   r;   ÚIdentityÚnorm1r$   Ú
conv_blockr<   r[   Ú	MaxPool2dÚ	AvgPool2dÚnorm2)
r@   rY   rZ   r[   r+   r\   r.   r/   rA   rD   s
            €rE   r6   zDownsample2d.__init__^   sö   ø€ ð ¨Ñ/ˆÜ‰ÑÔØ’.˜Sˆà.8Ð.D‘Z Ñ* rÒ*Ì"Ï+É+Ë-ˆŒ
Ü% cÑE°YÐEÀ"ÑEˆŒØÐ2Ñ2Ð2Ð2Ø˜ÒÜŸY™Y s¨G°Q¸¸1ÑOÀ5ÐOÈBÑOˆD�NØ˜%ÒØ˜'’>Ð!�>ÜŸ\™\°aÀÈ1ÔMˆD�Nà˜'’>Ð!�>ÜŸ\™\°aÔ8ˆDŒNØ2<Ð2H‘Z Ñ.¨2Ñ.ˆ�
ÌbÏkÉkËmˆ�
rF   c                 óŽ   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|S rH   )rg   rh   r[   rk   ©r@   rI   s     rE   rK   zDownsample2d.forwardy   s=   € Ø�J‰J�q‹MˆØ�O‰O˜AÓˆØ�N‰N˜1ÓˆØ�J‰J�q‹MˆØˆrF   )rL   rM   rN   r;   rP   r   r:   r   r9   r   rS   r6   rK   rT   rU   s   @rE   rW   rW   ]   so   ø„ ð &*Ø#Ø)+¯©Ø*5ØØñ\àð\ð ˜c‘]ð\ð ð	\ð
 ˜BŸI™I‘ð\ð ˜RŸY™Y™õ\ö6rF   rW   c            
       óp   ‡ — e Zd Zddej                  ddfdedededeej                     fˆ fd„Z	d	„ Z
ˆ xZS )
ÚFeatureBlockr   r^   NrY   Úlevelsr[   r+   c           	      óÐ  •— ||dœ}t         ‰| �  «        |}t        d|«      }|dk(  rt        t        j
                  d¬«      }	nt        t        j                  ddd¬«      }	t	        j                  «       | _        t        |«      D ]_  }
| j                  j                  d|
dz   › �t        |fd	|i|¤Ž«       |sŒ4| j                  j                  d
|
dz   › � |	«       «       |dz  }Œa y )Nr-   r   r_   r`   re   r2   ra   rX   r+   Úpool)r5   r6   r^   r   r;   rj   ri   Ú
SequentialÚblocksÚrangeÚ
add_moduler$   )r@   rY   rp   r[   r+   r.   r/   rA   Ú
reductionsÚpool_fnÚirD   s              €rE   r6   zFeatureBlock.__init__‚   sÐ   ø€ ð ¨Ñ/ˆÜ‰ÑÔØˆ
Ü�Q˜“ˆØ˜ÒÜœbŸl™l¸Ô:‰GäœbŸl™l¸À!ÈQÔOˆGÜ—m‘m“oˆŒÜ�v“ò 	 ˆAØ�K‰K×"Ñ" T¨!¨A©#¨ <´¸SÑ1\ÈIÐ1\ÐY[Ñ1\Ô]ÚØ—‘×&Ñ&¨¨a°©c¨U |±W³YÔ?Ø˜a‘‘
ñ		 rF   c                 ó$   — | j                  |«      S rH   )rt   rm   s     rE   rK   zFeatureBlock.forwardš   s   € Ø�{‰{˜1‹~ÐrF   )rL   rM   rN   r;   rP   r:   r9   r   rS   r6   rK   rT   rU   s   @rE   ro   ro   �   sP   ø„ ð Ø"Ø)+¯©ØØñ àð ð ð ð ð	 ð
 ˜BŸI™I‘õ ö0rF   ro   c            
       óŒ   ‡ — e Zd Zddej                  eddfdededeej                     deej                     fˆ fd„Z	d	„ Z
ˆ xZS )
ÚStemr2   é`   Nr&   r'   r+   r\   c                 ó”   •— t         ‰| �  «        ||dœ}t        j                  ||fddddœ|¤Ž| _        t        |f||dœ|¤Ž| _        y )Nr-   r2   r`   r   ra   )r+   r\   )r5   r6   r;   r<   Úconv1rW   Údown)	r@   r&   r'   r+   r\   r.   r/   rA   rD   s	           €rE   r6   zStem.__init__Ÿ   sU   ø€ ô 	‰ÑÔØ¨Ñ/ˆÜ—Y‘Y˜v wÐY¸AÀaÐQRÑYÐVXÑYˆŒ
Ü  Ð[°IÈ*Ñ[ÐXZÑ[ˆ�	rF   c                 óJ   — | j                  |«      }| j                  |«      }|S rH   )r   r€   rm   s     rE   rK   zStem.forward­   s!   € Ø�J‰J�q‹MˆØ�I‰I�a‹LˆØˆrF   )rL   rM   rN   r;   rP   r   r:   r   rS   r6   rK   rT   rU   s   @rE   r|   r|   ž   s`   ø„ ð ØØ)+¯©Ø*5ØØñ\àð\ð ð\ð ˜BŸI™I‘ð	\ð
 ˜RŸY™Y™õ\örF   r|   c                   ó|   ‡ — e Zd Z	 	 	 	 	 	 ddededeeef   dedededefˆ fd„Zdd	ee	j                     fd
„Zˆ xZS )ÚWindowAttentionGlobalrY   Ú	num_headsÚwindow_sizeÚ
use_globalÚqkv_biasÚ	attn_dropÚ	proj_dropc
                 ó  •— ||	dœ}
t         ‰| �  «        t        |«      }|| _        || _        ||z  | _        | j
                  dz  | _        || _        t        d||dœ|
¤Ž| _	        | j                  r"t        j                  ||dz  fd|i|
¤Ž| _        n!t        j                  ||dz  fd|i|
¤Ž| _        t        j                  |«      | _        t        j                  ||fi |
¤Ž| _        t        j                  |«      | _        y )Nr-   g      à¿)r…   r„   r`   r*   r2   r4   )r5   r6   r   r…   r„   Úhead_dimÚscaler†   r   Úrel_posr;   ÚLinearÚqkvÚDropoutrˆ   Úprojr‰   )r@   rY   r„   r…   r†   r‡   rˆ   r‰   r.   r/   rA   rD   s              €rE   r6   zWindowAttentionGlobal.__init__µ   së   ø€ ð ¨Ñ/ˆÜ‰ÑÔÜ Ó,ˆØ&ˆÔØ"ˆŒØ˜yÑ(ˆŒØ—]‘] dÑ*ˆŒ
Ø$ˆŒä!ÐU¨kÀYÑUÐRTÑUˆŒØ�?Š?Ü—y‘y  c¨A¡gÑC°HÐCÀÑCˆD�Hä—y‘y  c¨A¡gÑC°HÐCÀÑCˆDŒHÜŸ™ IÓ.ˆŒÜ—I‘I˜c 3Ñ-¨"Ñ-ˆŒ	ÜŸ™ IÓ.ˆ�rF   Úq_globalc                 ó0  — |j                   \  }}}| j                  rë|�ét        |j                   d   |j                   d   k(  d«       | j                  |«      }|j	                  ||d| j
                  | j                  «      j                  ddddd«      }|j                  d«      \  }}|j                  ||j                   d   z  ddd«      }	|	j	                  ||| j
                  | j                  «      j                  dddd«      }	n`| j                  |«      j	                  ||d| j
                  | j                  «      j                  ddddd«      }
|
j                  d«      \  }	}}|	| j                  z  }	|	|j                  dd«      j                  «       z  }| j                  |«      }|j                  d¬	«      }| j                  |«      }||z  j                  dd«      j	                  |||«      }| j!                  |«      }| j#                  |«      }|S )
Néÿÿÿÿz*x and q_global seq lengths should be equalr`   r   r2   r   é   éþÿÿÿ)rY   )Úshaper†   r   r�   Úreshaper„   r‹   ÚpermuteÚunbindÚrepeatrŒ   Ú	transposeÚ
contiguousr�   Úsoftmaxrˆ   r‘   r‰   )r@   rI   r’   ÚBÚNÚCÚkvÚkÚvÚqr�   Úattns               rE   rK   zWindowAttentionGlobal.forwardÓ   sÏ  € Ø—'‘'‰ˆˆ1ˆaØ�?Š?˜xÐ3Ü�A—G‘G˜B‘K 8§>¡>°"Ñ#5Ñ5Ð7cÔdà—‘˜!“ˆBØ—‘˜A˜q ! T§^¡^°T·]±]ÓC×KÑKÈAÈqÐRSÐUVÐXYÓZˆBØ—9‘9˜Q“<‰DˆAˆqà—‘  X§^¡^°AÑ%6Ñ 6¸¸1¸aÓ@ˆAØ—	‘	˜!˜Q §¡°·±Ó>×FÑFÀqÈ!ÈQÐPQÓR‰Aà—(‘(˜1“+×%Ñ% a¨¨A¨t¯~©~¸t¿}¹}ÓM×UÑUÐVWÐYZÐ\]Ð_`ÐbcÓdˆCØ—j‘j “m‰GˆAˆq�!Ø�—
‘
‰Nˆà�1—;‘;˜r 2Ó&×1Ñ1Ó3Ñ3ˆØ�|‰|˜DÓ!ˆØ�|‰| ˆ|Ó#ˆØ�~‰~˜dÓ#ˆà�A‰X× Ñ   AÓ&×.Ñ.¨q°!°QÓ7ˆØ�I‰I�a‹LˆØ�N‰N˜1ÓˆØˆrF   )TTç        r§   NNrH   )rL   rM   rN   r:   r   rR   rQ   r6   r   ÚtorchÚTensorrK   rT   rU   s   @rE   rƒ   rƒ   ³   s}   ø„ ð  $Ø!Ø!Ø!ØØñ/àð/ð ð/ð ˜s C˜x™ð	/ð
 ð/ð ð/ð ð/ð õ/ñ< 8¨E¯L©LÑ#9÷ rF   rƒ   r…   c                 óð   — | j                   \  }}}}| j                  |||d   z  |d   ||d   z  |d   |«      } | j                  dddddd«      j                  «       j                  d|d   |d   |«      }|S )Nr   r   r2   r`   r•   é   r”   ©r—   Úviewr™   r�   )rI   r…   rŸ   ÚHÚWr¡   Úwindowss          rE   Úwindow_partitionr±   î   sŠ   € Ø—‘�J€A€qˆ!ˆQØ	�‰ˆq�!�{ 1‘~Ñ% {°1¡~°q¸KÈ¹NÑ7JÈKÐXYÉNÐ\]Ó^€AØ�i‰i˜˜1˜a  A qÓ)×4Ñ4Ó6×;Ñ;¸BÀÈAÁÐP[Ð\]ÑP^Ð`aÓb€GØ€NrF   Úimg_sizec                 óê   — |\  }}| j                   d   }| j                  d||d   z  ||d   z  |d   |d   |«      }|j                  dddddd«      j                  «       j                  d|||«      }|S )Nr”   r   r   r2   r`   r•   r«   r¬   )r°   r…   r²   r®   r¯   r¡   rI   s          rE   Úwindow_reverser´   õ   s†   € à�D€A€qØ�‰�bÑ€AØ�‰�R˜˜k¨!™nÑ,¨a°;¸q±>Ñ.AÀ;ÈqÁ>ÐS^Ð_`ÑSaÐcdÓe€AØ	�	‰	�!�Q˜˜1˜a Ó#×.Ñ.Ó0×5Ñ5°b¸!¸QÀÓB€AØ€HrF   c                   ó0  ‡ — e Zd Zddddddddeej
                  ej                  ddfdedeeef   ded	ed
e	de
de
dee	   de	de	de	dedeej                     deej                     fˆ fd„Zddeej"                     fd„Zddeej"                     fd„Zˆ xZS )ÚGlobalContextVitBlocké   ç      @TNr§   rY   Ú	feat_sizer„   r…   Ú	mlp_ratior†   r‡   Úlayer_scaler‰   rˆ   Ú	drop_pathr)   r+   r\   c           
      ó¤  •— ||dœ}t         ‰| �  «        t        |«      }t        |«      }|| _        t	        |d   |d   z  |d   |d   z  z  «      | _         ||fi |¤Ž| _         ||f|||||
|	dœ|¤Ž| _        |�t        ||fi |¤Žnt        j                  «       | _        |dkD  rt        |«      nt        j                  «       | _         ||fi |¤Ž| _        t        d|t	        ||z  «      ||	dœ|¤Ž| _        |�t        ||fi |¤Žnt        j                  «       | _        |dkD  rt        |«      | _        y t        j                  «       | _        y )Nr-   r   r   )r„   r…   r†   r‡   rˆ   r‰   r§   )Úin_featuresÚhidden_featuresr+   Údropr4   )r5   r6   r   r…   r:   Únum_windowsrg   r¦   r   r;   rf   Úls1r   Ú
drop_path1rk   r   ÚmlpÚls2Ú
drop_path2)r@   rY   r¹   r„   r…   rº   r†   r‡   r»   r‰   rˆ   r¼   r)   r+   r\   r.   r/   rA   rD   s                     €rE   r6   zGlobalContextVitBlock.__init__ÿ   s`  ø€ ð& ¨Ñ/ˆÜ‰ÑÔÜ˜iÓ(ˆ	Ü Ó,ˆØ&ˆÔÜ 	¨!¡°¸A±Ñ >À9ÈQÁ<ÐS^Ð_`ÑSaÑCaÑbÓcˆÔá Ñ* rÑ*ˆŒ
ÙØð	
àØ#Ø!ØØØñ	
ð ñ	
ˆŒ	ð :EÐ9P”:˜c ;Ñ5°"Ò5ÔVX×VaÑVaÓVcˆŒØ1:¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒá Ñ* rÑ*ˆŒ
ÜÐx 3¼¸CÀ)¹OÓ8LÐXaÐhqÑxÐuwÑxˆŒØ9DÐ9P”:˜c ;Ñ5°"Ò5ÔVX×VaÑVaÓVcˆŒØ1:¸R²œ( 9Ó-ˆ�ÄRÇ[Á[Ã]ˆ�rF   r’   c                 ó
  — |j                   \  }}}}t        || j                  «      }|j                  d| j                  d   | j                  d   z  |«      }| j	                  ||«      }t        || j                  ||f«      }|S )Nr”   r   r   )r—   r±   r…   r­   r¦   r´   )	r@   rI   r’   rŸ   r®   r¯   r¡   Úx_winÚattn_wins	            rE   Ú_window_attnz"GlobalContextVitBlock._window_attn,  s   € Ø—W‘W‰
ˆˆ1ˆa�Ü   D×$4Ñ$4Ó5ˆØ—
‘
˜2˜t×/Ñ/°Ñ2°T×5EÑ5EÀaÑ5HÑHÈ!ÓLˆØ—9‘9˜U HÓ-ˆÜ˜8 T×%5Ñ%5¸¸1°vÓ>ˆØˆrF   c           
      ó  — || j                  | j                  | j                  | j                  |«      |«      «      «      z   }|| j	                  | j                  | j                  | j                  |«      «      «      «      z   }|S rH   )rÃ   rÂ   rÊ   rg   rÆ   rÅ   rÄ   rk   )r@   rI   r’   s      rE   rK   zGlobalContextVitBlock.forward4  se   € Ø�—‘ §¡¨×):Ñ):¸4¿:¹:Àa»=È(Ó)SÓ TÓUÑUˆØ�—‘ §¡¨¯©°$·*±*¸Q³-Ó)@Ó AÓBÑBˆØˆrF   rH   )rL   rM   rN   rƒ   r;   rP   Ú	LayerNormr:   r   rQ   rR   r   r   r   rS   r6   r¨   r©   rÊ   rK   rT   rU   s   @rE   r¶   r¶   þ   s  ø„ ð  !Ø!Ø#Ø!Ø+/Ø!Ø!Ø!Ø#8Ø)+¯©Ø*,¯,©,ØØñ#+Sàð+Sð ˜S #˜X‘ð+Sð ð	+Sð
 ð+Sð ð+Sð ð+Sð ð+Sð " %™ð+Sð ð+Sð ð+Sð ð+Sð !ð+Sð ˜BŸI™I‘ð+Sð ˜RŸY™Y™õ+SñZ¨°·±Ñ(>ó ñ 8¨E¯L©LÑ#9÷ rF   r¶   c            $       ó(  ‡ — e Zd Zdddddddddej                  ej
                  eddfdededed	eeef   d
eeef   de	de	de	de
de	dee
   de
de
deee
   e
f   deej                     deej                     deej                     f"ˆ fd„Zd„ Zˆ xZS )ÚGlobalContextVitStageTFr¸   Nr§   rY   Údepthr„   r¹   r…   Ú
downsampleÚglobal_normÚ
stage_normrº   r‡   r»   r‰   rˆ   r¼   r+   r\   Únorm_layer_clc                 óô  •— ||dœ}t         ‰| �  «        |r-t        d||dz  |dœ|¤Ž| _        |dz  }|d   dz  |d   dz  f}nt	        j
                  «       | _        || _        t        |«      }t        t        j                  t        |«      t        |«      z  «      «      }t        ||fi |¤Ž| _        |r	 ||fi |¤Žnt	        j
                  «       | _        t	        j                  t!        |«      D �cg c]6  }t#        d|||||	|
|dz  dk7  |||t%        |t&        «      r||   n|||dœ|¤Ž‘Œ8 c}«      | _        |r	 ||fi |¤Žnt	        j
                  «       | _        || _        || _        d| _        y c c}w )	Nr-   r`   )rY   rZ   r\   r   r   )rY   r„   r¹   r…   rº   r‡   r†   r»   r‰   rˆ   r¼   r+   r\   Fr4   )r5   r6   rW   rÐ   r;   rf   r¹   r   r:   ÚmathÚlog2Úminro   Úglobal_blockrÑ   Ú
ModuleListru   r¶   r8   Úlistrt   ÚnormrY   Úgrad_checkpointing)r@   rY   rÏ   r„   r¹   r…   rÐ   rÑ   rÒ   rº   r‡   r»   r‰   rˆ   r¼   r+   r\   rÓ   r.   r/   rA   Úfeat_levelsry   rD   s                          €rE   r6   zGlobalContextVitStage.__init__;  s“  ø€ ð, ¨Ñ/ˆÜ‰ÑÔÙÜ*ð ØØ˜a™Ø%ñð ñ	ˆDŒOð ˜‘'ˆCØ" 1™¨Ñ*¨I°a©L¸AÑ,=Ð>‰Iä Ÿk™k›mˆDŒOØ"ˆŒÜ Ó,ˆäœ$Ÿ)™)¤C¨	£N´S¸Ó5EÑ$EÓFÓGˆÜ(¨¨kÑ@¸RÑ@ˆÔÙ7B™=¨Ñ3°Ò3ÌÏÉËˆÔä—m‘mô" ˜5“\ö#%
ð" ô! "ð ØØ#Ø#Ø'Ø#Ø!Ø ™E Q™JØ'Ø#Ø#Ü*4°YÄÔ*E˜) Aš,È9Ø#Ø(ñð óò%
ó ˆŒñ& 1;‘M #Ñ,¨Ò,ÄÇÁÃˆŒ	ØˆŒØ"ˆŒØ"'ˆÕùò-%
s   Ã4;E5c                 óÂ  — | j                  |«      }| j                  |«      }|j                  dddd«      }| j                  |j                  dddd«      «      }| j                  D ]C  }| j
                  r,t        j                  j                  «       st        |||«      }Œ; |||«      }ŒE | j                  |«      }|j                  dddd«      j                  «       }|S )Nr   r`   r2   r   )rÐ   rØ   r™   rÑ   rt   rÜ   r¨   ÚjitÚis_scriptingr   rÛ   r�   )r@   rI   Úglobal_queryÚblks       rE   rK   zGlobalContextVitStage.forward}  sÍ   € à�O‰O˜AÓˆØ×(Ñ(¨Ó+ˆð �I‰I�a˜˜A˜qÓ!ˆØ×'Ñ'¨×(<Ñ(<¸QÀÀ1ÀaÓ(HÓIˆØ—;‘;ò 	)ˆCØ×&Ò&¬u¯y©y×/EÑ/EÔ/GÜ˜s A |Ó4‘á˜˜<Ó(‘ð		)ð
 �I‰I�a‹LˆØ�I‰I�a˜˜A˜qÓ!×,Ñ,Ó.ˆØˆrF   )rL   rM   rN   r;   rP   rÌ   r   r:   r   rR   rQ   r   r	   r   r   rS   r6   rK   rT   rU   s   @rE   rÎ   rÎ   :  s6  ø„ ð  $Ø %Ø$Ø!Ø!Ø+/Ø!Ø!Ø36Ø)+¯©Ø*,¯,©,Ø-8ØØñ)@(àð@(ð ð@(ð ð	@(ð
 ˜S #˜X‘ð@(ð ˜s C˜x™ð@(ð ð@(ð ð@(ð ð@(ð ð@(ð ð@(ð " %™ð@(ð ð@(ð ð@(ð ˜T %™[¨%Ð/Ñ0ð@(ð  ˜BŸI™I‘ð!@(ð" ˜RŸY™Y™ð#@(ð$   §	¡	™?õ%@(öDrF   rÎ   c            +       ót  ‡ — e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d-dedededeeeeef   f   deedf   deeeeedf   f      ded	eedf   d
eedf   dede	dee   dededededededededef*ˆ fd„Z
d.d„Zej                  j                  d„ «       Zej                  j                  d/d„«       Zej                  j                  d0d„«       Zej                  j                  dej&                  fd„«       Zd1dedee   fd„Z	 	 	 	 	 d2dej,                  d eeeee   f      d!e	d"e	d#ed$e	deeej,                     eej,                  eej,                     f   f   fd%„Z	 	 	 d3d eeee   f   d&e	d'e	fd(„Zdej,                  dej,                  fd)„Zd/d*e	fd+„Zdej,                  dej,                  fd,„Zˆ xZS )4r"   Úin_chansÚnum_classesÚglobal_poolr²   Úwindow_ratio.r…   Ú	embed_dimÚdepthsr„   rº   r‡   r»   Ú	drop_rateÚproj_drop_rateÚattn_drop_rateÚdrop_path_rateÚweight_initr+   r\   rÓ   Únorm_epsc                 ó¬  •— t         ‰!| �  «        ||dœ}t        |«      }t        t	        |«      |¬«      }t        t	        |«      |¬«      }g | _        t        |«      }t        d„ |D «       «      }|| _        || _	        || _
        || _        t        |«      }t        |d|dz
  z  z  «      x| _        | _        |� t!        |«      |«      }n:|€J ‚t         t!        |«      |«      D �cg c]  }|d   |z  |d   |z  f‘Œ c}«      }t#        d||||dœ|¤Ž| _        t'        ||d¬	«      }g }t)        |«      D ]¾  }||dz
  k(  }dt+        |dz
  d«      z  } |j-                  t/        di d
|| z  “d||   “d|	|   “d|d   | z  |d   | z  f“d||   “d|dk7  “d|“d|
“d|“d|“d|“d|“d||   “d|“d|“d|“|¤Ž«       | xj
                  t1        |d   j2                  d|dz   z  d|› �¬«      gz  c_        ŒÀ t5        j6                  |Ž | _        t;        | j                  |f||dœ|¤Ž| _        |r"t?        t        | j@                  |¬«      | «       y y c c}w ) Nr-   )Úepsc              3   ó&   K  — | ]	  }|d z  –— Œ y­w)r•   Nr4   )Ú.0Úds     rE   ú	<genexpr>z,GlobalContextVit.__init__.<locals>.<genexpr>±  s   è ø€ Ò3 Q˜!˜q�&Ñ3ùs   ‚r`   r   r   )r&   r'   r+   r\   T)Ú	stagewiserY   rÏ   r„   r¹   r…   rÐ   rÒ   rº   r‡   r»   r‰   rˆ   r¼   r+   r\   rÓ   r”   zstages.)Únum_chsr[   Úmodule©Ú	pool_typerê   )Úschemer4   )!r5   r6   r   r   r   Úfeature_infor   Útupleræ   rå   rä   rê   Úlenr:   Únum_featuresÚhead_hidden_sizer   r|   Ústemr   ru   r^   ÚappendrÎ   r7   rY   r;   rs   Ústagesr   Úheadr   Ú_init_weights)"r@   rä   rå   ræ   r²   rç   r…   rè   ré   r„   rº   r‡   r»   rê   rë   rì   rí   rî   r+   r\   rÓ   rï   r.   r/   rA   r¹   Ú
num_stagesÚrÚdprr  ry   Ú
last_stageÚstage_scalerD   s"                                    €rE   r6   zGlobalContextVit.__init__�  sý  ø€ ô4 	‰ÑÔØ¨Ñ/ˆÜ! )Ó,ˆ	Üœ^¨JÓ7¸XÔFˆ
Ü¤¨}Ó =À8ÔLˆØˆÔÜ˜XÓ&ˆÜÑ3¨(Ô3Ó3ˆ	Ø&ˆÔØ&ˆÔØ ˆŒØ"ˆŒÜ˜“[ˆ
Ü47¸	ÀAÈ*ÐWXÉ.ÑDYÑ8YÓ4ZÐZˆÔ˜DÔ1ØÐ"Ø/œ) JÓ/°Ó<‰KàÐ+Ð+Ð+ÜÐOdÌyÐYcÓOdÐeqÓOrÖ sÈ! (¨1¡+°Ñ"2°H¸Q±KÀ1Ñ4DÒ!EÒ sÓtˆKäð 
ØØØØ!ñ	
ð
 ñ
ˆŒ	ô (¨¸È$ÔOˆØˆÜ�zÓ"ò 	jˆAØ˜j¨1™nÑ,ˆJØœs 1 q¡5¨!›}Ñ,ˆKØ�M‰MÔ/ò Ø Ò+ðà˜Q’iðð $ Aš,ðð % Q™<¨;Ñ6¸	À!¹ÈÑ8SÑTð	ð
 (¨šNðð  š6ðñ &ðñ $ðñ "ðñ (ðñ )ðñ )ðð ˜aš&ðñ $ðñ &ðñ  ,Øñ#ô ð& ×Ò¤$¨v°b©z¯~©~ÈÈQÈqÉSÉÐ\cÐdeÐcfÐZgÔ"hÐ!iÑiÖð-	jô. —m‘m VÐ,ˆŒô # 4×#4Ñ#4°kÐtÈ[ÐdmÑtÐqsÑtˆŒ	áÜœ × 2Ñ 2¸;ÔGÈÕNð ùòQ !ts   Ã'Ic                 ó\  — |dk(  r«t        |t        j                  «      r�t        j                  j	                  |j
                  «       |j                  �Zd|v r,t        j                  j                  |j                  d¬«       y t        j                  j                  |j                  «       y y y t        |t        j                  «      rbt        j                  j                  |j
                  d¬«       |j                  �*t        j                  j                  |j                  «       y y y )NÚvitrÄ   g�íµ ÷Æ°>)Ústdg{®Gáz”?)	r8   r;   rŽ   ÚinitÚxavier_uniform_Úweightr*   Únormal_Úzeros_)r@   rø   Únamerû   s       rE   r  zGlobalContextVit._init_weightsç  sÄ   € à�UŠ?Ü˜&¤"§)¡)Ô,Ü—‘×'Ñ'¨¯©Ô6Ø—;‘;Ð*Ø ‘}ÜŸ™Ÿ™¨¯©¸˜Õ>äŸ™Ÿ™ v§{¡{Õ3ð	 +ð -ô ˜&¤"§)¡)Ô,Ü—‘—‘ §¡°3�Ô7Ø—;‘;Ð*Ü—G‘G—N‘N 6§;¡;Õ/ð +ð -rF   c                 ó€   ‡— | j                  «       D �‡�ch c]  \  Š}t        ˆfd„dD «       «      r‰’Œ c}}S c c}}w )Nc              3   ó&   •K  — | ]  }|‰v –— Œ
 y ­wrH   r4   )ró   Únr£   s     €rE   rõ   z3GlobalContextVit.no_weight_decay.<locals>.<genexpr>û  s   øè ø€ ÒS˜a�1˜”6ÑSùs   ƒ)Úrelative_position_bias_tablezrel_pos.mlp)Únamed_parametersÚany)r@   r£   Ú_s    ` rE   Úno_weight_decayz GlobalContextVit.no_weight_decay÷  sK   ø€ ð ×/Ñ/Ó1÷Uð UÙ�!�QÜÓSÐ#RÔSÔSò óUð 	Uùó Us   – :c                 ó    — t        dd¬«      }|S )Nz^stemz^stages\.(\d+))r  rt   )r7   )r@   ÚcoarseÚmatchers      rE   Úgroup_matcherzGlobalContextVit.group_matcherý  s   € äØØ$ô
ˆð ˆrF   c                 ó4   — | j                   D ]	  }||_        Œ y rH   )r  rÜ   )r@   ÚenableÚss      rE   Úset_grad_checkpointingz'GlobalContextVit.set_grad_checkpointing  s   € à—‘ò 	*ˆAØ#)ˆAÕ ñ	*rF   Úreturnc                 ó.   — | j                   j                  S rH   )r  Úfc)r@   s    rE   Úget_classifierzGlobalContextVit.get_classifier
  s   € à�y‰y�|‰|ÐrF   c                 ó²   — ||dœ}|| _         |€ | j                  j                  j                  }t	        | j
                  |f|| j                  dœ|¤Ž| _        y )Nr-   rù   )rå   r  ræ   rú   r   rÿ   rê   )r@   rå   ræ   r.   r/   rA   s         rE   Úreset_classifierz!GlobalContextVit.reset_classifier  sW   € Ø¨Ñ/ˆØ&ˆÔØÐØŸ)™)×/Ñ/×9Ñ9ˆKÜ" 4×#4Ñ#4°kÐyÈ[Ðdh×drÑdrÑyÐvxÑyˆ�	rF   rI   ÚindicesrÛ   Ú
stop_earlyÚ
output_fmtÚintermediates_onlyc                 ór  — |dv sJ d«       ‚g }t        t        | j                  «      |«      \  }}	| j                  |«      }t        j
                  j                  «       s|s| j                  }
n| j                  d|	dz    }
t        |
«      D ]#  \  }} ||«      }||v sŒ|j                  |«       Œ% |r|S ||fS )aï   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )ÚNCHWzOutput shape must be NCHW.Nr   )	r   rþ   r  r  r¨   rß   rà   Ú	enumerater  )r@   rI   r*  rÛ   r+  r,  r-  ÚintermediatesÚtake_indicesÚ	max_indexr  Úfeat_idxÚstages                rE   Úforward_intermediatesz&GlobalContextVit.forward_intermediates  sÂ   € ð* ˜YÑ&ÐDÐ(DÓDÐ&ØˆÜ"6´s¸4¿;¹;Ó7GÈÓ"QÑˆ�ið �I‰I�a‹LˆÜ�9‰9×!Ñ!Ô#©:Ø—[‘[‰Fà—[‘[  )¨a¡-Ð0ˆFä(¨Ó0ò 	(‰OˆH�eÙ�a“ˆAØ˜<Ò'Ø×$Ñ$ QÕ'ð	(ñ
 Ø Ð à�-ÐÐrF   Ú
prune_normÚ
prune_headc                 ó    — t        t        | j                  «      |«      \  }}| j                  d|dz    | _        |r| j                  dd«       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   Ú )r   rþ   r  r)  )r@   r*  r7  r8  r2  r3  s         rE   Úprune_intermediate_layersz*GlobalContextVit.prune_intermediate_layers?  sM   € ô #7´s¸4¿;¹;Ó7GÈÓ"QÑˆ�iØ—k‘k . 9¨q¡=Ð1ˆŒÙØ×!Ñ! ! RÔ(ØÐrF   c                 óJ   — | j                  |«      }| j                  |«      }|S rH   )r  r  rm   s     rE   Úforward_featuresz!GlobalContextVit.forward_featuresM  s!   € Ø�I‰I�a‹LˆØ�K‰K˜‹NˆØˆrF   Ú
pre_logitsc                 óN   — |r| j                  ||¬«      S | j                  |«      S )N)r>  )r  )r@   rI   r>  s      rE   Úforward_headzGlobalContextVit.forward_headR  s%   € Ù6@ˆt�y‰y˜ zˆyÓ2ÐRÀdÇiÁiÐPQÃlÐRrF   c                 óJ   — | j                  |«      }| j                  |«      }|S rH   )r=  r@  rm   s     rE   rK   zGlobalContextVit.forwardU  s'   € Ø×!Ñ! !Ó$ˆØ×Ñ˜aÓ ˆØˆrF   )r2   éè  r_   éà   )é    rD  é   rD  Né@   ©r2   r•   é   r«   ©r`   r•   é   rE  g      @TNr§   r§   r§   r§   r:  ÚgeluÚlayernorm2dÚ	layernormçñhãˆµøä>NN)r  ©F)T)NNN)NFFr/  F)r   FT)rL   rM   rN   r:   r9   r	   r   r   rQ   rR   r6   r  r¨   rß   Úignorer  r  r#  r;   rS   r'  r)  r©   r   r6  r;  r=  r@  rK   rT   rU   s   @rE   r"   r"   �  sM  ø„ ð Ø#Ø$Ø47Ø,<ØAEØØ&3Ø)6Ø"Ø!Ø+/Ø!Ø$&Ø$&Ø$&Ø!Ø#Ø+Ø!,Ø"ØØñ1UOàðUOð ðUOð ð	UOð
 ˜C  s¨C x¡Ð0Ñ1ðUOð    S ™/ðUOð " %¨¨U°3¸°8©_Ð(<Ñ"=Ñ>ðUOð ðUOð ˜#˜s˜(‘OðUOð ˜S #˜X‘ðUOð ðUOð ðUOð " %™ðUOð ðUOð "ðUOð  "ð!UOð" "ð#UOð$ ð%UOð& ð'UOð( ð)UOð* ð+UOð, õ-UOón0ð  ‡Y�Y×ÑñUó ðUð
 ‡Y�Y×Ñòó ðð ‡Y�Y×Ñò*ó ð*ð ‡Y�Y×Ñð §	¡	ò ó ðñz¨Cð z¸hÀs¹mó zð 8<ØØ$Ø$Ø',ñ( à�|‰|ð( ð ˜e C¨¨c© NÑ3Ñ4ð( ð ð	( ð
 ð( ð ð( ð !%ð( ð 
ˆt�E—L‘LÑ! 5¨¯©°t¸E¿L¹LÑ7IÐ)IÑ#JÐJÑ	Kó( ðX ./Ø$Ø#ñ	à˜3  S¡	˜>Ñ*ðð ðð ó	ð %§,¡,ð °5·<±<ó ñ
S¨$ó Sð˜Ÿ™ð ¨%¯,©,÷ rF   c                 óD   — t        t        | |fdt        dd¬«      i|¤Ž}|S )NÚfeature_cfg)r   r   r`   r2   T)Úout_indicesÚflatten_sequential)r   r"   r7   )ÚvariantÚ
pretrainedÚkwargsÚmodels       rE   Ú_create_gcvitrY  [  s3   € Ü Ü˜' :ñä \ÀdÔKðð ñ€Eð
 €LrF   c                 ó4   — | dddddt         t        dddd	d
œ|¥S )NrB  )r2   rC  rC  )r·   r·   g      ì?Úbicubicz
stem.conv1zhead.fcTz
apache-2.0)Úurlrå   Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚmeanr  Ú
first_convÚ
classifierÚfixed_input_sizeÚlicenser
   )r\  rW  s     rE   Ú_cfgrf  d  s6   € à 4°}ÐSYØ¨IÜ%Ô.BØ"°)Ø Øñð ðð rF   z}https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_xxtiny_224_nvidia-d1d86009.pth)r\  z|https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_xtiny_224_nvidia-274b92b7.pthz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_tiny_224_nvidia-ac783954.pthz|https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_small_224_nvidia-4e98afa2.pthz{https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights-morevit/gcvit_base_224_nvidia-f009139b.pth)zgcvit_xxtiny.in1kzgcvit_xtiny.in1kzgcvit_tiny.in1kzgcvit_small.in1kzgcvit_base.in1kr$  c                 ó8   — t        ddddœ|¤Ž}t        dd| i|¤ŽS )N)r`   r`   é   r`   rI  ©ré   r„   rV  r4   )Úgcvit_xxtiny©r7   rY  ©rV  rW  Úmodel_kwargss      rE   rj  rj  ~  s6   € äð ØØñð ñ€Lô ÑO°JÐOÀ,ÑOÐOrF   c                 ó8   — t        ddddœ|¤Ž}t        dd| i|¤ŽS )N)r2   r•   rh  r«   rI  ri  rV  r4   )Úgcvit_xtinyrk  rl  s      rE   ro  ro  ‡  s6   € äð ØØñð ñ€Lô ÑN°:ÐNÀÑNÐNrF   c                 ó8   — t        ddddœ|¤Ž}t        dd| i|¤ŽS )NrG  rI  ri  rV  r4   )Ú
gcvit_tinyrk  rl  s      rE   rq  rq  �  s6   € äð ØØñð ñ€Lô ÑM°*ÐMÀÑMÐMrF   c           	      ó>   — t        dddddddœ|¤Ž}t        d	d| i|¤ŽS )
NrG  )r2   rh  é   é   r}   r`   rN  ©ré   r„   rè   rº   r»   rV  r4   )Úgcvit_smallrk  rl  s      rE   rv  rv  ™  s?   € äð ØØ ØØØñð ñ€Lô ÑN°:ÐNÀÑNÐNrF   c           	      ó>   — t        dddddddœ|¤Ž}t        d	d| i|¤ŽS )
NrG  )r•   rJ  rE  rD  é€   r`   rN  ru  rV  r4   )Ú
gcvit_baserk  rl  s      rE   ry  ry  ¥  s?   € äð ØØ ØØØñð ñ€Lô ÑM°*ÐMÀÑMÐMrF   rO  )r:  )@rO   rÕ   Ú	functoolsr   Útypingr   r   r   r   r   r	   r¨   Útorch.nnr;   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   r   r   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_features_fxr   Ú_manipulater   r   Ú	_registryr    r!   Ú__all__rS   r$   rW   ro   r|   rƒ   r:   r±   r´   r¶   rÎ   r"   rY  rf  Údefault_cfgsrj  ro  rq  rv  ry  r4   rF   rE   ú<module>r†     s  ðñó* Ý ß ?× ?ã Ý ç A÷÷ ÷ õ õ +Ý +Ý 3ß 0ß <àÐ
€ô$�"—)‘)ô $ôN!�2—9‘9ô !ôH�2—9‘9ô ô:ˆ2�9‰9ô ô*8˜BŸI™Iô 8ðv U¨3°¨8¡_ó ð ð¨¨s°C¨x©ð ÀEÈ#ÈsÈ(ÁOò ó ðô9˜BŸI™Iô 9ôxR˜BŸI™Iô RôjI�r—y‘yô IóXó	ñ %Ùð LôMáð KôLáð JôKáð KôLáð JôKñ&ó €ð ñPÐ0@ò Pó ðPð ñOÐ/?ò Oó ðOð ñNÐ.>ò Nó ðNð ñOÐ/?ò Oó ðOð ñNÐ.>ò Nó ñNrF   