Ë
    éÓ^j   ã                   óŽ  — d Z ddlmZ ddlmZ ddlmZmZmZm	Z	m
Z
mZ ddlZddlmZ ddlmZmZ ddlmZmZmZmZmZmZ dd	lmZ dd
lmZ ddlmZmZ ddl m!Z!m"Z" dgZ# G d„ dejH                  «      Z% G d„ dejH                  «      Z& G d„ dejH                  «      Z' G d„ dejP                  «      Z) G d„ dejP                  «      Z* G d„ dejH                  «      Z+ G d„ dejH                  «      Z, G d„ dejH                  «      Z- G d„ dejH                  «      Z. G d „ d!ejH                  «      Z/ G d"„ d#ejH                  «      Z0 G d$„ d%ejH                  «      Z1 G d&„ dejH                  «      Z2 G d'„ d(e2«      Z3d)„ Z4 e5 e5d*d+d,d-¬.«       e5d*d+d/d0¬.«       e5d1d2d3d0¬.«       e5d4d2d,d0¬.«       e5d5d2d6d0¬.«       e5d5d2d6d0d7d8¬9«       e5d:d;d<d0d7d8¬9«       e5d=d;d>d0d7¬?«       e5d4d2d,d@d7¬?«       e5dAd;dBd@d7¬?«      ¬C«
      Z6dsdD„Z7dtdE„Z8 e!i dF e8dG¬H«      “dI e8dG¬H«      “dJ e8dG¬H«      “dK e8dG¬H«      “dL e8dG¬H«      “dM e8dGdN¬O«      “dP e8dGdN¬O«      “dQ e8dGdN¬O«      “dR e8dGdN¬O«      “dS e8dGdN¬O«      “dT e8dU¬V«      “dW e8dU¬V«      “dX e8dU¬V«      “dY e8dU¬V«      “dZ e8dU¬V«      “d[ e8dU¬V«      “d\ e8dU¬V«      “ e8dU¬V«       e8dU¬V«       e8dU¬V«      d]œ¥«      Z9e"dud^e2fd_„«       Z:e"dud^e2fd`„«       Z;e"dud^e2fda„«       Z<e"dud^e2fdb„«       Z=e"dud^e2fdc„«       Z>e"dud^e2fdd„«       Z?e"dud^e2fde„«       Z@e"dud^e2fdf„«       ZAe"dud^e2fdg„«       ZBe"dud^e2fdh„«       ZCe"dud^e2fdi„«       ZDe"dud^e2fdj„«       ZEe"dud^e2fdk„«       ZFe"dud^e2fdl„«       ZGe"dud^e2fdm„«       ZHe"dud^e2fdn„«       ZIe"dud^e2fdo„«       ZJe"dud^e2fdp„«       ZKe"dud^e2fdq„«       ZLe"dud^e2fdr„«       ZMy)vaÈ   LeViT

Paper: `LeViT: a Vision Transformer in ConvNet's Clothing for Faster Inference`
    - https://arxiv.org/abs/2104.01136

@article{graham2021levit,
  title={LeViT: a Vision Transformer in ConvNet's Clothing for Faster Inference},
  author={Benjamin Graham and Alaaeldin El-Nouby and Hugo Touvron and Pierre Stock and Armand Joulin and Herv'e J'egou and Matthijs Douze},
  journal={arXiv preprint arXiv:22104.01136},
  year={2021}
}

Adapted from official impl at https://github.com/facebookresearch/LeViT, original copyright bellow.

This version combines both conv/linear models and fixes torchscript compatibility.

Modifications and additions for timm hacked together by / Copyright 2021, Ross Wightman
é    )ÚOrderedDict)Úpartial)ÚDictÚListÚOptionalÚTupleÚTypeÚUnionN)ÚIMAGENET_DEFAULT_STDÚIMAGENET_DEFAULT_MEAN)Ú	to_ntupleÚ	to_2tupleÚget_act_layerÚDropPathÚtrunc_normal_Úndgridé   )Úbuild_model_with_cfg)Úfeature_take_indices)Ú
checkpointÚcheckpoint_seq)Úgenerate_default_cfgsÚregister_modelÚLevitc                   ó†   ‡ — e Zd Z	 	 	 	 	 	 	 	 ddedededededededefˆ fd	„Z ej                  «       d
„ «       Zd„ Z	ˆ xZ
S )ÚConvNormÚin_chsÚout_chsÚkernel_sizeÚstrideÚpaddingÚdilationÚgroupsÚbn_weight_initc           	      ó  •— |	|
dœ}t         ‰| �  «        t        j                  |||||||fddi|¤Ž| _        t        j
                  |fi |¤Ž| _        t        j                  j                  | j                  j                  |«       y ©N©ÚdeviceÚdtypeÚbiasF)
ÚsuperÚ__init__ÚnnÚConv2dÚlinearÚBatchNorm2dÚbnÚinitÚ	constant_Úweight)Úselfr   r   r   r    r!   r"   r#   r$   r(   r)   ÚddÚ	__class__s               €ú\/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/models/levit.pyr,   zConvNorm.__init__,   st   ø€ ð ¨Ñ/ˆÜ‰ÑÔÜ—i‘i ¨°¸fÀgÈxÐY_ÑrÐfkÐrÐoqÑrˆŒÜ—.‘. Ñ/¨BÑ/ˆŒä
�‰×Ñ˜$Ÿ'™'Ÿ.™.¨.Õ9ó    c           	      ó   — | j                   | j                  }}|j                  |j                  |j                  z   dz  z  }|j                  |d d …d d d f   z  }|j
                  |j                  |j                  z  |j                  |j                  z   dz  z  z
  }t        j                  |j                  d«      |j                  d«      |j                  dd  | j                   j                  | j                   j                  | j                   j                  | j                   j                  ¬«      }|j                  j                  j!                  |«       |j
                  j                  j!                  |«       |S )Nç      à?r   r   é   )r    r!   r"   r#   )r/   r1   r4   Úrunning_varÚepsr*   Úrunning_meanr-   r.   ÚsizeÚshaper    r!   r"   r#   ÚdataÚcopy_)r5   Úcr1   ÚwÚbÚms         r8   ÚfusezConvNorm.fuse@   s  € à—‘˜TŸW™Wˆ2ˆØ�I‰I˜Ÿ™¨"¯&©&Ñ0°SÑ8Ñ8ˆØ�H‰H�qš˜D $¨Ð,Ñ-Ñ-ˆØ�G‰G�b—o‘o¨¯	©	Ñ1°R·^±^ÀbÇfÁfÑ5LÐQTÑ4TÑTÑTˆÜ�I‰IØ�F‰F�1‹I�q—v‘v˜a“y !§'¡'¨!¨" +°d·k±k×6HÑ6HØ—K‘K×'Ñ'°$·+±+×2FÑ2FÈtÏ{É{×OaÑOaôcˆð 	
�‰�‰×Ñ˜AÔØ	�‰�‰×Ñ˜!ÔØˆr9   c                 óB   — | j                  | j                  |«      «      S ©N)r1   r/   ©r5   Úxs     r8   ÚforwardzConvNorm.forwardM   s   € Ø�w‰w�t—{‘{ 1“~Ó&Ð&r9   )r   r   r   r   r   r   NN©Ú__name__Ú
__module__Ú__qualname__ÚintÚfloatr,   ÚtorchÚno_gradrH   rM   Ú__classcell__©r7   s   @r8   r   r   +   s‘   ø„ ð
  !ØØØØØ$%ØØñ:àð:ð ð:ð ð	:ð
 ð:ð ð:ð ð:ð ð:ð "õ:ð( €U‡]�]ƒ_ñ
ó ð
ö'r9   r   c                   óh   ‡ — e Zd Z	 	 	 ddededefˆ fd„Z ej                  «       d„ «       Zd„ Z	ˆ xZ
S )Ú
LinearNormÚin_featuresÚout_featuresr$   c                 ó  •— ||dœ}t         ‰| �  «        t        j                  ||fddi|¤Ž| _        t        j
                  |fi |¤Ž| _        t        j                  j                  | j                  j                  |«       y r&   )
r+   r,   r-   ÚLinearr/   ÚBatchNorm1dr1   r2   r3   r4   )r5   rZ   r[   r$   r(   r)   r6   r7   s          €r8   r,   zLinearNorm.__init__R   sg   ø€ ð ¨Ñ/ˆÜ‰ÑÔÜ—i‘i ¨\ÑLÀÐLÈÑLˆŒÜ—.‘. Ñ4°Ñ4ˆŒä
�‰×Ñ˜$Ÿ'™'Ÿ.™.¨.Õ9r9   c                 ó6  — | j                   | j                  }}|j                  |j                  |j                  z   dz  z  }|j                  |d d …d f   z  }|j
                  |j                  |j                  z  |j                  |j                  z   dz  z  z
  }t        j                  |j                  d«      |j                  d«      «      }|j                  j                  j                  |«       |j
                  j                  j                  |«       |S )Nr;   r   r   )r/   r1   r4   r=   r>   r*   r?   r-   r]   r@   rB   rC   )r5   Úlr1   rE   rF   rG   s         r8   rH   zLinearNorm.fusea   sÍ   € à—‘˜TŸW™Wˆ2ˆØ�I‰I˜Ÿ™¨"¯&©&Ñ0°SÑ8Ñ8ˆØ�H‰H�qš˜D˜‘zÑ!ˆØ�G‰G�b—o‘o¨¯	©	Ñ1°R·^±^ÀbÇfÁfÑ5LÐQTÑ4TÑTÑTˆÜ�I‰I�a—f‘f˜Q“i §¡¨£Ó+ˆØ	�‰�‰×Ñ˜AÔØ	�‰�‰×Ñ˜!ÔØˆr9   c                 ó„   — | j                  |«      }| j                  |j                  dd«      «      j                  |«      S )Nr   r   )r/   r1   ÚflattenÚ
reshape_asrK   s     r8   rM   zLinearNorm.forwardl   s3   € Ø�K‰K˜‹NˆØ�w‰w�q—y‘y  A“Ó'×2Ñ2°1Ó5Ð5r9   )r   NNrN   rW   s   @r8   rY   rY   Q   sP   ø„ ð
 %&ØØñ:àð:ð ð:ð "õ	:ð €U‡]�]ƒ_ñó ðö6r9   rY   c                   ót   ‡ — e Zd Z	 	 	 	 	 d	dededededef
ˆ fd„Z ej                  «       d„ «       Z	d„ Z
ˆ xZS )
Ú
NormLinearrZ   r[   r*   ÚstdÚdropc                 óª  •— ||dœ}t         ‰	| �  «        t        j                  |fi |¤Ž| _        t        j
                  |«      | _        t        j                  ||fd|i|¤Ž| _        t        | j                  j                  |¬«       | j                  j                  �5t        j                  j                  | j                  j                  d«       y y )Nr'   r*   )rf   r   )r+   r,   r-   r^   r1   ÚDropoutrg   r]   r/   r   r4   r*   r2   r3   )
r5   rZ   r[   r*   rf   rg   r(   r)   r6   r7   s
            €r8   r,   zNormLinear.__init__r   s¢   ø€ ð ¨Ñ/ˆÜ‰ÑÔÜ—.‘. Ñ3°Ñ3ˆŒÜ—J‘J˜tÓ$ˆŒ	Ü—i‘i ¨\ÑKÀÐKÈÑKˆŒä�d—k‘k×(Ñ(¨cÕ2Ø�;‰;×ÑÐ'Ü�G‰G×Ñ˜dŸk™k×.Ñ.°Õ2ð (r9   c                 ó6  — | j                   | j                  }}|j                  |j                  |j                  z   dz  z  }|j
                  | j                   j                  | j                   j                  z  |j                  |j                  z   dz  z  z
  }|j                  |d d d …f   z  }|j
                  €$|| j                  j                  j                  z  }n<|j                  |d d …d f   z  j                  d«      | j                  j
                  z   }t        j                  |j                  d«      |j                  d«      «      }|j                  j                  j                  |«       |j
                  j                  j                  |«       |S )Nr;   éÿÿÿÿr   r   )r1   r/   r4   r=   r>   r*   r?   ÚTÚviewr-   r]   r@   rB   rC   )r5   r1   r`   rE   rF   rG   s         r8   rH   zNormLinear.fuse†   s-  € à—‘˜Ÿ™ˆAˆØ�I‰I˜Ÿ™¨"¯&©&Ñ0°SÑ8Ñ8ˆØ�G‰G�d—g‘g×*Ñ*¨T¯W©W¯^©^Ñ;¸r¿~¹~ÐPR×PVÑPVÑ?VÐ[^Ñ>^Ñ^Ñ^ˆØ�H‰H�q˜šq˜‘zÑ!ˆØ�6‰6ˆ>Ø�D—K‘K×&Ñ&×(Ñ(Ñ(‰Aà—‘˜Aša ˜g™JÑ&×,Ñ,¨RÓ0°4·;±;×3CÑ3CÑCˆAÜ�I‰I�a—f‘f˜Q“i §¡¨£Ó+ˆØ	�‰�‰×Ñ˜AÔØ	�‰�‰×Ñ˜!ÔØˆr9   c                 ó`   — | j                  | j                  | j                  |«      «      «      S rJ   )r/   rg   r1   rK   s     r8   rM   zNormLinear.forward•   s"   € Ø�{‰{˜4Ÿ9™9 T§W¡W¨Q£ZÓ0Ó1Ð1r9   )Tg{®Gáz”?ç        NN)rO   rP   rQ   rR   ÚboolrS   r,   rT   rU   rH   rM   rV   rW   s   @r8   re   re   q   sj   ø„ ð
 ØØØØñ3àð3ð ð3ð ð	3ð
 ð3ð õ3ð( €U‡]�]ƒ_ñó ðö2r9   re   c                   óL   ‡ — e Zd Z	 	 ddededeej                     fˆ fd„Zˆ xZS )ÚStem8r   r   Ú	act_layerc           	      ót  •— ||dœ}t         ‰| �  «        d| _        | j                  dt	        ||dz  dfdddœ|¤Ž«       | j                  d	 |«       «       | j                  d
t	        |dz  |dz  dfdddœ|¤Ž«       | j                  d |«       «       | j                  dt	        |dz  |dfdddœ|¤Ž«       y )Nr'   é   Úconv1é   é   r<   r   ©r    r!   Úact1Úconv2Úact2Úconv3©r+   r,   r    Ú
add_moduler   ©r5   r   r   rs   r(   r)   r6   r7   s          €r8   r,   zStem8.__init__š   sÁ   ø€ ð ¨Ñ/ˆÜ‰ÑÔØˆŒà�‰˜¤¨&°'¸Q±,ÀÐ!]È!ÐUVÑ!]ÐZ\Ñ!]Ô^Ø�‰˜¡	£Ô,Ø�‰˜¤¨'°Q©,¸À1¹ÀaÐ!cÐPQÐ[\Ñ!cÐ`bÑ!cÔdØ�‰˜¡	£Ô,Ø�‰˜¤¨'°Q©,¸ÀÐ!^È1ÐVWÑ!^Ð[]Ñ!^Õ_r9   ©NN©	rO   rP   rQ   rR   r	   r-   ÚModuler,   rV   rW   s   @r8   rr   rr   ™   s>   ø„ ð Øñ`àð`ð ð`ð ˜BŸI™I‘÷	`ñ `r9   rr   c                   óL   ‡ — e Zd Z	 	 ddededeej                     fˆ fd„Zˆ xZS )ÚStem16r   r   rs   c           	      óð  •— ||dœ}t         ‰| �  «        d| _        | j                  dt	        ||dz  dfdddœ|¤Ž«       | j                  d	 |«       «       | j                  d
t	        |dz  |dz  dfdddœ|¤Ž«       | j                  d |«       «       | j                  dt	        |dz  |dz  dfdddœ|¤Ž«       | j                  d |«       «       | j                  dt	        |dz  |dfdddœ|¤Ž«       y )Nr'   é   rv   ru   rx   r<   r   ry   rz   r{   rw   r|   r}   Úact3Úconv4r~   r€   s          €r8   r,   zStem16.__init__®   sÿ   ø€ ð ¨Ñ/ˆÜ‰ÑÔØˆŒà�‰˜¤¨&°'¸Q±,ÀÐ!]È!ÐUVÑ!]ÐZ\Ñ!]Ô^Ø�‰˜¡	£Ô,Ø�‰˜¤¨'°Q©,¸À1¹ÀaÐ!cÐPQÐ[\Ñ!cÐ`bÑ!cÔdØ�‰˜¡	£Ô,Ø�‰˜¤¨'°Q©,¸À1¹ÀaÐ!cÐPQÐ[\Ñ!cÐ`bÑ!cÔdØ�‰˜¡	£Ô,Ø�‰˜¤¨'°Q©,¸ÀÐ!^È1ÐVWÑ!^Ð[]Ñ!^Õ_r9   r�   r‚   rW   s   @r8   r…   r…   ­   s>   ø„ ð Øñ`àð`ð ð`ð ˜BŸI™I‘÷	`ñ `r9   r…   c            	       óN   ‡ — e Zd Z	 	 	 ddedeeeeef   f   defˆ fd„Zd„ Zˆ xZ	S )Ú
Downsampler    Ú
resolutionÚuse_poolc                 ó    •— t         ‰| �  «        || _        t        |«      | _        |rt        j                  d|dd¬«      | _        y d | _        y )Nrx   r   F)r    r!   Úcount_include_pad)r+   r,   r    r   rŒ   r-   Ú	AvgPool2dÚpool)r5   r    rŒ   r�   r(   r)   r7   s         €r8   r,   zDownsample.__init__Ä   sC   ø€ ô 	‰ÑÔØˆŒÜ# JÓ/ˆŒÙZb”B—L‘L ¨6¸1ÐPUÔVˆ�	Ðhlˆ�	r9   c                 óp  — |j                   \  }}}|j                  || j                  d   | j                  d   |«      }| j                  �6| j                  |j	                  dddd«      «      j	                  dddd«      }n$|d d …d d | j
                  …d d | j
                  …f   }|j                  |d|«      S )Nr   r   rx   r<   rk   )rA   rm   rŒ   r‘   Úpermuter    Úreshape)r5   rL   ÚBÚNÚCs        r8   rM   zDownsample.forwardÑ   s£   € Ø—'‘'‰ˆˆ1ˆaØ�F‰F�1�d—o‘o aÑ(¨$¯/©/¸!Ñ*<¸aÓ@ˆØ�9‰9Ð Ø—	‘	˜!Ÿ)™) A q¨!¨QÓ/Ó0×8Ñ8¸¸A¸qÀ!ÓD‰Aà’!‘]�t—{‘{�]¡M d§k¡k MÐ1Ñ2ˆAØ�y‰y˜˜B Ó"Ð"r9   )FNN)
rO   rP   rQ   rR   r
   r   rp   r,   rM   rV   rW   s   @r8   r‹   r‹   Ã   sJ   ø„ ð
 #ØØñmàðmð ˜c 5¨¨c¨¡?Ð2Ñ3ðmð õ	mö#r9   r‹   c                   óJ  ‡ — e Zd ZU eeej                  f   ed<   ddddej                  ddfde
de
d	e
d
edee
ee
e
f   f   dedeej                      fˆ fd„Z ej$                  «       dˆ fd„	«       Zdd„Zdd„Zdd„Zdd„Zdej0                  dej                  fd„Zd„ Zˆ xZS )Ú	AttentionÚattention_bias_cacheru   ç      @é   FNÚdimÚkey_dimÚ	num_headsÚ
attn_ratiorŒ   Úuse_convrs   c
                 óü  •— ||	dœ}
t         ‰| �  «        |rt        nt        }t	        |«      }|| _        || _        |dz  | _        || _        ||z  | _	        t        ||z  «      | _        t        ||z  «      |z  | _        || _         ||| j                  | j                  dz  z   fi |
¤Ž| _        t        j                   t#        d |«       fd || j                  |fddi|
¤Žfg«      «      | _        |d   |d   z  }t        j&                  t)        j*                  ||fi |
¤Ž«      | _        | j/                  d	t)        j*                  ||f|t(        j0                  ¬«      d
¬«       i | _        | j5                  «        y )Nr'   ç      à¿r<   ÚactÚlnr$   r   r   Úattention_bias_idxsF©Ú
persistent)r+   r,   r   rY   r   r¡   rŸ   Úscalerž   Úkey_attn_dimrR   Úval_dimÚval_attn_dimrŒ   Úqkvr-   Ú
Sequentialr   ÚprojÚ	ParameterrT   ÚemptyÚattention_biasesÚregister_bufferÚlongrš   Úreset_parameters)r5   r�   rž   rŸ   r    rŒ   r¡   rs   r(   r)   r6   Úln_layerr–   r7   s                €r8   r,   zAttention.__init__Þ   ss  ø€ ð ¨Ñ/ˆÜ‰ÑÔÙ'•8¬ZˆÜ˜zÓ*ˆ
à ˆŒØ"ˆŒØ ‘_ˆŒ
ØˆŒØ# iÑ/ˆÔÜ˜:¨Ñ/Ó0ˆŒÜ 
¨WÑ 4Ó5¸	ÑAˆÔØ$ˆŒá˜C ×!2Ñ!2°T×5FÑ5FÈÑ5JÑ!JÑQÈbÑQˆŒÜ—M‘M¤+Ø‘I“KÐ Ø‘8˜D×-Ñ-¨sÑKÀ1ÐKÈÑKÐLð/
ó #ó ˆŒ	ð
 �q‰M˜J q™MÑ)ˆÜ "§¡¬U¯[©[¸ÀAÑ-LÈÑ-LÓ MˆÔØ×ÑØ!¤5§;¡;°°1¨v¸fÌEÏJÉJÔ#WÐdið 	ô 	kà$&ˆÔ!ð 	×ÑÕr9   c                 óR   •— t         ‰| �  |«       |r| j                  ri | _        y y y rJ   ©r+   Útrainrš   ©r5   Úmoder7   s     €r8   r¹   zAttention.train  ó)   ø€ ä‰‰�dÔÙ�D×-Ò-Ø(*ˆDÕ%ð .ˆ4r9   Úreturnc                 óv   — t         j                  j                  | j                  «       | j	                  «        y©z"Initialize parameters and buffers.N©r-   r2   Úzeros_r²   Ú_init_buffers©r5   s    r8   rµ   zAttention.reset_parameters  ó$   € ä
�‰�‰�t×,Ñ,Ô-Ø×ÑÕr9   c           
      ó¢  — t        j                  t        t        j                  | j                  d   |t         j
                  ¬«      t        j                  | j                  d   |t         j
                  ¬«      «      «      j                  d«      }|ddd…df   |dddd…f   z
  j                  «       }|d   | j                  d   z  |d   z   }|S )ú5Compute relative position indices for attention bias.r   r'   r   .N)rT   Ústackr   ÚarangerŒ   r´   rb   Úabs)r5   r(   ÚposÚrel_poss       r8   Ú_compute_attention_bias_idxsz&Attention._compute_attention_bias_idxs  s¯   € ä�k‰kœ&Ü�L‰L˜Ÿ™¨Ñ+°FÄ%Ç*Á*ÔMÜ�L‰L˜Ÿ™¨Ñ+°FÄ%Ç*Á*ÔMó
ó ÷ ‰7�1‹:ð 	ð �sšA˜t�|Ñ$ s¨3°²a¨<Ñ'8Ñ8×=Ñ=Ó?ˆØ˜1‘: §¡°Ñ 2Ñ2°g¸a±jÑ@ˆØˆr9   c                 ó�   — | j                   j                  | j                  | j                   j                  ¬«      «       i | _        y©z.Compute and fill non-persistent buffer values.)r(   N©r¦   rC   rÌ   r(   rš   rÃ   s    r8   rÂ   zAttention._init_buffers  ó=   € à× Ñ ×&Ñ&Ø×-Ñ-°T×5MÑ5M×5TÑ5TÐ-ÓUô	
ð %'ˆÕ!r9   c                 ó$   — | j                  «        y©z"Initialize non-persistent buffers.N©rÂ   rÃ   s    r8   Úinit_non_persistent_buffersz%Attention.init_non_persistent_buffers#  ó   € à×ÑÕr9   r(   c                 ó4  — t         j                  j                  «       s| j                  r| j                  d d …| j
                  f   S t        |«      }|| j                  vr*| j                  d d …| j
                  f   | j                  |<   | j                  |   S rJ   ©rT   ÚjitÚ
is_tracingÚtrainingr²   r¦   Ústrrš   ©r5   r(   Ú
device_keys      r8   Úget_attention_biaseszAttention.get_attention_biases'  óƒ   € Ü�9‰9×ÑÔ! T§]¢]Ø×(Ñ(ª¨D×,DÑ,DÐ)DÑEÐEä˜V›ˆJØ ×!:Ñ!:Ñ:Ø8<×8MÑ8MÊaÐQU×QiÑQiÐNiÑ8j�×)Ñ)¨*Ñ5Ø×,Ñ,¨ZÑ8Ð8r9   c                 ó^  — | j                   rð|j                  \  }}}}| j                  |«      j                  || j                  d||z  «      j                  | j                  | j                  | j                  gd¬«      \  }}}|j                  dd«      |z  | j                  z  | j                  |j                  «      z   }	|	j                  d¬«      }	||	j                  dd«      z  j                  |d||«      }�n|j                  \  }}
}| j                  |«      j                  ||
| j                  d«      j                  | j                  | j                  | j                  gd¬«      \  }}}|j                  dddd«      }|j                  dddd«      }|j                  dddd«      }||z  | j                  z  | j                  |j                  «      z   }	|	j                  d¬«      }	|	|z  j                  dd«      j                  ||
| j                  «      }| j!                  |«      }|S )Nrk   r<   ©r�   éþÿÿÿrx   r   r   )r¡   rA   r­   rm   rŸ   Úsplitrž   r«   Ú	transposer©   rÞ   r(   Úsoftmaxr“   r”   r¬   r¯   )r5   rL   r•   r—   ÚHÚWÚqÚkÚvÚattnr–   s              r8   rM   zAttention.forward0  sì  € Ø�=Š=ØŸ™‰JˆAˆq�!�QØ—h‘h˜q“k×&Ñ&Ø�4—>‘> 2 q¨1¡uó.ß.3©e°T·\±\À4Ç<Á<ÐQU×Q]ÑQ]Ð4^Ðde¨eÓ.fñ ˆAˆq�!ð —K‘K  BÓ'¨!Ñ+¨t¯z©zÑ9¸D×<UÑ<UÐVW×V^ÑV^Ó<_Ñ_ˆDØ—<‘< B�<Ó'ˆDà�T—^‘^ B¨Ó+Ñ+×1Ñ1°!°R¸¸AÓ>ŠAà—g‘g‰GˆAˆq�!Ø—h‘h˜q“k×&Ñ&Ø�1�d—n‘n bó*ß*/©%°·±¸t¿|¹|ÈTÏ\É\Ð0ZÐ`a¨%Ó*bñ ˆAˆq�!à—	‘	˜!˜Q  1Ó%ˆAØ—	‘	˜!˜Q  1Ó%ˆAØ—	‘	˜!˜Q  1Ó%ˆAà�q‘5˜4Ÿ:™:Ñ%¨×(AÑ(AÀ!Ç(Á(Ó(KÑKˆDØ—<‘< B�<Ó'ˆDà˜‘×$Ñ$ Q¨Ó*×2Ñ2°1°a¸×9JÑ9JÓKˆAØ�I‰I�a‹LˆØˆr9   ©T©r½   NrJ   ©rO   rP   rQ   r   rÛ   rT   ÚTensorÚ__annotations__r-   ÚSiLUrR   rS   r
   r   rp   r	   rƒ   r,   rU   r¹   rµ   rÌ   rÂ   rÔ   r(   rÞ   rM   rV   rW   s   @r8   r™   r™   Û   sæ   ø… Ø˜s E§L¡LÐ0Ñ1Ó1ð Ø "Ø68Ø"Ø)+¯©ØØñ' àð' ð ð' ð ð	' ð
 ð' ð ˜c 5¨¨c¨¡?Ð2Ñ3ð' ð ð' ð ˜BŸI™I‘õ' ðR €U‡]�]ƒ_ô+ó ð+ó
ó
ó'óð9¨5¯<©<ð 9¸E¿L¹Ló 9ör9   r™   c                   óZ  ‡ — e Zd ZU eeej                  f   ed<   ddddddej                  ddf	de
d	e
d
e
de
dede
dee
ee
e
f   f   dededeej                      fˆ fd„Z ej$                  «       dˆ fd„	«       Zdd„Zdd„Zdd„Zdd„Zdej0                  dej                  fd„Zd„ Zˆ xZS )ÚAttentionDownsamplerš   ru   ç       @r<   rœ   FNÚin_dimÚout_dimrž   rŸ   r    r    rŒ   r¡   r�   rs   c                 óD  •— ||dœ}t         ‰| �  «        t        |«      }|| _        || _        || _        || _        ||z  | _        t        ||z  «      | _	        | j                  | j
                  z  | _
        |dz  | _        || _        | j                  r,t        }t        t        j                   |	rdnd|	rdndd¬«      }nt"        }t        t$        f||	dœ|¤Ž} ||| j                  | j                  z   fi |¤Ž| _        t        j(                  t+        d	 ||¬
«      fd ||| j                  fi |¤Žfg«      «      | _        t        j(                  t+        d |
«       fd || j                  |fi |¤Žfg«      «      | _        |d   |d   z  }|d    |z   |d    |z   z  }t        j0                  t3        j4                  ||fi |¤Ž«      | _        | j9                  dt3        j4                  ||f|t2        j:                  ¬«      d¬«       i | _        | j?                  «        y )Nr'   r£   rx   r   r   F)r   r!   r�   )rŒ   r�   Údown)r    r¥   r¤   r¦   r§   ) r+   r,   r   r    rŒ   rŸ   rž   rª   rR   r«   r¬   r©   r¡   r   r   r-   r�   rY   r‹   Úkvr®   r   rè   r¯   r°   rT   r±   r²   r³   r´   rš   rµ   )r5   rõ   rö   rž   rŸ   r    r    rŒ   r¡   r�   rs   r(   r)   r6   r¶   Ú	sub_layerÚN_kÚN_qr7   s                     €r8   r,   zAttentionDownsample.__init__M  s  ø€ ð ¨Ñ/ˆÜ‰ÑÔÜ˜zÓ*ˆ
àˆŒØ$ˆŒØ"ˆŒØˆŒØ# iÑ/ˆÔÜ˜:¨Ñ/Ó0ˆŒØ ŸL™L¨4¯>©>Ñ9ˆÔØ ‘_ˆŒ
Ø ˆŒà�=Š=ÜˆHÜÜ—‘Ù!)™A¨q¹x¹!ÈQÐbgôi‰Iô "ˆHÜ¤
Ð[°zÈHÑ[ÐXZÑ[ˆIá˜6 4×#4Ñ#4°t×7HÑ7HÑ#HÑOÈBÑOˆŒÜ—‘œ{Ø‘Y fÔ-Ð.Ø‘8˜F D×$5Ñ$5Ñ<¸Ñ<Ð=ð,
ó  ó ˆŒô —M‘M¤+Ø‘I“KÐ Ø‘8˜D×-Ñ-¨wÑ=¸"Ñ=Ð>ð/
ó #ó ˆŒ	ð
 ˜‰m˜j¨™mÑ+ˆØ˜A‘� &Ñ(Ð)¨z¸!©}¨nÀÑ.FÐ,GÑGˆÜ "§¡¬U¯[©[¸ÀCÑ-NÈ2Ñ-NÓ OˆÔØ×ÑÐ2´E·K±KÀÀcÀ
ÐSYÔaf×akÑakÔ4lÐy~ÐÔØ$&ˆÔ!ð 	×ÑÕr9   c                 óR   •— t         ‰| �  |«       |r| j                  ri | _        y y y rJ   r¸   rº   s     €r8   r¹   zAttentionDownsample.train†  r¼   r9   r½   c                 óv   — t         j                  j                  | j                  «       | j	                  «        yr¿   rÀ   rÃ   s    r8   rµ   z$AttentionDownsample.reset_parametersŒ  rÄ   r9   c                 óò  — t        j                  t        t        j                  | j                  d   |t         j
                  ¬«      t        j                  | j                  d   |t         j
                  ¬«      «      «      j                  d«      }t        j                  t        t        j                  d| j                  d   | j                  |t         j
                  ¬«      t        j                  d| j                  d   | j                  |t         j
                  ¬«      «      «      j                  d«      }|ddd…df   |dddd…f   z
  j                  «       }|d   | j                  d   z  |d   z   }|S )rÆ   r   r'   r   )Ústepr(   r)   .N)	rT   rÇ   r   rÈ   rŒ   r´   rb   r    rÉ   )r5   r(   Úk_posÚq_posrË   s        r8   rÌ   z0AttentionDownsample._compute_attention_bias_idxs‘  s$  € ä—‘œFÜ�L‰L˜Ÿ™¨Ñ+°FÄ%Ç*Á*ÔMÜ�L‰L˜Ÿ™¨Ñ+°FÄ%Ç*Á*ÔMó
ó ÷ ‰7�1‹:ð 	ô —‘œFÜ�L‰L˜˜DŸO™O¨AÑ.°T·[±[ÈÔW\×WaÑWaÔbÜ�L‰L˜˜DŸO™O¨AÑ.°T·[±[ÈÔW\×WaÑWaÔbó
ó ÷ ‰7�1‹:ð 	ð ˜ša ˜Ñ&¨¨s°Dº!¨|Ñ)<Ñ<×AÑAÓCˆØ˜1‘: §¡°Ñ 2Ñ2°g¸a±jÑ@ˆØˆr9   c                 ó�   — | j                   j                  | j                  | j                   j                  ¬«      «       i | _        yrÎ   rÏ   rÃ   s    r8   rÂ   z!AttentionDownsample._init_buffersŸ  rÐ   r9   c                 ó$   — | j                  «        yrÒ   rÓ   rÃ   s    r8   rÔ   z/AttentionDownsample.init_non_persistent_buffers¦  rÕ   r9   r(   c                 ó4  — t         j                  j                  «       s| j                  r| j                  d d …| j
                  f   S t        |«      }|| j                  vr*| j                  d d …| j
                  f   | j                  |<   | j                  |   S rJ   r×   rÜ   s      r8   rÞ   z(AttentionDownsample.get_attention_biasesª  rß   r9   c                 óp  — | j                   �rO|j                  \  }}}}|dz
  | j                  z  dz   |dz
  | j                  z  dz   }}| j                  |«      j	                  || j
                  d||z  «      j                  | j                  | j                  gd¬«      \  }}	| j                  |«      j	                  || j
                  | j                  d«      }
|
j                  dd«      |z  | j                  z  | j                  |j                  «      z   }|j                  d¬«      }|	|j                  dd«      z  j                  || j                   ||«      }�nH|j                  \  }}}| j                  |«      j	                  ||| j
                  d«      j                  | j                  | j                  gd¬«      \  }}	|j#                  dddd«      }|	j#                  dddd«      }	| j                  |«      j	                  |d| j
                  | j                  «      j#                  dddd«      }
|
|z  | j                  z  | j                  |j                  «      z   }|j                  d¬«      }||	z  j                  dd«      j                  |d| j                   «      }| j%                  |«      }|S )Nr   rk   r<   rá   râ   rx   r   )r¡   rA   r    rù   rm   rŸ   rã   rž   r«   rè   rä   r©   rÞ   r(   rå   r”   r¬   r“   r¯   )r5   rL   r•   r—   ræ   rç   ÚHHÚWWré   rê   rè   rë   r–   s                r8   rM   zAttentionDownsample.forward³  sK  € Ø�=‹=ØŸ™‰JˆAˆq�!�QØ˜!‘e §¡Ñ+¨aÑ/°!°a±%¸D¿K¹KÑ1GÈ!Ñ1K�ˆBØ—7‘7˜1“:—?‘? 1 d§n¡n°b¸!¸a¹%Ó@×FÑFÈÏÉÐVZ×VbÑVbÐGcÐijÐFÓk‰DˆAˆqØ—‘�q“	—‘˜q $§.¡.°$·,±,ÀÓCˆAà—K‘K  BÓ'¨!Ñ+¨t¯z©zÑ9¸D×<UÑ<UÐVW×V^ÑV^Ó<_Ñ_ˆDØ—<‘< B�<Ó'ˆDà�T—^‘^ B¨Ó+Ñ+×4Ñ4°Q¸×8IÑ8IÈ2ÈrÓRŠAà—g‘g‰GˆAˆq�!Ø—7‘7˜1“:—?‘? 1 a¨¯©¸Ó<×BÑBÀDÇLÁLÐRV×R^ÑR^ÐC_ÐefÐBÓg‰DˆAˆqØ—	‘	˜!˜Q  1Ó%ˆAØ—	‘	˜!˜Q  1Ó%ˆAØ—‘�q“	—‘˜q " d§n¡n°d·l±lÓC×KÑKÈAÈqÐRSÐUVÓWˆAà�q‘5˜4Ÿ:™:Ñ%¨×(AÑ(AÀ!Ç(Á(Ó(KÑKˆDØ—<‘< B�<Ó'ˆDà˜‘×$Ñ$ Q¨Ó*×2Ñ2°1°b¸$×:KÑ:KÓLˆAØ�I‰I�a‹LˆØˆr9   rì   rí   rJ   rî   rW   s   @r8   ró   ró   J  s
  ø… Ø˜s E§L¡LÐ0Ñ1Ó1ð Ø #ØØ68Ø"Ø"Ø)+¯©ØØñ7 àð7 ð ð7 ð ð	7 ð
 ð7 ð ð7 ð ð7 ð ˜c 5¨¨c¨¡?Ð2Ñ3ð7 ð ð7 ð ð7 ð ˜BŸI™I‘õ7 ðr €U‡]�]ƒ_ô+ó ð+ó
ó
ó'óð9¨5¯<©<ð 9¸E¿L¹Ló 9ör9   ró   c                   óŒ   ‡ — e Zd ZdZdddej
                  dddfdedee   dee   ded	e	ej                     d
efˆ fd„Zd„ Zˆ xZS )ÚLevitMlpzL MLP for Levit w/ normalization + ability to switch btw conv and linear
    NFro   rZ   Úhidden_featuresr[   r¡   rs   rg   c	                 óì   •— ||dœ}	t         ‰| �  «        |xs |}|xs |}|rt        nt        }
 |
||fi |	¤Ž| _         |«       | _        t        j                  |«      | _         |
||fddi|	¤Ž| _	        y )Nr'   r$   r   )
r+   r,   r   rY   Úln1r¤   r-   ri   rg   Úln2)r5   rZ   r  r[   r¡   rs   rg   r(   r)   r6   r¶   r7   s              €r8   r,   zLevitMlp.__init__Ð  s{   ø€ ð ¨Ñ/ˆÜ‰ÑÔØ#Ò2 {ˆØ)Ò8¨[ˆÙ'•8¬Zˆá˜K¨Ñ?¸BÑ?ˆŒÙ“;ˆŒÜ—J‘J˜tÓ$ˆŒ	Ù˜O¨\ÑRÈ!ÐRÈrÑRˆ�r9   c                 óŽ   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|S rJ   )r  r¤   rg   r  rK   s     r8   rM   zLevitMlp.forwardæ  s;   € Ø�H‰H�Q‹KˆØ�H‰H�Q‹KˆØ�I‰I�a‹LˆØ�H‰H�Q‹KˆØˆr9   )rO   rP   rQ   Ú__doc__r-   rñ   rR   r   rp   r	   rƒ   rS   r,   rM   rV   rW   s   @r8   r
  r
  Í  s~   ø„ ñð
 .2Ø*.Ø"Ø)+¯©ØØØñSàðSð & c™]ðSð # 3™-ð	Sð
 ðSð ˜BŸI™I‘ðSð õSö,r9   r
  c                   óÐ   ‡ — e Zd Zdddej                  dddddddfded	ed
ededededeej                     de	eej                        de
eeeef   f   dededefˆ fd„Zd„ Zˆ xZS )ÚLevitDownsampleru   r›   rô   Nrœ   Fro   rõ   rö   rž   rŸ   r    Ú	mlp_ratiors   Úattn_act_layerrŒ   r¡   r�   Ú	drop_pathc                 ó  •— ||dœ}t         ‰| �  «        |xs |}t        d|||||||	|
|dœ	|¤Ž| _        t	        |t        ||z  «      f|
|dœ|¤Ž| _        |dkD  rt        |«      | _
        y t        j                  «       | _
        y )Nr'   )	rõ   rö   rž   rŸ   r    rs   rŒ   r¡   r�   ©r¡   rs   ro   © )r+   r,   ró   Úattn_downsampler
  rR   Úmlpr   r-   ÚIdentityr  )r5   rõ   rö   rž   rŸ   r    r  rs   r  rŒ   r¡   r�   r  r(   r)   r6   r7   s                   €r8   r,   zLevitDownsample.__init__ï  s°   ø€ ð" ¨Ñ/ˆÜ‰ÑÔØ'Ò4¨9ˆä2ð  
ØØØØØ!Ø$Ø!ØØñ 
ð ñ 
ˆÔô ØÜ�˜)Ñ#Ó$ð
ð Øñ	
ð
 ñ
ˆŒð 1:¸B²œ )Ó,ˆ�ÄBÇKÁKÃMˆ�r9   c                 ón   — | j                  |«      }|| j                  | j                  |«      «      z   }|S rJ   )r  r  r  rK   s     r8   rM   zLevitDownsample.forward  s2   € Ø× Ñ  Ó#ˆØ�—‘˜tŸx™x¨›{Ó+Ñ+ˆØˆr9   )rO   rP   rQ   r-   rñ   rR   rS   r	   rƒ   r   r
   r   rp   r,   rM   rV   rW   s   @r8   r  r  î  sß   ø„ ð Ø "Ø!Ø)+¯©Ø8<Ø68Ø"Ø"Ø!ØØñ)Ràð)Rð ð)Rð ð	)Rð
 ð)Rð ð)Rð ð)Rð ˜BŸI™I‘ð)Rð % T¨"¯)©)¡_Ñ5ð)Rð ˜c 5¨¨c¨¡?Ð2Ñ3ð)Rð ð)Rð ð)Rð õ)RöVr9   r  c                   óÆ   ‡ — e Zd Zdddddej                  ddddf
ded	ed
edededeeeeef   f   de	de
ej                     dee
ej                        defˆ fd„Zd„ Zˆ xZS )Ú
LevitBlockru   r›   rô   rœ   FNro   r�   rž   rŸ   r    r  rŒ   r¡   rs   r  r  c                 óZ  •— ||dœ}t         ‰| �  «        |	xs |}	t        d|||||||	dœ|¤Ž| _        |
dkD  rt	        |
«      nt        j                  «       | _        t        |t        ||z  «      f||dœ|¤Ž| _
        |
dkD  rt	        |
«      | _        y t        j                  «       | _        y )Nr'   )r�   rž   rŸ   r    rŒ   r¡   rs   ro   r  r  )r+   r,   r™   rë   r   r-   r  Ú
drop_path1r
  rR   r  Ú
drop_path2)r5   r�   rž   rŸ   r    r  rŒ   r¡   rs   r  r  r(   r)   r6   r7   s                 €r8   r,   zLevitBlock.__init__!  sÂ   ø€ ð ¨Ñ/ˆÜ‰ÑÔØ'Ò4¨9ˆäð 	ØØØØ!Ø!ØØ$ñ	ð ñ	ˆŒ	ð 2;¸R²œ( 9Ô-ÄRÇ[Á[Ã]ˆŒäØÜ��i‘Ó ð
ð Øñ	
ð
 ñ
ˆŒð 2;¸R²œ( 9Ó-ˆ�ÄRÇ[Á[Ã]ˆ�r9   c                 ó’   — || j                  | j                  |«      «      z   }|| j                  | j                  |«      «      z   }|S rJ   )r   rë   r!  r  rK   s     r8   rM   zLevitBlock.forwardI  s=   € Ø�—‘ §	¡	¨!£Ó-Ñ-ˆØ�—‘ §¡¨£Ó,Ñ,ˆØˆr9   )rO   rP   rQ   r-   rñ   rR   rS   r
   r   rp   r	   rƒ   r   r,   rM   rV   rW   s   @r8   r  r     sÆ   ø„ ð
 Ø "Ø!Ø68Ø"Ø)+¯©Ø8<Ø!ØØñ&Sàð&Sð ð&Sð ð	&Sð
 ð&Sð ð&Sð ˜c 5¨¨c¨¡?Ð2Ñ3ð&Sð ð&Sð ˜BŸI™I‘ð&Sð % T¨"¯)©)¡_Ñ5ð&Sð õ&SöPr9   r  c                   óÖ   ‡ — e Zd Zddddej                  dddddddfd	ed
ededededededeej                     de	eej                        de
eeeef   f   dededefˆ fd„Zd„ Zˆ xZS )Ú
LevitStagerw   ru   r›   Nrœ   Ú Fro   rõ   rö   rž   ÚdepthrŸ   r    r  rs   r  rŒ   Ú
downsampler¡   r  c                 ó†  •— ||dœ}t         ‰| �  «        t        |
«      }
|r8t        ||f|||z  dd||	|
||dœ	|¤Ž| _        |
D �cg c]  }|dz
  dz  dz   ‘Œ }
}n ||k(  sJ ‚t        j                  «       | _        g }t        |«      D ]  }|t        ||f|||||	|
||dœ|¤Žgz  }Œ t        j                  |Ž | _
        y c c}w )Nr'   r›   rô   )	rž   rŸ   r    r  rs   r  rŒ   r¡   r  r   r<   )rŸ   r    r  rs   r  rŒ   r¡   r  )r+   r,   r   r  r'  r-   r  Úranger  r®   Úblocks)r5   rõ   rö   rž   r&  rŸ   r    r  rs   r  rŒ   r'  r¡   r  r(   r)   r6   Úrr*  Ú_r7   s                       €r8   r,   zLevitStage.__init__P  s  ø€ ð$ ¨Ñ/ˆÜ‰ÑÔÜ˜zÓ*ˆ
áÜ-ØØðð  Ø  GÑ+ØØØ#Ø-Ø%Ø!Ø#ñð ñˆDŒOð 5?Ö?¨q˜1˜q™5 Q™,¨Ó*Ð?ˆJÑ?à˜WÒ$Ð$Ð$Ü Ÿk™k›mˆDŒOàˆÜ�u“ò 	ˆAØ”zØØðð $Ø%Ø#Ø#Ø-Ø%Ø!Ø#ñð ñð ñ ‰Fð	ô —m‘m VÐ,ˆ�ùò) @s   ÁB>c                 óJ   — | j                  |«      }| j                  |«      }|S rJ   )r'  r*  rK   s     r8   rM   zLevitStage.forward‹  s"   € Ø�O‰O˜AÓˆØ�K‰K˜‹NˆØˆr9   )rO   rP   rQ   r-   rñ   rR   rS   r	   rƒ   r   r
   r   rÛ   rp   r,   rM   rV   rW   s   @r8   r$  r$  O  sß   ø„ ð ØØ #Ø"Ø)+¯©Ø8<Ø68Ø Ø"Ø!ØØñ!9-àð9-ð ð9-ð ð	9-ð
 ð9-ð ð9-ð ð9-ð ð9-ð ˜BŸI™I‘ð9-ð % T¨"¯)©)¡_Ñ5ð9-ð ˜c 5¨¨c¨¡?Ð2Ñ3ð9-ð ð9-ð ð9-ð õ9-övr9   r$  c            '       óp  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d0deeeeef   f   dededeedf   ded	eedf   d
eeeedf   f   deeeedf   f   deeeedf   f   dee	j                     dee   dedededee   dedededef&ˆ fd„Zd1defd„Zd1de	j                  deddfd„Zej"                  j$                  d„ «       Zej"                  j$                  d2d„«       Zej"                  j$                  d1d„«       Zej"                  j$                  de	j                  fd „«       Zd3dedee   fd!„Z	 	 	 	 	 d4d"ej0                  d#eeeee   f      d$ed%ed&ed'edeeej0                     eej0                  eej0                     f   f   fd(„Z	 	 	 d5d#eeee   f   d)ed*efd+„Zd,„ Zd2d-efd.„Zd/„ Zˆ xZS )6r   zß Vision Transformer with support for patch or hybrid CNN input stage

    NOTE: distillation is defaulted to True since pretrained weights use it, will cause problems
    w/ train scripts that don't take tuple outputs,
    NÚimg_sizeÚin_chansÚnum_classesÚ	embed_dim.rž   r&  rŸ   r    r  Ústem_backboneÚstem_strideÚ	stem_typeÚdown_oprs   r  r¡   Úglobal_poolÚ	drop_rateÚdrop_path_ratec                 óä  •— t         ‰ | �  «        ||dœ}t        |«      }t        |xs |«      }|| _        || _        || _        || _        |d   x| _        | _        || _	        || _
        d| _        g | _        t        |«      }t        |«      |k(  sJ ‚ t        |«      |«      } t        |«      |«      } t        |«      |	«      }	|
�|dk\  sJ ‚|
| _        |}nP|dv sJ ‚|dk(  rt!        ||d   fd|i|¤Ž| _        nt#        ||d   fd|i|¤Ž| _        | j                  j$                  }t'        t)        t+        |«      t+        |«      «      D ��cg c]
  \  }}||z  ‘Œ c}}«      }|d   }g }t-        |«      D ]’  }|dkD  rdnd	}|t/        |||   |f||   ||   ||   |	|   |||||dk(  r|nd
|dœ
|¤Žgz  }||z  }t'        |D �cg c]  }|d	z
  |z  d	z   ‘Œ c}«      }| xj                  t1        ||   |d|› �¬«      gz  c_        ||   }Œ” t3        j4                  |Ž | _        |dkD  rt9        |d   |fd|i|¤Žnt3        j:                  «       | _        | j?                  d¬«       y c c}}w c c}w )Nr'   rk   Fr<   )Ús16Ús8r;  r   rs   r   r%  )
r&  rŸ   r    r  rs   r  rŒ   r¡   r'  r  zstages.)Únum_chsÚ	reductionÚmodulerg   ©Úneeds_reset) r+   r,   r   r¡   r1  r0  r7  Únum_featuresÚhead_hidden_sizer2  r8  Úgrad_checkpointingÚfeature_infoÚlenr   Ústemr…   rr   r    ÚtupleÚzipr   r)  r$  Údictr-   r®   Ústagesre   r  ÚheadÚinit_weights)!r5   r/  r0  r1  r2  rž   r&  rŸ   r    r  r3  r4  r5  r6  rs   r  r¡   r7  r8  r9  r(   r)   r6   Ú
num_stagesr    ÚiÚprŒ   rõ   rK  Ústage_strider+  r7   s!                                   €r8   r,   zLevit.__init__˜  sâ  ø€ ô0 	‰ÑÔØ¨Ñ/ˆÜ! )Ó,ˆ	Ü& ~Ò'B¸ÓCˆØ ˆŒØ&ˆÔØ ˆŒØ&ˆÔØ4=¸b±MÐAˆÔ˜DÔ1Ø"ˆŒØ"ˆŒØ"'ˆÔØˆÔä˜“^ˆ
Ü�5‹z˜ZÒ'Ð'Ð'Ø)”I˜jÓ)¨)Ó4ˆ	Ø*”Y˜zÓ*¨:Ó6ˆ
Ø)”I˜jÓ)¨)Ó4ˆ	àÐ$Ø !Ò#Ð#Ð#Ø%ˆDŒIØ ‰Fà Ñ-Ð-Ð-Ø˜EÒ!Ü" 8¨Y°q©\ÑUÀYÐUÐRTÑU�•	ä! (¨I°a©LÑTÀIÐTÐQSÑT�”	Ø—Y‘Y×%Ñ%ˆFÜ¬s´9¸XÓ3FÌ	ÐRXÓHYÓ/Z×[¡t q¨!˜A ›FÓ[Ó\ˆ
à˜1‘ˆØˆÜ�zÓ"ò 	"ˆAØ ! A¢™1¨1ˆLØ”zØØ˜!‘Øðð ˜A‘hØ# A™,Ø% a™=Ø# A™,Ø#Ø-Ø%Ø!Ø&2°aÒ&7™7¸RØ(ñð ñð ñ ˆFð  �lÑ"ˆFÜÀZÖPÀ  Q¡¨<Ñ 7¸!Ó ;ÒPÓQˆJØ×Ò¤$¨y¸©|ÀvÐX_Ð`aÐ_bÐVcÔ"dÐ!eÑeÕØ˜q‘\‰Fð+	"ô, —m‘m VÐ,ˆŒð U`ÐbcÒTc”J˜y¨™}¨kÑPÀ	ÐPÈRÒPÔik×itÑitÓivˆŒ	ð 	×Ñ eÐÕ,ùóA \ùò.  Qs   ÅI'
ÇI-
rA  c                 óP   — | j                  t        | j                  |¬«      «       y )Nr@  )Úapplyr   Ú_init_weights)r5   rA  s     r8   rM  zLevit.init_weightsñ  s   € Ø�
‰
”7˜4×-Ñ-¸;ÔGÕHr9   rG   r½   c                 óD   — |rt        |d«      r|j                  «        y y y )Nrµ   )Úhasattrrµ   )r5   rG   rA  s      r8   rT  zLevit._init_weightsô  s"   € Ùœ7 1Ð&8Ô9Ø×ÑÕ ð :ˆ;r9   c                 ón   — | j                  «       j                  «       D �ch c]	  }d|v sŒ|’Œ c}S c c}w )Nr²   )Ú
state_dictÚkeysrK   s     r8   Úno_weight_decayzLevit.no_weight_decayø  s.   € àŸ?™?Ó,×1Ñ1Ó3ÖO�aÐ7IÈQÒ7N’ÒOÐOùÒOs   ¡	2«2c                 ó$   — t        dddg¬«      }|S )Nz ^cls_token|pos_embed|patch_embed)z^blocks\.(\d+)N)z^norm)iŸ† )rG  r*  )rJ  )r5   ÚcoarseÚmatchers      r8   Úgroup_matcherzLevit.group_matcherü  s   € äØ4Ø-Ð/CÐDô
ˆð ˆr9   c                 ó   — || _         y rJ   )rD  ©r5   Úenables     r8   Úset_grad_checkpointingzLevit.set_grad_checkpointing  ó
   € à"(ˆÕr9   c                 ó   — | j                   S rJ   )rL  rÃ   s    r8   Úget_classifierzLevit.get_classifier  s   € à�y‰yÐr9   c                 ó°   — || _         |�|| _        |dkD  r(t        | j                  || j                  ¬«      | _        y t        j                  «       | _        y ©Nr   )rg   )r1  r7  re   rB  r8  r-   r  rL  ©r5   r1  r7  s      r8   Úreset_classifierzLevit.reset_classifier  sS   € Ø&ˆÔØÐ"Ø*ˆDÔàDOÐRSÂOô Ø×Ñ˜{°·±ôAˆ�	ÜY[×YdÑYdÓYfð 	�	r9   rL   ÚindicesÚnormÚ
stop_earlyÚ
output_fmtÚintermediates_onlyc           	      ó
  — |dv sJ d«       ‚g }t        t        | j                  «      |«      \  }}	| j                  |«      }|j                  \  }
}}}| j
                  s!|j                  d«      j                  dd«      }t        j                  j                  «       s|s| j                  }n| j                  d|	dz    }t        |«      D ]±  \  }}| j                  r+t        j                  j                  «       st        ||«      }n ||«      }||v rS| j
                  r|j                  |«       n5|j                  |j                  |
||d«      j!                  dddd«      «       |dz   dz
  dz  }|dz   dz
  dz  }Œ³ |r|S ||fS )	aï   Forward features that returns intermediates.

        Args:
            x: Input image tensor
            indices: Take last n blocks if int, all if None, select matching indices if sequence
            norm: Apply norm layer to compatible intermediates
            stop_early: Stop iterating over blocks when last desired intermediate hit
            output_fmt: Shape of intermediate feature outputs
            intermediates_only: Only return intermediate features
        Returns:

        )ÚNCHWzOutput shape must be NCHW.r<   r   Nrk   r   rx   )r   rF  rK  rG  rA   r¡   rb   rä   rT   rØ   Úis_scriptingÚ	enumeraterD  r   Úappendr”   r“   )r5   rL   rj  rk  rl  rm  rn  ÚintermediatesÚtake_indicesÚ	max_indexr•   r—   ræ   rç   rK  Úfeat_idxÚstages                    r8   Úforward_intermediateszLevit.forward_intermediates  sr  € ð* ˜YÑ&ÐDÐ(DÓDÐ&ØˆÜ"6´s¸4¿;¹;Ó7GÈÓ"QÑˆ�ið �I‰I�a‹LˆØ—W‘W‰
ˆˆ1ˆa�Ø�}Š}Ø—	‘	˜!“×&Ñ& q¨!Ó,ˆAä�9‰9×!Ñ!Ô#©:Ø—[‘[‰Fà—[‘[  )¨a¡-Ð0ˆFÜ(¨Ó0ò 	!‰OˆH�eØ×&Ò&¬u¯y©y×/EÑ/EÔ/GÜ˜u aÓ(‘á˜!“H�Ø˜<Ñ'Ø—=’=Ø!×(Ñ(¨Õ+à!×(Ñ(¨¯©°1°a¸¸BÓ)?×)GÑ)GÈÈ1ÈaÐQRÓ)SÔTØ�Q‘˜‘˜qÑ ˆAØ�Q‘˜‘˜qÑ ‰Að	!ñ Ø Ð à�-ÐÐr9   Ú
prune_normÚ
prune_headc                 ó    — t        t        | j                  «      |«      \  }}| j                  d|dz    | _        |r| j                  dd«       |S )z@ Prune layers not required for specified intermediates.
        Nr   r   r%  )r   rF  rK  ri  )r5   rj  rz  r{  ru  rv  s         r8   Úprune_intermediate_layerszLevit.prune_intermediate_layersH  sM   € ô #7´s¸4¿;¹;Ó7GÈÓ"QÑˆ�iØ—k‘k . 9¨q¡=Ð1ˆŒÙØ×!Ñ! ! RÔ(ØÐr9   c                 ó(  — | j                  |«      }| j                  s!|j                  d«      j                  dd«      }| j                  r6t
        j                  j                  «       st        | j                  |«      }|S | j                  |«      }|S )Nr<   r   )
rG  r¡   rb   rä   rD  rT   rØ   rq  r   rK  rK   s     r8   Úforward_featureszLevit.forward_featuresV  ss   € Ø�I‰I�a‹LˆØ�}Š}Ø—	‘	˜!“×&Ñ& q¨!Ó,ˆAØ×"Ò"¬5¯9©9×+AÑ+AÔ+CÜ˜tŸ{™{¨AÓ.ˆAð ˆð —‘˜A“ˆAØˆr9   Ú
pre_logitsc                 óª   — | j                   dk(  r0| j                  r|j                  d¬«      n|j                  d¬«      }|r|S | j                  |«      S )NÚavg©râ   rk   rá   r   )r7  r¡   ÚmeanrL  )r5   rL   r€  s      r8   Úforward_headzLevit.forward_head`  sG   € Ø×Ñ˜uÒ$Ø(,¯ª�—‘˜8�Ô$¸1¿6¹6Àa¸6»=ˆAÙˆqÐ0 D§I¡I¨a£LÐ0r9   c                 óJ   — | j                  |«      }| j                  |«      }|S rJ   )r  r…  rK   s     r8   rM   zLevit.forwarde  s'   € Ø×!Ñ! !Ó$ˆØ×Ñ˜aÓ ˆØˆr9   )éà   rx   éè  )éÀ   é@   )é   )rx   rô   rô   NNr;  Ú	subsampleÚ
hard_swishNFr‚  ro   ro   NNrì   ©FrJ   )NFFrp  F)r   FT) rO   rP   rQ   r  r
   rR   r   rS   r   r-   rƒ   rÛ   rp   r,   rM  rT  rT   rØ   ÚignorerZ  r^  rb  re  ri  rï   r   ry  r}  r  r…  rM   rV   rW   s   @r8   r   r   ‘  s?  ø„ ñð 58ØØ#Ø)/ØØ%*Ø59Ø:<Ø9;Ø15Ø)-Ø"Ø&Ø)Ø,0Ø"Ø$Ø!Ø$&ØØñ-W-à˜C  s¨C x¡Ð0Ñ1ðW-ð ðW-ð ð	W-ð
 ˜S #˜X‘ðW-ð ðW-ð ˜˜c˜‘?ðW-ð ˜S %¨¨S¨¡/Ð1Ñ2ðW-ð ˜e U¨5°#¨:Ñ%6Ð6Ñ7ðW-ð ˜U E¨%°¨*Ñ$5Ð5Ñ6ðW-ð $ B§I¡IÑ.ðW-ð " #™ðW-ð ðW-ð ðW-ð ðW-ð  % S™Mð!W-ð" ð#W-ð$ ð%W-ð& ð'W-ð( "õ)W-ñrI¨ó Iñ!˜rŸy™yð !°tð !Àtó !ð ‡Y�Y×ÑñPó ðPð ‡Y�Y×Ñòó ðð ‡Y�Y×Ñò)ó ð)ð ‡Y�Y×Ñð §	¡	ò ó ðñg¨Cð g¸xÈ¹}ó gð 8<ØØ$Ø$Ø',ñ3 à�|‰|ð3 ð ˜e C¨¨c© NÑ3Ñ4ð3 ð ð	3 ð
 ð3 ð ð3 ð !%ð3 ð 
ˆt�E—L‘LÑ! 5¨¯©°t¸E¿L¹LÑ7IÐ)IÑ#JÐJÑ	Kó3 ðn ./Ø$Ø#ñ	à˜3  S¡	˜>Ñ*ðð ðð ó	òñ1¨$ó 1ö
r9   c                   óÐ   ‡ — e Zd Zˆ fd„Zej
                  j                  dej                  fd„«       Z	d
de
dee   fd„Zej
                  j                  dd„«       Zddefd	„Zˆ xZS )ÚLevitDistilledc                 ó  •— t        ‰| �  |i |¤Ž |j                  dd «      |j                  dd «      dœ}| j                  dkD  r!t	        | j
                  | j                  fi |¤Žnt        j                  «       | _        d| _	        y )Nr(   r)   r'   r   F)
r+   r,   Úgetr1  re   rB  r-   r  Ú	head_distÚdistilled_training)r5   ÚargsÚkwargsr6   r7   s       €r8   r,   zLevitDistilled.__init__l  sw   ø€ Ü‰Ñ˜$Ð) &Ò)ØŸ
™
 8¨TÓ2¸V¿Z¹ZÈÐQUÓ=VÑWˆØRV×RbÑRbÐefÒRfœ D×$5Ñ$5°t×7GÑ7GÑNÈ2ÒNÔln×lwÑlwÓlyˆŒØ"'ˆÕr9   r½   c                 ó2   — | j                   | j                  fS rJ   )rL  r”  rÃ   s    r8   re  zLevitDistilled.get_classifierr  s   € à�y‰y˜$Ÿ.™.Ð(Ð(r9   r1  r7  c                 ó  — || _         |�|| _        |dkD  r"t        | j                  || j                  ¬«      nt        j                  «       | _        |dkD  rt        | j                  |«      | _        y t        j                  «       | _        y rg  )	r1  r7  re   rB  r8  r-   r  rL  r”  rh  s      r8   ri  zLevitDistilled.reset_classifierv  sz   € Ø&ˆÔØÐ"Ø*ˆDÔàDOÐRSÂOô Ø×Ñ˜{°·±õAÜY[×YdÑYdÓYfð 	Œ	àGRÐUVÂœ D×$5Ñ$5°{ÓCˆ�Ô\^×\gÑ\gÓ\iˆ�r9   c                 ó   — || _         y rJ   )r•  r`  s     r8   Úset_distilled_trainingz%LevitDistilled.set_distilled_training~  rc  r9   r€  c                 óP  — | j                   dk(  r0| j                  r|j                  d¬«      n|j                  d¬«      }|r|S | j                  |«      | j	                  |«      }}| j
                  r.| j                  r"t        j                  j                  «       s||fS ||z   dz  S )Nr‚  rƒ  rá   r   r<   )
r7  r¡   r„  rL  r”  r•  rÚ   rT   rØ   rq  )r5   rL   r€  Úx_dists       r8   r…  zLevitDistilled.forward_head‚  s�   € Ø×Ñ˜uÒ$Ø(,¯ª�—‘˜8�Ô$¸1¿6¹6Àa¸6»=ˆAÙØˆHØ—I‘I˜a“L $§.¡.°Ó"3ˆ6ˆØ×"Ò" t§}¢}¼U¿Y¹Y×=SÑ=SÔ=Uà�f�9Ðð ˜‘J !Ñ#Ð#r9   rJ   rì   rŽ  )rO   rP   rQ   r,   rT   rØ   r�  r-   rƒ   re  rR   r   rÛ   ri  r›  rp   r…  rV   rW   s   @r8   r‘  r‘  k  su   ø„ ô(ð ‡Y�Y×Ñð) §	¡	ò )ó ð)ñj¨Cð j¸hÀs¹mó jð ‡Y�Y×Ñò)ó ð)ñ$¨$÷ $r9   r‘  c                 ót   — d| v r| d   } | j                  «       D ��ci c]  \  }}d|vsŒ||“Œ } }}| S c c}}w )NÚmodelr¦   )Úitems)rX  rŸ  ré   rê   s       r8   Úcheckpoint_filter_fnr¡  �  sQ   € Ø�*ÑØ Ñ(ˆ
ð $.×#3Ñ#3Ó#5×X™4˜1˜aÐ9NÐVWÒ9W�!�Q‘$ÐX€JÑXð Ðùó Ys   �4ª4)é€   é   é€  r‡   )rw   é   ru   )r<   rx   rw   )r2  rž   rŸ   r&  )rw   ru   r‹  )rw   rw   rw   )r‰  i   r¤  é    )rx   é   r¥  )r£  r¤  é   )r¤  r¨  é   )r¥  é	   r‹  Úsilur<  )r2  rž   rŸ   r&  rs   r5  )r¨  é€  i€  rŠ  )ru   é
   rœ   )r¨  r©  i   )ru   r‹  r‡   )r2  rž   rŸ   r&  rs   )rw   ru   r¥  )r¨  r¬  r©  )ru   r­  r‹  )
Ú
levit_128sÚ	levit_128Ú	levit_192Ú	levit_256Ú	levit_384Úlevit_384_s8Úlevit_512_s8Ú	levit_512Ú
levit_256dÚ
levit_512dc                 ó>  — d| v }|j                  dd«      }|j                  dd«      r|s|j                  dd«       |€| t        v r| }n|r| j	                  dd«      }t        t        |   fi |¤Ž}t        |rt        nt        | |ft        t        d	|¬
«      dœ|¤Ž}|S )NÚ_convÚout_indices)r   r   r<   Úfeatures_onlyFÚfeature_clsÚgetterr%  T)Úflatten_sequentialrº  )Úpretrained_filter_fnÚfeature_cfg)
Úpopr“  Ú
setdefaultÚ
model_cfgsÚreplacerJ  r   r‘  r   r¡  )	ÚvariantÚcfg_variantÚ
pretrainedÚ	distilledr—  Úis_convrº  Ú	model_cfgrŸ  s	            r8   Úcreate_levitrË  Å  s²   € Ø˜Ð €GØ—*‘*˜]¨IÓ6€KØ‡z�z�/ 5Ô)±'Ø×Ñ˜-¨Ô2ØÐØ”jÑ Ø!‰KÙØ!Ÿ/™/¨'°2Ó6ˆKä”Z Ñ,Ñ7°Ñ7€IÜ Ù#�¬ØØðô 2Ü¨D¸kÔJñð ñ€Eð €Lr9   c                 ó4   — | ddd dddt         t        dddd	œ|¥S )
Nrˆ  )rx   r‡  r‡  gÍÌÌÌÌÌì?ÚbicubicTzstem.conv1.linear)úhead.linearzhead_dist.linearz
apache-2.0)Úurlr1  Ú
input_sizeÚ	pool_sizeÚcrop_pctÚinterpolationÚfixed_input_sizer„  rf   Ú
first_convÚ
classifierÚlicense)r   r   )rÏ  r—  s     r8   Ú_cfgrØ  Ü  s6   € àØ¨=ÀtØ¨ÈÜ%Ô.BØ)Ð9\Øñð ðð r9   zlevit_128s.fb_dist_in1kztimm/)Ú	hf_hub_idzlevit_128.fb_dist_in1kzlevit_192.fb_dist_in1kzlevit_256.fb_dist_in1kzlevit_384.fb_dist_in1kzlevit_conv_128s.fb_dist_in1k)rw   rw   )rÙ  rÑ  zlevit_conv_128.fb_dist_in1kzlevit_conv_192.fb_dist_in1kzlevit_conv_256.fb_dist_in1kzlevit_conv_384.fb_dist_in1kzlevit_384_s8.untrainedrÎ  )rÖ  zlevit_512_s8.untrainedzlevit_512.untrainedzlevit_256d.untrainedzlevit_512d.untrainedzlevit_conv_384_s8.untrainedzlevit_conv_512_s8.untrained)zlevit_conv_512.untrainedzlevit_conv_256d.untrainedzlevit_conv_512d.untrainedr½   c                 ó   — t        dd| i|¤ŽS )NrÇ  )r®  ©rË  ©rÇ  r—  s     r8   r®  r®    s   € äÑF°ÐF¸vÑFÐFr9   c                 ó   — t        dd| i|¤ŽS )NrÇ  )r¯  rÛ  rÜ  s     r8   r¯  r¯  #  ó   € äÑE°
ÐE¸fÑEÐEr9   c                 ó   — t        dd| i|¤ŽS )NrÇ  )r°  rÛ  rÜ  s     r8   r°  r°  (  rÞ  r9   c                 ó   — t        dd| i|¤ŽS )NrÇ  )r±  rÛ  rÜ  s     r8   r±  r±  -  rÞ  r9   c                 ó   — t        dd| i|¤ŽS )NrÇ  )r²  rÛ  rÜ  s     r8   r²  r²  2  rÞ  r9   c                 ó   — t        dd| i|¤ŽS )NrÇ  )r³  rÛ  rÜ  s     r8   r³  r³  7  s   € äÑH°:ÐHÀÑHÐHr9   c                 ó   — t        d| ddœ|¤ŽS )NF©rÇ  rÈ  )r´  rÛ  rÜ  s     r8   r´  r´  <  s   € äÐY°:ÈÑYÐRXÑYÐYr9   c                 ó   — t        d| ddœ|¤ŽS )NFrä  )rµ  rÛ  rÜ  s     r8   rµ  rµ  A  s   € äÐV°
ÀeÑVÈvÑVÐVr9   c                 ó   — t        d| ddœ|¤ŽS )NFrä  )r¶  rÛ  rÜ  s     r8   r¶  r¶  F  ó   € äÐW°ÀuÑWÐPVÑWÐWr9   c                 ó   — t        d| ddœ|¤ŽS )NFrä  )r·  rÛ  rÜ  s     r8   r·  r·  K  rç  r9   c                 ó   — t        d| ddœ|¤ŽS )NT©rÇ  r¡   )Úlevit_conv_128srÛ  rÜ  s     r8   rë  rë  P  s   € äÐZ°jÈ4ÑZÐSYÑZÐZr9   c                 ó   — t        d| ddœ|¤ŽS )NTrê  )Úlevit_conv_128rÛ  rÜ  s     r8   rí  rí  U  ó   € äÐY°ZÈ$ÑYÐRXÑYÐYr9   c                 ó   — t        d| ddœ|¤ŽS )NTrê  )Úlevit_conv_192rÛ  rÜ  s     r8   rð  rð  Z  rî  r9   c                 ó   — t        d| ddœ|¤ŽS )NTrê  )Úlevit_conv_256rÛ  rÜ  s     r8   rò  rò  _  rî  r9   c                 ó   — t        d| ddœ|¤ŽS )NTrê  )Úlevit_conv_384rÛ  rÜ  s     r8   rô  rô  d  rî  r9   c                 ó   — t        d| ddœ|¤ŽS )NTrê  )Úlevit_conv_384_s8rÛ  rÜ  s     r8   rö  rö  i  s   € äÐ\¸
ÈTÑ\ÐU[Ñ\Ð\r9   c                 ó    — t        d| dddœ|¤ŽS )NTF©rÇ  r¡   rÈ  )Úlevit_conv_512_s8rÛ  rÜ  s     r8   rù  rù  n  s   € äÐm¸
ÈTÐ]bÑmÐflÑmÐmr9   c                 ó    — t        d| dddœ|¤ŽS )NTFrø  )Úlevit_conv_512rÛ  rÜ  s     r8   rû  rû  s  s   € äÐj°ZÈ$ÐZ_ÑjÐciÑjÐjr9   c                 ó    — t        d| dddœ|¤ŽS )NTFrø  )Úlevit_conv_256drÛ  rÜ  s     r8   rý  rý  x  ó   € äÐk°jÈ4Ð[`ÑkÐdjÑkÐkr9   c                 ó    — t        d| dddœ|¤ŽS )NTFrø  )Úlevit_conv_512drÛ  rÜ  s     r8   r   r   }  rþ  r9   )NFT)r%  rŽ  )Nr  Úcollectionsr   Ú	functoolsr   Útypingr   r   r   r   r	   r
   rT   Útorch.nnr-   Ú	timm.datar   r   Útimm.layersr   r   r   r   r   r   Ú_builderr   Ú	_featuresr   Ú_manipulater   r   Ú	_registryr   r   Ú__all__rƒ   r   rY   re   r®   rr   r…   r‹   r™   ró   r
  r  r  r$  r   r‘  r¡  rJ  rÃ  rË  rØ  Údefault_cfgsr®  r¯  r°  r±  r²  r³  r´  rµ  r¶  r·  rë  rí  rð  rò  rô  rö  rù  rû  rý  r   r  r9   r8   ú<module>r     sû  ðñõ2 $Ý ß ;× ;ã Ý ç Aß \× \Ý *Ý +ß 3ß <àˆ)€ô#'ˆr�y‰yô #'ôL6�—‘ô 6ô@%2�—‘ô %2ôP`ˆB�M‰Mô `ô(`ˆR�]‰]ô `ô,#�—‘ô #ô0l�—	‘	ô lô^@˜"Ÿ)™)ô @ôFˆr�y‰yô ôB/�b—i‘iô /ôd,�—‘ô ,ô^?�—‘ô ?ôDWˆB�I‰Iô Wôt"$�Uô "$òJñ* ÙØ!¨2¸È)ôUáØ!¨2¸È9ôVáØ!¨2¸È)ôUáØ!¨2¸È)ôUáØ!¨2¸È9ôVñ Ø!¨2¸È9Ø Dô*ñ Ø!¨2¸ÈIØ Dô*ñ
 Ø"¨B¸+ÈYÐbhôjñ Ø!¨2¸È)Ð_eôgáØ!¨2¸ÈIÐagôiô7€
ó@ó.	ñ %ð 3&à™tØô ð3&ð
 ™dØôð3&ð ™dØôð3&ð ™dØôð3&ð ™dØôð3&ð& #¡DØØô%ð'3&ð. "¡4ØØô$ð/3&ð6 "¡4ØØô$ð73&ð> "¡4ØØô$ð?3&ðF "¡4ØØô$ðG3&ðP ™d¨mÔ<ðQ3&ðR ™d¨mÔ<ðS3&ðT ™4¨=Ô9ðU3&ðV ™D¨MÔ:ðW3&ðX ™D¨MÔ:ðY3&ð\ "¡4°=Ô#Að]3&ð^ "¡4°=Ô#Að_3&ñ` !%°Ô >Ù!%°Ô!?Ù!%°Ô!?òe3&ó 3€ðl ñG¨eò Gó ðGð ñF¨Uò Fó ðFð ñF¨Uò Fó ðFð ñF¨Uò Fó ðFð ñF¨Uò Fó ðFð ñI°ò Ió ðIð ñZ°ò Zó ðZð ñW¨Uò Wó ðWð ñX¨eò Xó ðXð ñX¨eò Xó ðXð ñ[°5ò [ó ð[ð ñZ°%ò Zó ðZð ñZ°%ò Zó ðZð ñZ°%ò Zó ðZð ñZ°%ò Zó ðZð ñ]°Uò ]ó ð]ð ñn°Uò nó ðnð ñk°%ò kó ðkð ñl°5ò ló ðlð ñl°5ò ló ñlr9   