Ë
    éÓ^j  ã                   ó:   — d Z ddlZddlZddlmZ  G d„ de«      Zy)a  RAdam Optimizer.
Implementation lifted from: https://github.com/LiyuanLucasLiu/RAdam
Paper: `On the Variance of the Adaptive Learning Rate and Beyond` - https://arxiv.org/abs/1908.03265

NOTE: This impl has been deprecated in favour of torch.optim.RAdam and remains as a reference
é    N)Ú	Optimizerc                   óf   ‡ — e Zd ZdZ	 	 	 	 dˆ fd„	Zˆ fd„Z ej                  «       dd„«       Zˆ xZ	S )ÚRAdamLegacyz€ PyTorch RAdam optimizer

    NOTE: This impl has been deprecated in favour of torch.optim.AdamW and remains as a reference
    c                 óŠ   •— t        ||||t        d«      D �cg c]  }g d¢‘Œ c}¬«      }t        t        | �  ||«       y c c}w )Né
   )NNN)ÚlrÚbetasÚepsÚweight_decayÚbuffer)ÚdictÚrangeÚsuperr   Ú__init__)	ÚselfÚparamsr   r	   r
   r   Ú_ÚdefaultsÚ	__class__s	           €ú[/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/optim/radam.pyr   zRAdamLegacy.__init__   sF   ø€ ô ØØØØ%Ü05°b³	Ö:¨1Ô&Ò:ô
ˆô 	Œk˜4Ñ)¨&°(Õ;ùò ;s   ˜A c                 ó,   •— t         t        | �  |«       y ©N)r   r   Ú__setstate__)r   Ústater   s     €r   r   zRAdamLegacy.__setstate__"   s   ø€ ÜŒk˜4Ñ-¨eÕ4ó    c                 ó¸  — d }|�$t        j                  «       5   |«       }d d d «       | j                  D �]”  }|d   D �]ˆ  }|j                  €Œ|j                  j	                  «       }|j
                  rt        d«      ‚|j	                  «       }| j                  |   }t        |«      dk(  r6d|d<   t        j                  |«      |d<   t        j                  |«      |d<   n.|d   j                  |«      |d<   |d   j                  |«      |d<   |d   |d   }	}|d   \  }
}|	j                  |«      j                  ||d|z
  ¬	«       |j                  |
«      j                  |d|
z
  ¬
«       |dxx   dz  cc<   |d   t        |d   dz  «         }|d   |d   k(  r|d   |d   }}n˜|d   |d<   ||d   z  }dd|z
  z  dz
  }|d|d   z  |z  d|z
  z  z
  }||d<   |dk\  rI|d   t        j                   d|z
  |dz
  z  |dz
  z  |dz
  z  |z  |z  |dz
  z  «      z  d|
|d   z  z
  z  }n|d   d|
|d   z  z
  z  }||d<   |d   dk7  r|j                  ||d    |d   z  ¬
«       |dk\  r8|	j!                  «       j                  |d   «      }|j#                  ||| ¬	«       n|j                  || ¬
«       |j%                  |«       �Œ‹ �Œ— |S # 1 sw Y   �Œ°xY w)Nr   z'RAdam does not support sparse gradientsr   ÚstepÚexp_avgÚ
exp_avg_sqr	   é   )Úvalue)Úalphar   r   é   é   r   é   r   r
   )ÚtorchÚenable_gradÚparam_groupsÚgradÚfloatÚ	is_sparseÚRuntimeErrorr   ÚlenÚ
zeros_likeÚtype_asÚmul_Úaddcmul_Úadd_ÚintÚmathÚsqrtÚaddcdiv_Úcopy_)r   ÚclosureÚlossÚgroupÚpr)   Úp_fp32r   r   r   Úbeta1Úbeta2ÚbufferedÚnum_smaÚ	step_sizeÚbeta2_tÚnum_sma_maxÚdenoms                     r   r   zRAdamLegacy.step%   sX  € àˆØÐÜ×"Ñ"Ó$ñ !Ù“y�÷!ð ×&Ñ&ó ;	 ˆEà˜8‘_ó 9 �Ø—6‘6�>ØØ—v‘v—|‘|“~�Ø—>’>Ü&Ð'PÓQÐQàŸ™›�àŸ
™
 1™�ä�u“: ’?Ø$%�E˜&‘MÜ',×'7Ñ'7¸Ó'?�E˜)Ñ$Ü*/×*:Ñ*:¸6Ó*B�E˜,Ò'à',¨YÑ'7×'?Ñ'?ÀÓ'G�E˜)Ñ$Ø*/°Ñ*=×*EÑ*EÀfÓ*M�E˜,Ñ'à&+¨IÑ&6¸¸lÑ8K˜�Ø$ W™~‘��uà—‘ Ó&×/Ñ/°°dÀ!ÀeÁ)Ð/ÔLØ—‘˜UÓ#×(Ñ(¨°Q¸±YÐ(Ô?à�f“ Ñ"“Ø  ™?¬3¨u°V©}¸rÑ/AÓ+BÑC�Ø˜‘= H¨Q¡KÒ/Ø)1°!©°h¸q±k˜Y‘Gà"'¨¡-�H˜Q‘KØ# u¨V¡}Ñ4�GØ"# q¨5¡y¡/°AÑ"5�KØ)¨A°°f±Ñ,=ÀÑ,GÈ1ÈwÉ;Ñ,WÑW�GØ")�H˜Q‘Kð  !’|Ø$)¨$¡K´$·)±)Ø ™[Ø$ q™[ñ*Ø-8¸1©_ñ>à$ q™[ñ*à,3ñ4ð (ñ(ð ,7¸©?ñ<ó3=ñ %=ð ABÀEÈUÐSYÉ]ÑDZÑ@Zñ	%\™	ð %*¨$¡K°1°uÀÀfÁÑ7MÑ3MÑ$N˜	Ø"+�H˜Q‘Kà˜Ñ(¨AÒ-Ø—K‘K ¨u°^Ñ/DÐ.DÀuÈTÁ{Ñ.R�KÔSð ˜a’<Ø&ŸO™OÓ-×2Ñ2°5¸±<Ó@�EØ—O‘O G¨U¸9¸*�OÕEà—K‘K °	¨z�KÔ:à—‘˜–òs9 ð;	 ðz ˆ÷A!ñ !ús   ™KËK)gü©ñÒMbP?)gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?g:Œ0âŽyE>r   r   )
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r&   Úno_gradr   Ú__classcell__)r   s   @r   r   r      s>   ø„ ñð ØØØõ<ô"5ð €U‡]�]ƒ_òCó ôCr   r   )rH   r4   r&   Útorch.optim.optimizerr   r   © r   r   ú<module>rM      s#   ðñó Û Ý +ô]�)õ ]r   