Ë
    éÓ^j×  ã                   óV   — d Z ddlZddlmZ ddlmZmZmZmZ ddl	m
Z
  G d„ de«      Zy)	a	   RMSProp modified to behave like Tensorflow impl

Originally cut & paste from PyTorch RMSProp
https://github.com/pytorch/pytorch/blob/063946d2b3f3f1e953a2a3b54e0b34f1393de295/torch/optim/rmsprop.py
Licensed under BSD-Clause 3 (ish), https://github.com/pytorch/pytorch/blob/master/LICENSE

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Why Gradients Rapidly Increase Near the End of Training: https://arxiv.org/abs/2506.02285

Modifications Copyright 2021 Ross Wightman
é    N)Ú	Optimizeré   )Ú_add_scaled_Ú_addcdiv_scaled_Ú_init_scalarÚ_validate_scalar)ÚParamsTc                   ó    ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 ddededededededed	ed
ededefˆ fd„Zˆ fd„Z e	j                  «       dd„«       Zˆ xZS )Ú	RMSpropTFaÃ  Implements RMSprop algorithm (TensorFlow style epsilon)

    NOTE: This is a direct cut-and-paste of PyTorch RMSprop with eps applied before sqrt
    and a few other modifications to closer match Tensorflow for matching hyper-params.

    Noteworthy changes include:
    1. Epsilon applied inside square-root
    2. square_avg initialized to ones
    3. LR scaling of update accumulated in momentum buffer

    Proposed by G. Hinton in his
    `course <http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf>`_.

    The centered version first appears in `Generating Sequences
    With Recurrent Neural Networks <https://arxiv.org/pdf/1308.0850v5.pdf>`_.

    Args:
        params: iterable of parameters to optimize or dicts defining parameter groups
        lr: learning rate
        momentum: momentum factor
        alpha: smoothing (decay) constant
        eps: term added to the denominator to improve numerical stability
        centered: if ``True``, compute the centered RMSProp, the gradient is normalized by an estimation of its variance
        weight_decay: weight decay (L2 penalty) (default: 0)
        decoupled_decay: decoupled weight decay as per https://arxiv.org/abs/1711.05101
        corrected_weight_decay: apply corrected weight decay (lr**2 / max_lr) when decoupled_decay is True
        lr_in_momentum: learning rate scaling is included in the momentum buffer update as per defaults in Tensorflow
        caution: apply caution
    ÚparamsÚlrÚalphaÚepsÚweight_decayÚmomentumÚcenteredÚdecoupled_decayÚcorrected_weight_decayÚlr_in_momentumÚcautionc                 óö   •— t        d|«       t        d|«       t        d|«       t        d|«       d|k  st        dj                  |«      «      ‚t        ||||||||	|
|¬«
      }t        t
        | �  ||«       y )Nzlearning rateÚepsilonr   r   ç        zInvalid alpha value: {})
r   r   r   r   r   r   r   r   r   r   )r   Ú
ValueErrorÚformatÚdictÚsuperr   Ú__init__)Úselfr   r   r   r   r   r   r   r   r   r   r   ÚdefaultsÚ	__class__s                €ú`/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/optim/rmsprop_tf.pyr   zRMSpropTF.__init__4   s†   ø€ ô 	˜¨"Ô-Ü˜ CÔ(Ü˜ XÔ.Ü˜¨Ô6Ø�eŠ|ÜÐ6×=Ñ=¸eÓDÓEÐEäØØØØØØ%Ø+Ø#9Ø)Øô
ˆô 	Œi˜Ñ'¨°Õ9ó    c                 ó`  •— t         t        | �  |«       | j                  D ]‹  }|j	                  dd«       |j	                  dd«       |j	                  dd«       |j	                  dd«       |d   D ]9  }| j
                  j                  |i «      }|sŒ"d|v sŒ't        |d   d	¬
«      |d<   Œ; Œ� y )Nr   r   r   Fr   r   r   ÚstepÚcpu©Údevice)r   r   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetr   )r   r,   ÚgroupÚpÚp_stater!   s        €r"   r)   zRMSpropTF.__setstate__W   s±   ø€ ÜŒi˜Ñ+¨EÔ2Ø×&Ñ&ò 	RˆEØ×Ñ˜Z¨Ô+Ø×Ñ˜Z¨Ô/Ø×Ñ˜Y¨Ô.Ø×ÑÐ5°uÔ=Ø˜8‘_ò R�ØŸ*™*Ÿ.™.¨¨BÓ/�Ú˜v¨Ò0Ü&2°7¸6±?È5Ô&Q�G˜F’OñRñ	Rr#   c           	      óŽ  — d}|�$t        j                  «       5   |«       }ddd«       | j                  D �]  }|d   D �]s  }|j                  €Œ|j                  }|j                  rt        d«      ‚| j                  |   }t        |«      dk(  rdt        d¬«      |d<   t        j                  |«      |d<   |d	   dkD  rt        j                  |«      |d
<   |d   rt        j                  |«      |d<   |d   }d|d   z
  }|d   j                  d«       |d   dk7  rY|d   r>|d   r|d   dz  | j                  d   z  }	n|d   }	|j                  d|	|d   z  z
  «       n|j                  ||d   ¬«      }|j                  |j                  d«      |z
  |¬«       |d   rP|d   }
|
j                  ||
z
  |¬«       |j!                  |
|
d¬«      j                  |d   «      j#                  «       }n"|j                  |d   «      j#                  «       }|d	   dkD  r‡|d
   }|j                  |d	   «       d„ }|d   r3t%        ||||d   «       |d   r	 |||«      }|j                  | «       �Œ/|j'                  ||«       |d   r	 |||«      }t)        |||d    «       �Œbt%        ||||d    «       �Œv �Œ‚ |S # 1 sw Y   �Œ›xY w)z¶Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   z)RMSprop does not support sparse gradientsr   r&   r'   r%   Ú
square_avgr   Úmomentum_bufferr   Úgrad_avgg      ð?r   r   r   r   r   r   é   )r   éÿÿÿÿ)Úvaluer   c                 ó¬   — | |z  dkD  j                  |j                  «      }|j                  |j                  «       j	                  d¬«      «       | |z  S )Nr   gü©ñÒMbP?)Úmin)ÚtoÚdtypeÚdiv_ÚmeanÚclamp_)Ú_mÚ_gÚmasks      r"   Ú_apply_cautionz&RMSpropTF.step.<locals>._apply_caution¡   sH   € à " R¡¨!¡×/Ñ/°·±Ó9˜ØŸ	™	 $§)¡)£+×"4Ñ"4¸Ð"4Ó">Ô?Ø! D™yÐ(r#   r   r   )ÚtorchÚenable_gradr*   ÚgradÚ	is_sparseÚRuntimeErrorr,   Úlenr   Ú	ones_likeÚ
zeros_likeÚadd_r    Úmul_ÚaddÚpowÚaddcmulÚsqrt_r   Úaddcdiv_r   )r   ÚclosureÚlossr.   r/   rE   r,   r2   Úone_minus_alphaÚwd_scaler4   ÚavgÚbufrB   s                 r"   r%   zRMSpropTF.stepc   s   € ð ˆØÐÜ×"Ñ"Ó$ñ !Ù“y�÷!ð ×&Ñ&ó D	AˆEØ˜8‘_ó CA�Ø—6‘6�>ØØ—v‘v�Ø—>’>Ü&Ð'RÓSÐSØŸ
™
 1™�ô �u“: ’?Ü$0¸Ô$>�E˜&‘MÜ*/¯/©/¸!Ó*<�E˜,Ñ'Ø˜ZÑ(¨1Ò,Ü38×3CÑ3CÀAÓ3F˜Ð/Ñ0Ø˜ZÒ(Ü,1×,<Ñ,<¸QÓ,?˜˜jÑ)à" <Ñ0�
Ø"$ u¨W¡~Ñ"5�à�f‘×"Ñ" 1Ô%à˜Ñ(¨AÒ-ØÐ.Ò/Ø Ð!9Ò:Ø',¨T¡{°aÑ'7¸$¿-¹-ÈÑ:MÑ'M™Hà',¨T¡{˜HØŸ™˜r H¨u°^Ñ/DÑ$DÑDÕEà#Ÿx™x¨°°~Ñ1F˜xÓG˜ð —‘ §¡¨£¨jÑ 8À�ÔPð ˜Ò$Ø$ ZÑ0�HØ—M‘M $¨¡/¸�MÔIØ$×,Ñ,¨X°xÀrÐ,ÓJ×NÑNÈuÐUZÉ|Ó\×bÑbÓd‘Cð %Ÿ.™.¨¨u©Ó6×<Ñ<Ó>�Cà˜Ñ$ qÒ(ØÐ 1Ñ2�CØ—H‘H˜U :Ñ.Ô/ò)ð Ð-Ò.ä(¨¨d°C¸¸t¹ÔEØ  Ò+Ù"0°°dÓ";˜CØŸ™ ˜tžð Ÿ™ T¨3Ô/Ø  Ò+Ù"0°°dÓ";˜CÜ$ Q¨¨e°D©k¨\Ö:ä$ Q¨¨c°E¸$±K°<Ö@òGCAðD	AðL ˆ÷S!ñ !ús   ™J:Ê:K)
g{®Gáz„?gÍÌÌÌÌÌì?g»½×Ùß|Û=r   r   FFFTF)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r	   ÚfloatÚboolr   r)   rC   Úno_gradr%   Ú__classcell__)r!   s   @r"   r   r      s¿   ø„ ñðB ØØØ"#Ø Ø"Ø$)Ø+0Ø#'Ø!ñ!:àð!:ð ð!:ð ð	!:ð
 ð!:ð  ð!:ð ð!:ð ð!:ð "ð!:ð %)ð!:ð !ð!:ð õ!:ôF
Rð €U‡]�]ƒ_òRó ôRr#   r   )r[   rC   Útorch.optimr   Ú_helpersr   r   r   r   Ú_typesr	   r   © r#   r"   ú<module>rd      s)   ðñó Ý !ç TÓ TÝ ôa�	õ ar#   