
    ^j                     V    d Z ddlZddlmZ ddlmZmZmZmZ ddl	m
Z
  G d de      Zy)	a	   RMSProp modified to behave like Tensorflow impl

Originally cut & paste from PyTorch RMSProp
https://github.com/pytorch/pytorch/blob/063946d2b3f3f1e953a2a3b54e0b34f1393de295/torch/optim/rmsprop.py
Licensed under BSD-Clause 3 (ish), https://github.com/pytorch/pytorch/blob/master/LICENSE

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Why Gradients Rapidly Increase Near the End of Training: https://arxiv.org/abs/2506.02285

Modifications Copyright 2021 Ross Wightman
    N)	Optimizer   )_add_scaled__addcdiv_scaled__init_scalar_validate_scalar)ParamsTc                        e Zd ZdZ	 	 	 	 	 	 	 	 	 	 ddededededededed	ed
ededef fdZ fdZ e	j                         dd       Z xZS )	RMSpropTFa  Implements RMSprop algorithm (TensorFlow style epsilon)

    NOTE: This is a direct cut-and-paste of PyTorch RMSprop with eps applied before sqrt
    and a few other modifications to closer match Tensorflow for matching hyper-params.

    Noteworthy changes include:
    1. Epsilon applied inside square-root
    2. square_avg initialized to ones
    3. LR scaling of update accumulated in momentum buffer

    Proposed by G. Hinton in his
    `course <http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf>`_.

    The centered version first appears in `Generating Sequences
    With Recurrent Neural Networks <https://arxiv.org/pdf/1308.0850v5.pdf>`_.

    Args:
        params: iterable of parameters to optimize or dicts defining parameter groups
        lr: learning rate
        momentum: momentum factor
        alpha: smoothing (decay) constant
        eps: term added to the denominator to improve numerical stability
        centered: if ``True``, compute the centered RMSProp, the gradient is normalized by an estimation of its variance
        weight_decay: weight decay (L2 penalty) (default: 0)
        decoupled_decay: decoupled weight decay as per https://arxiv.org/abs/1711.05101
        corrected_weight_decay: apply corrected weight decay (lr**2 / max_lr) when decoupled_decay is True
        lr_in_momentum: learning rate scaling is included in the momentum buffer update as per defaults in Tensorflow
        caution: apply caution
    paramslralphaepsweight_decaymomentumcentereddecoupled_decaycorrected_weight_decaylr_in_momentumcautionc                     t        d|       t        d|       t        d|       t        d|       d|k  st        dj                  |            t        ||||||||	|
|
      }t        t
        |   ||       y )Nzlearning rateepsilonr   r           zInvalid alpha value: {})
r   r   r   r   r   r   r   r   r   r   )r   
ValueErrorformatdictsuperr   __init__)selfr   r   r   r   r   r   r   r   r   r   r   defaults	__class__s                `/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/optim/rmsprop_tf.pyr   zRMSpropTF.__init__4   s     	"-C(X.6e|6==eDEE%+#9)
 	i'9    c                 `   t         t        |   |       | j                  D ]  }|j	                  dd       |j	                  dd       |j	                  dd       |j	                  dd       |d   D ]9  }| j
                  j                  |i       }|s"d|v s't        |d   d	
      |d<   ;  y )Nr   r   r   Fr   r   r   stepcpudevice)r   r   __setstate__param_groups
setdefaultstategetr   )r   r,   grouppp_stater!   s        r"   r)   zRMSpropTF.__setstate__W   s    i+E2&& 	REZ+Z/Y.5u=8_ R**..B/v0&276?5&QGFOR	Rr#   c           	         d}|$t        j                         5   |       }ddd       | j                  D ]  }|d   D ]s  }|j                  |j                  }|j                  rt        d      | j                  |   }t        |      dk(  rdt        d      |d<   t        j                  |      |d<   |d	   dkD  rt        j                  |      |d
<   |d   rt        j                  |      |d<   |d   }d|d   z
  }|d   j                  d       |d   dk7  rY|d   r>|d   r|d   dz  | j                  d   z  }	n|d   }	|j                  d|	|d   z  z
         n|j                  ||d         }|j                  |j                  d      |z
  |       |d   rP|d   }
|
j                  ||
z
  |       |j!                  |
|
d      j                  |d         j#                         }n"|j                  |d         j#                         }|d	   dkD  r|d
   }|j                  |d	          d }|d   r3t%        ||||d          |d   r	 |||      }|j                  |        /|j'                  ||       |d   r	 |||      }t)        |||d           bt%        ||||d           v  |S # 1 sw Y   xY w)zPerforms a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   z)RMSprop does not support sparse gradientsr   r&   r'   r%   
square_avgr   momentum_bufferr   grad_avgg      ?r   r   r   r   r   r      )r   )valuer   c                     | |z  dkD  j                  |j                        }|j                  |j                         j	                  d             | |z  S )Nr   gMbP?)min)todtypediv_meanclamp_)_m_gmasks      r"   _apply_cautionz&RMSpropTF.step.<locals>._apply_caution   sH     "R!//9		$))+"4"4"4">?!Dy(r#   r   r   )torchenable_gradr*   grad	is_sparseRuntimeErrorr,   lenr   	ones_like
zeros_likeadd_r    mul_addpowaddcmulsqrt_r   addcdiv_r   )r   closurelossr.   r/   rE   r,   r2   one_minus_alphawd_scaler4   avgbufrB   s                 r"   r%   zRMSpropTF.stepc   s     ""$ !y! && D	AE8_ CA66>vv>>&'RSS

1 u:?$0$>E&M*///!*<E,'Z(1,383C3CA3F/0Z(,1,<,<Q,?j)"<0
"$uW~"5f""1%(A-./ !9:',T{a'7$--:M'MH',T{HrHu^/D$DDE#xx~1FxG j 8P $$Z0HMM$/MI$,,Xxr,JNNuUZ|\bbdC %..u6<<>C$q( 12CHHU:./) -.(dCtE +"0d";Ct T3/ +"0d";C$QeDk\:$QcE$K<@GCAD	AL S! !s   J::K)
g{Gz?g?g|=r   r   FFFTF)N)__name__
__module____qualname____doc__r	   floatboolr   r)   rC   no_gradr%   __classcell__)r!   s   @r"   r   r      s    B "# "$)+0#'!!:!: !: 	!:
 !:  !: !: !: "!: %)!: !!: !:F
R U]]_R Rr#   r   )r[   rC   torch.optimr   _helpersr   r   r   r   _typesr	   r    r#   r"   <module>rd      s)     ! T T a	 ar#   