Ë
    éÓ^j  ã                   ó^   — d Z ddlmZ ddlmZ ddlZddlmZmZm	Z	 ddl
mZ  G d„ d	e«      Zy)
an   PyTorch impl of LaProp optimizer

Code simplified from https://github.com/Z-T-WANG/LaProp-Optimizer, MIT License

Paper: LaProp: Separating Momentum and Adaptivity in Adam, https://arxiv.org/abs/2002.04839

@article{ziyin2020laprop,
  title={LaProp: a Better Way to Combine Momentum with Adaptive Gradient},
  author={Ziyin, Liu and Wang, Zhikang T and Ueda, Masahito},
  journal={arXiv preprint arXiv:2002.04839},
  year={2020}
}

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Why Gradients Rapidly Increase Near the End of Training: https://arxiv.org/abs/2506.02285

é    )ÚTuple)Ú	OptimizerNé   )Ú_add_scaled_Ú_init_scalarÚ_validate_scalar)ÚParamsTc                   ó’   ‡ — e Zd ZdZ	 	 	 	 	 	 ddededeeef   dedededefˆ fd	„Zˆ fd
„Z	 e
j                  «       dd„«       Zˆ xZS )ÚLaPropzw LaProp Optimizer

    Paper: LaProp: Separating Momentum and Adaptivity in Adam, https://arxiv.org/abs/2002.04839
    ÚparamsÚlrÚbetasÚepsÚweight_decayÚcautionÚcorrected_weight_decayc                 ó4  •— t        d|«       t        d|«       d|d   cxk  rdk  sn t        dj                  |d   «      «      ‚d|d   cxk  rdk  sn t        dj                  |d   «      «      ‚t        ||||||¬	«      }t        t
        | �  ||«       y )
Nzlearning rateÚepsilonç        r   ç      ð?z%Invalid beta parameter at index 0: {}r   z%Invalid beta parameter at index 1: {})r   r   r   r   r   r   )r   Ú
ValueErrorÚformatÚdictÚsuperr   Ú__init__)
Úselfr   r   r   r   r   r   r   ÚdefaultsÚ	__class__s
            €ú\/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/optim/laprop.pyr   zLaProp.__init__!   s¢   ø€ ô 	˜¨"Ô-Ü˜ CÔ(Ø�e˜A‘hÔ$ Ô$ÜÐD×KÑKÈEÐRSÉHÓUÓVÐVØ�e˜A‘hÔ$ Ô$ÜÐD×KÑKÈEÐRSÉHÓUÓVÐVÜØØØØ%ØØ#9ô
ˆô 	Œf�dÑ$ V¨XÕ6ó    c                 óÒ  •— t         ‰| �  |«       | j                  D ]È  }|j                  dd«       |j                  dd«       |d   D ]š  }| j                  j                  |i «      }|sŒ"d|v rt        |d   d¬«      |d<   d|v rFt        j                  |d	   «      r.t        |d   |d	   j                  |d	   j                  ¬
«      |d<   d|v sŒˆt        |d   d¬«      |d<   Œœ ŒÊ y )Nr   Fr   r   ÚstepÚcpu©ÚdeviceÚexp_avg_lr_1r   )Údtyper%   Úexp_avg_lr_2)r   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetr   ÚtorchÚ	is_tensorr'   r%   )r   r,   ÚgroupÚpÚp_stater   s        €r   r)   zLaProp.__setstate__;   sü   ø€ Ü‰Ñ˜UÔ#Ø×&Ñ&ò 	bˆEØ×Ñ˜Y¨Ô.Ø×ÑÐ5°uÔ=Ø˜8‘_ò b�ØŸ*™*Ÿ.™.¨¨BÓ/�ÙØØ˜WÑ$Ü&2°7¸6±?È5Ô&Q�G˜F‘OØ! WÑ,´·±ÀÀtÁÔ1MÜ.:Ø Ñ/Ø# D™k×/Ñ/Ø$ T™{×1Ñ1ô/�G˜NÑ+ð
 " WÒ,Ü.:¸7À>Ñ;RÐ[`Ô.a�G˜NÒ+ñbñ	br    c           
      óŒ  — d}|�$t        j                  «       5   |«       }ddd«       | j                  D �]þ  }|d   D �]ò  }|j                  €Œ|j                  }|j                  rt        d«      ‚| j                  |   }t        |«      dk(  rƒt        d¬«      |d<   t        j                  |«      |d<   t        j                  |d	   «      rt        j                  |d	   «      nd
|d<   t        d¬«      |d<   t        j                  |«      |d<   |d   |d   }}|d   \  }	}
|d   j                  d«       d|
z
  }d|	z
  }|j                  |
«      j                  |||¬«       |d   |	z  ||d	   z  z   |d<   |d   |
z  |z   |d<   t        j                  |d	   «      rpt        j                  |d	   d
k7  |d	   t        j                  |d	   «      «      }t        j                  |d	   d
k7  |d   |z  t        j                  |d	   «      «      }n|d	   d
k7  r|d   |d	   z  nd}|d   }d|z  }|j!                  |«      j#                  «       j                  |d   «      }||z  }|j                  |	«       t%        |||d	   |z  «       |d   rU||z  dkD  j'                  |j(                  «      }|j+                  |j-                  «       j/                  d¬«      «       ||z  }t%        ||| «       |d   dk7  s�Œ¼|d   r|d	   dz  | j0                  d	   z  }n|d	   }t%        ||| |d   z  «       �Œõ �Œ |S # 1 sw Y   �ŒxY w)z¶Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   z(LaProp does not support sparse gradientsr   r#   r$   r"   Úexp_avgr   r   r&   r(   Ú
exp_avg_sqr   r   )Úvaluer   r   r   gü©ñÒMbP?)Úminr   r   é   )r.   Úenable_gradr*   ÚgradÚ	is_sparseÚRuntimeErrorr,   Úlenr   Ú
zeros_liker/   Úadd_Úmul_Úaddcmul_ÚwhereÚ	ones_likeÚdivÚsqrt_r   Útor'   Údiv_ÚmeanÚclamp_r   )r   ÚclosureÚlossr0   r1   r:   r,   r4   r5   Úbeta1Úbeta2Úone_minus_beta2Úone_minus_beta1Úlr_safeÚbias_correction1Úbias_correction2Ú	step_sizeÚdenomÚstep_of_this_gradÚmaskÚwd_scales                        r   r"   zLaProp.stepO   s…  € ð ˆØÐÜ×"Ñ"Ó$ñ !Ù“y�÷!ð ×&Ñ&ó A	JˆEØ˜8‘_ó @J�Ø—6‘6�>ØØ—v‘v�Ø—>’>Ü&Ð'QÓRÐRàŸ
™
 1™�ô �u“: ’?Ü$0¸Ô$>�E˜&‘Mä',×'7Ñ'7¸Ó':�E˜)Ñ$äMRÏ_É_Ð]bÐcgÑ]hÔMi¬E×,<Ñ,<¸UÀ4¹[Ô,IÐoq�E˜.Ñ)Ü,8ÀÔ,F�E˜.Ñ)ä*/×*:Ñ*:¸1Ó*=�E˜,Ñ'à&+¨IÑ&6¸¸lÑ8K˜�Ø$ W™~‘��uà�f‘×"Ñ" 1Ô%Ø"# e¡)�Ø"# e¡)�ð —‘ Ó&×/Ñ/°°dÀ/Ð/ÔRà(-¨nÑ(=ÀÑ(EÈÐZ_Ð`dÑZeÑHeÑ(e��nÑ%Ø(-¨nÑ(=ÀÑ(EÈÑ(W��nÑ%ô —?‘? 5¨¡;Ô/Ü#Ÿk™k¨%°©+¸Ñ*;¸UÀ4¹[Ì%Ï/É/ÐZ_Ð`dÑZeÓJfÓg�GÜ',§{¡{Ø˜d™ rÑ)Ø˜nÑ-°Ñ7ÜŸ™¨¨d©Ó4ó(Ñ$ð OTÐTXÉkÐ]_ÒN_ u¨^Ñ'<¸uÀT¹{Ò'JÐegÐ$Ø#(¨Ñ#8Ð ØÐ 0Ñ0�	à"Ÿ™Ð'7Ó8×>Ñ>Ó@×EÑEÀeÈEÁlÓS�Ø$(¨5¡LÐ!Ø—‘˜UÔ#Ü˜WÐ&7¸¸t¹ÀÑ9VÔWà˜Ò#à# d™N¨QÑ.×2Ñ2°4·:±:Ó>�DØ—I‘I˜dŸi™i›k×0Ñ0°TÐ0Ó:Ô;Ø%¨™n�Gä˜Q ¨)¨Ô4à˜Ñ(¨AÔ-ØÐ5Ò6Ø#(¨¡;°!Ñ#3°d·m±mÀDÑ6IÑ#I™à#(¨¡;˜Ü   A¨ y°5¸Ñ3HÑ'HÖIòA@JðA	JðF ˆ÷M!ñ !ús   ™L9Ì9M)g-Cëâ6:?)gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?gVçž¯Ò<r   FF)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r	   Úfloatr   Úboolr   r)   r.   Úno_gradr"   Ú__classcell__)r   s   @r   r   r      s’   ø„ ñð Ø)5ØØ"$Ø!Ø+0ñ7àð7ð ð7ð ˜ ˜Ñ&ð	7ð
 ð7ð  ð7ð ð7ð %)õ7ô4bð( €U‡]�]ƒ_òOó ôOr    r   )r[   Útypingr   Útorch.optimr   r.   Ú_helpersr   r   r   Ú_typesr	   r   © r    r   ú<module>re      s,   ðñõ$ å !Û ç BÑ BÝ ôCˆYõ Cr    