Ë
    éÓ^jY  ã                   ó:   — d Z ddlZddlmZ ddlZ G d„ de«      Zy)a6   Nvidia NovoGrad Optimizer.
Original impl by Nvidia from Jasper example:
    - https://github.com/NVIDIA/DeepLearningExamples/blob/master/PyTorch/SpeechRecognition/Jasper
Paper: `Stochastic Gradient Methods with Layer-wise Adaptive Moments for Training of Deep Networks`
    - https://arxiv.org/abs/1905.11286
é    N)Ú	Optimizerc                   ój   ‡ — e Zd ZdZ	 	 	 	 	 	 dˆ fd„	Zˆ fd„Z ej                  «       dd„«       Zˆ xZ	S )Ú
NvNovoGrada(  
    Implements Novograd algorithm.

    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.95, 0.98))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        grad_averaging: gradient averaging
        amsgrad (boolean, optional): whether to use the AMSGrad variant of this
            algorithm from the paper `On the Convergence of Adam and Beyond`_
            (default: False)
    c                 ó€  •— d|k  st        dj                  |«      «      ‚d|k  st        dj                  |«      «      ‚d|d   cxk  rdk  sn t        dj                  |d   «      «      ‚d|d   cxk  rdk  sn t        dj                  |d   «      «      ‚t        ||||||¬	«      }t        t        | �  ||«       y )
Ng        zInvalid learning rate: {}zInvalid epsilon value: {}r   g      ð?z%Invalid beta parameter at index 0: {}é   z%Invalid beta parameter at index 1: {})ÚlrÚbetasÚepsÚweight_decayÚgrad_averagingÚamsgrad)Ú
ValueErrorÚformatÚdictÚsuperr   Ú__init__)
ÚselfÚparamsr   r	   r
   r   r   r   ÚdefaultsÚ	__class__s
            €ú`/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/optim/nvnovograd.pyr   zNvNovoGrad.__init__    sÊ   ø€ ð �bŠyÜÐ8×?Ñ?ÀÓCÓDÐDØ�cŠzÜÐ8×?Ñ?ÀÓDÓEÐEØ�e˜A‘hÔ$ Ô$ÜÐD×KÑKÈEÐRSÉHÓUÓVÐVØ�e˜A‘hÔ$ Ô$ÜÐD×KÑKÈEÐRSÉHÓUÓVÐVÜØØØØ%Ø)Øô
ˆô 	Œj˜$Ñ(¨°Õ:ó    c                 ór   •— t         t        | �  |«       | j                  D ]  }|j	                  dd«       Œ y )Nr   F)r   r   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroupr   s      €r   r   zNvNovoGrad.__setstate__=   s7   ø€ ÜŒj˜$Ñ,¨UÔ3Ø×&Ñ&ò 	/ˆEØ×Ñ˜Y¨Õ.ñ	/r   c                 ó2  — d}|�$t        j                  «       5   |«       }ddd«       | j                  D �]Q  }|d   D �]E  }|j                  €Œ|j                  }|j                  rt        d«      ‚|d   }| j                  |   }t        |«      dk(  r‡d|d<   t        j                  |«      |d<   t        j                  g «      j                  |d   j                  «      |d<   |r4t        j                  g «      j                  |d   j                  «      |d	<   |d   |d   }	}|r|d	   }
|d
   \  }}|dxx   dz  cc<   t        j                  t        j                  |d«      «      }|	dk(  r|	j                  |«       n%|	j                  |«      j!                  |d|z
  ¬«       |r;t        j"                  
|	|
¬«       |
j%                  «       j!                  |d   «      }n"|	j%                  «       j!                  |d   «      }|j'                  |«       |d   dk7  r|j!                  ||d   ¬«       |d   r|j                  d|z
  «       |j                  |«      j!                  |«       |j!                  ||d    ¬«       �ŒH �ŒT |S # 1 sw Y   �ŒmxY w)z²Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
            and returns the loss.
        Nr   z#Sparse gradients are not supported.r   r   ÚstepÚexp_avgÚ
exp_avg_sqÚmax_exp_avg_sqr	   r   é   )Úalpha)Úoutr
   r   r   r   )ÚtorchÚenable_gradr   ÚgradÚ	is_sparseÚRuntimeErrorr   ÚlenÚ
zeros_likeÚzerosÚtoÚdeviceÚsumÚpowÚcopy_Úmul_Úadd_ÚmaxÚsqrtÚdiv_)r   ÚclosureÚlossr   Úpr)   r   r   r!   r"   r#   Úbeta1Úbeta2ÚnormÚdenoms                  r   r    zNvNovoGrad.stepB   sr  € ð ˆØÐÜ×"Ñ"Ó$ñ !Ù“y�÷!ð ×&Ñ&ó 3	4ˆEØ˜8‘_ó 24�Ø—6‘6�>ØØ—v‘v�Ø—>’>Ü&Ð'LÓMÐMØ 	Ñ*�àŸ
™
 1™�ô �u“: ’?Ø$%�E˜&‘Mä',×'7Ñ'7¸Ó':�E˜)Ñ$ä*/¯+©+°b«/×*<Ñ*<¸UÀ9Ñ=M×=TÑ=TÓ*U�E˜,Ñ'Ùä27·+±+¸b³/×2DÑ2DÀUÈ9ÑEU×E\ÑE\Ó2]˜Ð.Ñ/à&+¨IÑ&6¸¸lÑ8K˜�ÙØ%*Ð+;Ñ%<�NØ$ W™~‘��uà�f“ Ñ"“ä—y‘y¤§¡¨4°Ó!3Ó4�à ’?Ø×$Ñ$ TÕ*à—O‘O EÓ*×/Ñ/°¸AÀ¹IÐ/ÔFáä—I‘I˜n¨j¸nÕMà*×/Ñ/Ó1×6Ñ6°u¸U±|ÓD‘Eà&ŸO™OÓ-×2Ñ2°5¸±<Ó@�Eà—	‘	˜%Ô Ø˜Ñ(¨AÒ-Ø—I‘I˜a u¨^Ñ'<�IÔ=ØÐ)Ò*Ø—I‘I˜a %™iÔ(Ø—‘˜UÓ#×(Ñ(¨Ô.à—‘�w u¨T¡{ l�Ö3òe24ð3	4ðj ˆ÷q!ñ !ús   ™JÊJ)gü©ñÒMbP?)gffffffî?g\�Âõ(\ï?g:Œ0âŽyE>r   FF)N)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r'   Úno_gradr    Ú__classcell__)r   s   @r   r   r      sD   ø„ ñð* ØØØØ Øõ;ô:/ð
 €U‡]�]ƒ_òAó ôAr   r   )rC   r'   Útorch.optim.optimizerr   Úmathr   © r   r   ú<module>rI      s#   ðñó Ý +Û ôw�õ wr   