
    ^j?              !          d Z ddlmZmZmZ ddlZddlmZ ddlmZm	Z	m
Z
 ddlmZ  G d d	ej                  j                        Z	 	 dd
ee   dee   dee   dee   dee   dee   dededededededededee   ddf dZd
ee   dee   dee   dee   dee   dededededededededee   fdZd
ee   dee   dee   dee   dee   dededededededededee   fdZy)ad   NAdamW Optimizer

Based on simplified algorithm in https://github.com/mlcommons/algorithmic-efficiency/tree/main/baselines/nadamw

Added multi-tensor (foreach) path.

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Why Gradients Rapidly Increase Near the End of Training: https://arxiv.org/abs/2506.02285
    )ListOptionalTupleN)Tensor   )_check_capturable_devices
_get_value_init_scalar)ParamsTc                        e Zd ZdZ	 	 	 	 	 	 	 	 	 ddededeeef   dedededed	ed
ee   def fdZ	 fdZ
 ej                         dd       Z xZS )NAdamWa   Implements NAdamW algorithm.

    See Table 1 in https://arxiv.org/abs/1910.05446 for the implementation of
    the NAdam algorithm (there is also a comment in the code which highlights
    the only difference of NAdamW and AdamW).

    For further details regarding the algorithm we refer to
        - Decoupled Weight Decay Regularization: https://arxiv.org/abs/1711.05101
        - On the Convergence of Adam and Beyond: https://openreview.net/forum?id=ryQu7f-RZ

    Args:
        params: iterable of parameters to optimize or dicts defining parameter groups
        lr: learning rate
        betas: coefficients used for computing running averages of gradient and its square
        eps: term added to the denominator to improve numerical stability
        weight_decay: weight decay coefficient
        caution: enable caution
        corrected_weight_decay: apply corrected weight decay (lr**2 / max_lr)
    paramslrbetasepsweight_decaycautioncorrected_weight_decaymaximizeforeach
capturablec                 D   d|k  st        d|       d|k  st        d|       d|d   cxk  rdk  sn t        d|d          d|d   cxk  rdk  sn t        d|d          d|k  st        d	|       t        |||||||	||

	      }t        |   ||       y )Ng        zInvalid learning rate: zInvalid epsilon value: r         ?z#Invalid beta parameter at index 0: r   z#Invalid beta parameter at index 1: zInvalid weight_decay value: )	r   r   r   r   r   r   r   r   r   )
ValueErrordictsuper__init__)selfr   r   r   r   r   r   r   r   r   r   defaults	__class__s               \/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/optim/nadamw.pyr   zNAdamW.__init__*   s     by6rd;<<cz6se<==eAh$$B58*MNNeAh$$B58*MNNl";L>JKK%#9!

 	*    c                    t         |   |       | j                  D ]  }|j                  dd       |j                  dd       |j                  dd        |j                  dd       |j                  dd       |d   D ]J  }| j                  j                  |i       }|s"d|v s't        |d   |d   r|j                  nd	
      |d<   L  y )Nr   Fr   r   r   r   r   stepcpudevice)r   __setstate__param_groups
setdefaultstategetr
   r'   )r   r+   grouppp_stater    s        r!   r(   zNAdamW.__setstate__N   s    U#&& 	EY.5u=Y-Z/\518_ **..B/v0&2+0+>qxxE'GFO	r"   c                    t        | d      r| j                          nt        | d      r| j                          d}|$t        j                         5   |       }ddd       | j
                  D ]  }g }g }g }g }g }|d   \  }	}
|d   D ]&  }|j                  |j                  |       |j                  j                  rt        d      |j                  |j                         | j                  |   }t        |      dk(  rpt        |d   r|j                  nd	
      |d<   t        j                  |t        j                        |d<   t        j                  |t        j                        |d<   |j                  |d          |j                  |d          |j                  |d          ) t!        ||||||d   |	|
|d   |d   |d   |d   |d   |d   |d   r| j"                  d   nd        |S # 1 sw Y   xY w)zPerforms a single optimization step.

            Args:
              closure (callable, optional): A closure that reevaluates the model
                  and returns the loss.
        '_accelerator_graph_capture_health_check _cuda_graph_capture_health_checkNr   r   z(NAdamW does not support sparse gradientsr   r   r%   r&   r$   )memory_formatexp_avg
exp_avg_sqr   r   r   r   r   r   r   )
r   beta1beta2r   r   r   r   r   r   max_lr)hasattrr1   r2   torchenable_gradr)   gradappend	is_sparseRuntimeErrorr+   lenr
   r'   
zeros_likepreserve_formatnadamwr   )r   closurelossr-   params_with_gradgradsexp_avgsexp_avg_sqsstate_stepsr6   r7   r.   r+   s                r!   r$   zNAdamW.step^   s    4BC88:T=>113""$ !y! && .	E!EHKK >LE58_ 266> ''*66##&'QRRQVV$

1 u:?$0E,DW]b$cE&M','7'7I^I^'_E)$*/*:*:1ELaLa*bE,'i 01""5#67""5=1)2,  i(;">2%Li(z* ..34L.Mt}}T*SW=.	` g! !s   G;;H)	MbP?)g?g+?g:0yE>g{Gz?FFFNFN)__name__
__module____qualname____doc__r   floatr   boolr   r   r(   r:   no_gradr$   __classcell__)r    s   @r!   r   r      s    . )5"&!+0"&*$"+"+ "+ &	"+
 "+  "+ "+ %)"+ "+ d^"+ "+H  U]]_A Ar"   r   r   rG   rH   rI   rJ   r   r   r6   r7   r   r   r   r   r   r8   returnc                   t        d |D              st        d      |U	 | xs4 dt        j                  j                  j
                  j                         v }|rt        j                  |	      r|sd}|r%t        j                  j                         st        }nt        } || |||||||	|
|||||       y# t        $ r d}Y Qw xY w)zcFunctional API that performs NAdamW algorithm computation.
      See NAdamW class for details.
    c              3   P   K   | ]  }t        |t        j                           y wrL   )
isinstancer:   r   ).0ts     r!   	<genexpr>znadamw.<locals>.<genexpr>   s     @qz!U\\*@s   $&zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNScalarF)	r6   r7   r   r   r   r   r   r   r8   )allr?   r:   opsaten_foreach_maximum_	overloads	is_tensor	Exceptionjitis_scripting_multi_tensor_nadamw_single_tensor_nadamw)r   rG   rH   rI   rJ   r   r   r6   r7   r   r   r   r   r   r8   funcs                   r!   rC   rC      s    , @K@@!" 	" 	!k]X1Q1Q1[1[1]%]G5??2.z uyy--/#$!  	G	s   AB9 9CCc       	         p   |rt        | |       t        |       D ]  \  }}|s||   n||    }||   }||   }||   }|dz  }||n|dz  |z  }|j                  d||z  z
         |j                  |      j                  |d|z
         |j                  |      j	                  ||d|z
         |r|}dt        j                  ||      z
  }dt        j                  ||      z
  }||z  }|j                         }|j                         }|j                  |      j                  |d|z
        }|j                         ||z  z  j                  |	|z        }|
ra||z  dkD  j                  |j                        }|j                  |j                         j                  d             |j                  |       |j                  ||       t!        |      }d||z  z
  }d||z  z
  }||z  }|d	z  }|j                  |      j                  |d|z
        }|j                         |z  j                  |	      }|
ra||z  dkD  j                  |j                        }|j                  |j                         j                  d             |j                  |       |j                  |||         y )
Nr      r   alpha)valuer   rK   )min      ?)r   	enumeratemul_add_addcmul_r:   pownegsqrtmultodtypediv_meanclamp_addcdiv_r	   )r   rG   rH   rI   rJ   r6   r7   r   r   r   r   r   r   r8   iparamr<   r4   r5   step_twd_scaler$   bias_correction1bias_correction2	step_sizestep_size_negbias_correction2_sqrtdenommasks                                r!   rg   rg      s   " !&+6f% >=5'uQxeAhY1+ ^
Q 	!  2R1Wv-=

2<//0 	U  QY 7''d!e)'DD  !599UD#99 599UD#99--I%MMOM$4$9$9$;! kk%(--d!e)-DG__&*?-*OPVVWZ]jWjkE  $*..tzz:		$))+,,,67T"NN7E*f%D 5D=0 5D=0--I$4$;! kk%(--d!e)-DG__&)>>DDSIE$*..tzz:		$))+,,,67T"NN7E)N<}>=r"   c       	         X   t        |       dk(  ry |rt        | |       |rt        j                  t	        |            }|D cg c].  }t        j
                  |      rt        j                  |      n|0 }}|D cg c].  }t        j
                  |      rt        j                  |      n|0 }}|D cg c].  }t        j
                  |      rt        j                  |      n|0 }}| D cg c].  }t        j
                  |      rt        j                  |      n|0 } }t        j                  |d       ||n|dz  |z  }t        j                  | d||z  z
         t        j                  ||       t        j                  ||d|z
         t        j                  ||       t        j                  |||d|z
         |rk|D cg c]  }t        j                  ||       }}|D cg c]  }t        j                  ||       }}t        j                  |d       t        j                  |d       t        j                  |       t        j                  |       t        j                  ||      }t        j                  |       t        j                  |       t        j                  |      }t        j                   ||      }t        j                  ||d|z
         t        j                  |      }t        j"                  |t        j                   ||             t        j                  ||	      }t        j                  |       t        j$                  ||      }|
rt        j                   ||      }t'        ||      D cg c]#  \  }}|dkD  j)                  |j*                        % }}}|D cg c]  }|j-                          }}t        j.                  |d       t        j"                  ||       t        j                  ||       t        j0                  | ||       y |D cg c]  }d|t3        |      z  z
   }}|D cg c]  }d|t3        |      z  z
   }}|D cg c]
  }||z  dz   }}|D cg c]  }|dz  	 }}t        j                   ||      }t        j                  ||d|z
         t        j                  |      }t        j"                  ||       t        j$                  ||	      }|
rt        j                   ||      }t'        ||      D cg c]#  \  }}|dkD  j)                  |j*                        % }}}|D cg c]  }|j-                          }}t        j.                  |d       t        j"                  ||       t        j                  ||       t        j"                  ||       t        j0                  | ||       y c c}w c c}w c c}w c c}w c c}w c c}w c c}}w c c}w c c}w c c}w c c}w c c}w c c}}w c c}w )Nr   r   rj   rk   rK   ro   )r@   r   r:   _foreach_negtuple
is_complexview_as_real_foreach_add__foreach_mul__foreach_addcmul_rt   _foreach_sub__foreach_neg__foreach_div_foreach_reciprocal__foreach_sqrt_foreach_mul_foreach_div__foreach_addziprx   ry   r{   r`   _foreach_addcdiv_r	   )r   rG   rH   rI   rJ   r6   r7   r   r   r   r   r   r   r8   xr   r$   r   r   r   r   exp_avg_sq_sqrteps_over_step_sizer   masksmg
mask_scalebcs                                r!   rf   rf   4  s@   " 6{a!&+6""5<0JOPQe&6&6q&9U"q@PEPMUV)9)9!)<""1%!CVHVP[\1E,<,<Q,?5%%a(QF\K\KQRau'7'7':e  #ARFR 
Q' ^rq6)9H	X%< <= 
%(	%q5y9	U+	Kq5yA?JKtEIIeT2KK?JKtEIIeT2KK,a0,a0,-,- &&'7<	""9-I& % 3 34D E %%h6He1u9=--k:4i@	
 #//	3?""#56""?4FG&&x7E585FGTQa!eZZ(GEG,12q!&&(2J2##J5z2%0%8FQRdAD)9 99RRFQRdAD)9 99RR.>?b2g^?	?5E Frs F F %%h6He1u9=--k:O-BC""?C8&&x7E585FGTQa!eZZ(GEG,12q!&&(2J2##J5z2%0
 	E9-%8y QV\R$ LK> H2 SR? F H2sT   3W$;3W)43W.-3W3	W8,W=(X=X4XX0XX,(X!X')NF)rP   typingr   r   r   r:   r   _helpersr   r	   r
   _typesr   optim	Optimizerr   rR   rQ   rC   rg   rf    r"   r!   <module>r      si  	 ) (   I I KU[["" Kh #' 9V9F|9 v,9 &\	9
 &\9 $9 9 9 9 9 9 9 9 9  !9" 
#9xR=VR=F|R= v,R= &\	R=
 &\R= R= R= R= R= R= R= R= R= R=jv9Vv9F|v9 v,v9 &\	v9
 &\v9 v9 v9 v9 v9 v9 v9 v9 v9 v9r"   