Ë
    éÓ^j?  ã            !       óÎ  — d Z ddlmZmZmZ ddlZddlmZ ddlmZm	Z	m
Z
 ddlmZ  G d„ d	ej                  j                  «      Z	 	 dd
ee   dee   dee   dee   dee   dee   dededededededededee   ddf d„Zd
ee   dee   dee   dee   dee   dededededededededee   fd„Zd
ee   dee   dee   dee   dee   dededededededededee   fd„Zy)ad   NAdamW Optimizer

Based on simplified algorithm in https://github.com/mlcommons/algorithmic-efficiency/tree/main/baselines/nadamw

Added multi-tensor (foreach) path.

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Why Gradients Rapidly Increase Near the End of Training: https://arxiv.org/abs/2506.02285
é    )ÚListÚOptionalÚTupleN)ÚTensoré   )Ú_check_capturable_devicesÚ
_get_valueÚ_init_scalar)ÚParamsTc                   óª   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 ddededeeef   dedededed	ed
ee   defˆ fd„Z	ˆ fd„Z
 ej                  «       dd„«       Zˆ xZS )ÚNAdamWa”   Implements NAdamW algorithm.

    See Table 1 in https://arxiv.org/abs/1910.05446 for the implementation of
    the NAdam algorithm (there is also a comment in the code which highlights
    the only difference of NAdamW and AdamW).

    For further details regarding the algorithm we refer to
        - Decoupled Weight Decay Regularization: https://arxiv.org/abs/1711.05101
        - On the Convergence of Adam and Beyond: https://openreview.net/forum?id=ryQu7f-RZ

    Args:
        params: iterable of parameters to optimize or dicts defining parameter groups
        lr: learning rate
        betas: coefficients used for computing running averages of gradient and its square
        eps: term added to the denominator to improve numerical stability
        weight_decay: weight decay coefficient
        caution: enable caution
        corrected_weight_decay: apply corrected weight decay (lr**2 / max_lr)
    ÚparamsÚlrÚbetasÚepsÚweight_decayÚcautionÚcorrected_weight_decayÚmaximizeÚforeachÚ
capturablec                 óD  •— d|k  st        d|› �«      ‚d|k  st        d|› �«      ‚d|d   cxk  rdk  sn t        d|d   › �«      ‚d|d   cxk  rdk  sn t        d|d   › �«      ‚d|k  st        d	|› �«      ‚t        |||||||	||
¬
«	      }t        ‰| �  ||«       y )Ng        zInvalid learning rate: zInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: r   z#Invalid beta parameter at index 1: zInvalid weight_decay value: )	r   r   r   r   r   r   r   r   r   )Ú
ValueErrorÚdictÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   r   r   r   ÚdefaultsÚ	__class__s               €ú\/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/optim/nadamw.pyr   zNAdamW.__init__*   sÚ   ø€ ð �bŠyÜÐ6°r°dÐ;Ó<Ð<Ø�cŠzÜÐ6°s°eÐ<Ó=Ð=Ø�e˜A‘hÔ$ Ô$ÜÐBÀ5ÈÁ8À*ÐMÓNÐNØ�e˜A‘hÔ$ Ô$ÜÐBÀ5ÈÁ8À*ÐMÓNÐNØ�lÒ"ÜÐ;¸L¸>ÐJÓKÐKÜØØØØ%ØØ#9ØØØ!ô

ˆô 	‰Ñ˜ Õ*ó    c                 óž  •— t         ‰| �  |«       | j                  D ]®  }|j                  dd«       |j                  dd«       |j                  dd «       |j                  dd«       |j                  dd«       |d   D ]J  }| j                  j                  |i «      }|sŒ"d|v sŒ't        |d   |d   r|j                  nd	¬
«      |d<   ŒL Œ° y )Nr   Fr   r   r   r   r   ÚstepÚcpu©Údevice)r   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetr
   r'   )r   r+   ÚgroupÚpÚp_stater    s        €r!   r(   zNAdamW.__setstate__N   sÌ   ø€ Ü‰Ñ˜UÔ#Ø×&Ñ&ò 	ˆEØ×Ñ˜Y¨Ô.Ø×ÑÐ5°uÔ=Ø×Ñ˜Y¨Ô-Ø×Ñ˜Z¨Ô/Ø×Ñ˜\¨5Ô1Ø˜8‘_ò �ØŸ*™*Ÿ.™.¨¨BÓ/�Ú˜v¨Ò0Ü&2Ø ™Ø+0°Ò+>˜qŸxšxÀEô'�G˜F’Oññ	r"   c                 ó  — t        | d«      r| j                  «        nt        | d«      r| j                  «        d}|�$t        j                  «       5   |«       }ddd«       | j
                  D �]‡  }g }g }g }g }g }|d   \  }	}
|d   D �]&  }|j                  €Œ|j                  |«       |j                  j                  rt        d«      ‚|j                  |j                  «       | j                  |   }t        |«      dk(  rpt        |d   r|j                  nd	¬
«      |d<   t        j                  |t        j                  ¬«      |d<   t        j                  |t        j                  ¬«      |d<   |j                  |d   «       |j                  |d   «       |j                  |d   «       �Œ) t!        ||||||d   |	|
|d   |d   |d   |d   |d   |d   |d   r| j"                  d   nd¬«       �ŒŠ |S # 1 sw Y   �Œ£xY w)z¹Performs a single optimization step.

            Args:
              closure (callable, optional): A closure that reevaluates the model
                  and returns the loss.
        Ú'_accelerator_graph_capture_health_checkÚ _cuda_graph_capture_health_checkNr   r   z(NAdamW does not support sparse gradientsr   r   r%   r&   r$   )Úmemory_formatÚexp_avgÚ
exp_avg_sqr   r   r   r   r   r   r   )
r   Úbeta1Úbeta2r   r   r   r   r   r   Úmax_lr)Úhasattrr1   r2   ÚtorchÚenable_gradr)   ÚgradÚappendÚ	is_sparseÚRuntimeErrorr+   Úlenr
   r'   Ú
zeros_likeÚpreserve_formatÚnadamwr   )r   ÚclosureÚlossr-   Úparams_with_gradÚgradsÚexp_avgsÚexp_avg_sqsÚstate_stepsr6   r7   r.   r+   s                r!   r$   zNAdamW.step^   s  € ô �4ÐBÔCØ×8Ñ8Õ:Ü�TÐ=Ô>Ø×1Ñ1Ô3àˆØÐÜ×"Ñ"Ó$ñ !Ù“y�÷!ð ×&Ñ&ó .	ˆEØ!ÐØˆEØˆHØˆKØˆKØ  ™>‰LˆE�5à˜8‘_ó 2�Ø—6‘6�>ØØ ×'Ñ'¨Ô*Ø—6‘6×#Ò#Ü&Ð'QÓRÐRØ—‘˜QŸV™VÔ$àŸ
™
 1™�ô �u“: ’?Ü$0ÀEÈ,ÒDW¸¿ºÐ]bÔ$c�E˜&‘Mä',×'7Ñ'7¸Ì×I^ÑI^Ô'_�E˜)Ñ$ä*/×*:Ñ*:¸1ÌE×LaÑLaÔ*b�E˜,Ñ'à—‘  iÑ 0Ô1Ø×"Ñ" 5¨Ñ#6Ô7Ø×"Ñ" 5¨¡=Ö1ð)2ô, Ø ØØØØØ˜iÑ(ØØØ˜‘;Ø" >Ñ2Ø˜%‘LØ˜iÑ(Ø˜zÑ*Ø  Ñ.Ø.3Ð4LÒ.M�t—}‘} TÒ*ÐSW÷ð=.	ð` ˆ÷g!ñ !ús   ÁG;Ç;H)	çü©ñÒMbP?)gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?g:Œ0âŽyE>g{®Gáz„?FFFNF©N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úfloatr   Úboolr   r   r(   r:   Úno_gradr$   Ú__classcell__)r    s   @r!   r   r      s½   ø„ ñð. Ø)5ØØ"&Ø!Ø+0Ø"Ø&*Ø$ñ"+àð"+ð ð"+ð ˜ ˜Ñ&ð	"+ð
 ð"+ð  ð"+ð ð"+ð %)ð"+ð ð"+ð ˜d‘^ð"+ð õ"+ôHð  €U‡]�]ƒ_òAó ôAr"   r   r   rG   rH   rI   rJ   r   r   r6   r7   r   r   r   r   r   r8   Úreturnc                ó”  — t        d„ |D «       «      st        d«      ‚|€U	 | xs4 dt        j                  j                  j
                  j                  «       v }|rt        j                  |	«      r|sd}|r%t        j                  j                  «       st        }nt        } || |||||||	|
|||||¬«       y# t        $ r d}Y ŒQw xY w)zcFunctional API that performs NAdamW algorithm computation.
      See NAdamW class for details.
    c              3   óP   K  — | ]  }t        |t        j                  «      –— Œ  y ­wrL   )Ú
isinstancer:   r   )Ú.0Úts     r!   ú	<genexpr>znadamw.<locals>.<genexpr>¹   s   è ø€ Ò@¨qŒz˜!œUŸ\™\×*Ñ@ùs   ‚$&zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNÚScalarF)	r6   r7   r   r   r   r   r   r   r8   )Úallr?   r:   ÚopsÚatenÚ_foreach_maximum_Ú	overloadsÚ	is_tensorÚ	ExceptionÚjitÚis_scriptingÚ_multi_tensor_nadamwÚ_single_tensor_nadamw)r   rG   rH   rI   rJ   r   r   r6   r7   r   r   r   r   r   r8   Úfuncs                   r!   rC   rC   £   sÓ   € ô, Ñ@°KÔ@Ô@Üð!ó"ð 	"ð €ð	à!�kÒ] X´·±·±×1QÑ1Q×1[Ñ1[Ó1]Ð%]ˆGáœ5Ÿ?™?¨2Ô.±zØ�ñ ”u—y‘y×-Ñ-Ô/Ü#‰ä$ˆáØØØØØØØØØ!ØØØØØöøô ò 	ØŠGð	ús   ¡AB9 Â9CÃCc       	         óp  — |rt        | |«       t        | «      D �]™  \  }}|s||   n||    }||   }||   }||   }|dz  }|€|n|dz  |z  }|j                  d||z  z
  «       |j                  |«      j                  |d|z
  ¬«       |j                  |«      j	                  ||d|z
  ¬«       |�r|}dt        j                  ||«      z
  }dt        j                  ||«      z
  }||z  }|j                  «       }|j                  «       }|j                  |«      j                  |d|z
  ¬«      }|j                  «       ||z  z  j                  |	|z  «      }|
ra||z  dkD  j                  |j                  «      }|j                  |j                  «       j                  d¬«      «       |j                  |«       |j                  ||«       �Œ¶t!        |«      }d||z  z
  }d||z  z
  }||z  }|d	z  }|j                  |«      j                  |d|z
  ¬«      }|j                  «       |z  j                  |	«      }|
ra||z  dkD  j                  |j                  «      }|j                  |j                  «       j                  d¬«      «       |j                  |«       |j                  ||| ¬«       �Œœ y )
Nr   é   r   ©Úalpha)Úvaluer   rK   )Úminç      à?)r   Ú	enumerateÚmul_Úadd_Úaddcmul_r:   ÚpowÚnegÚsqrtÚmulÚtoÚdtypeÚdiv_ÚmeanÚclamp_Úaddcdiv_r	   )r   rG   rH   rI   rJ   r6   r7   r   r   r   r   r   r   r8   ÚiÚparamr<   r4   r5   Ústep_tÚwd_scaler$   Úbias_correction1Úbias_correction2Ú	step_sizeÚstep_size_negÚbias_correction2_sqrtÚdenomÚmasks                                r!   rg   rg   ß   s¶  € ñ" Ü! &¨+Ô6ä˜fÓ%ó >=‰ˆˆ5Ù'ˆu�QŠx¨e°A©h¨YˆØ˜1‘+ˆØ  ‘^ˆ
Ø˜Q‘ˆð 	�!‰ˆð  ˜‘2¨R°1©W°vÑ-=ˆØ�
‰
�2˜ <Ñ/Ñ/Ô0ð 	�‰�UÓ× Ñ  ¨Q°©YÐ Ô7Ø�‰˜Ó×'Ñ'¨¨d¸!¸e¹)Ð'ÔDâØˆDð  !¤5§9¡9¨U°DÓ#9Ñ9ÐØ ¤5§9¡9¨U°DÓ#9Ñ9ÐàÐ-Ñ-ˆIØ%ŸM™M›OˆMà$4×$9Ñ$9Ó$;Ð!ð —k‘k %Ó(×-Ñ-¨d¸!¸e¹)Ð-ÓDˆGà—_‘_Ó&Ð*?À-Ñ*OÑP×VÑVÐWZÐ]jÑWjÓkˆEáð   $™¨Ñ*×.Ñ.¨t¯z©zÓ:�Ø—	‘	˜$Ÿ)™)›+×,Ñ,°Ð,Ó6Ô7Ø—‘˜TÔ"à�N‰N˜7 EÖ*ä˜fÓ%ˆDØ  5¨D¡=Ñ0ÐØ  5¨D¡=Ñ0ÐØÐ-Ñ-ˆIØ$4¸Ñ$;Ð!ð —k‘k %Ó(×-Ñ-¨d¸!¸e¹)Ð-ÓDˆGØ—_‘_Ó&Ð)>Ñ>×DÑDÀSÓIˆEáà $™¨Ñ*×.Ñ.¨t¯z©zÓ:�Ø—	‘	˜$Ÿ)™)›+×,Ñ,°Ð,Ó6Ô7Ø—‘˜TÔ"à�N‰N˜7 E°)°ˆNÖ<ñ}>=r"   c       	         óX  — t        | «      dk(  ry |rt        | |«       |rt        j                  t	        |«      «      }|D �cg c].  }t        j
                  |«      rt        j                  |«      n|‘Œ0 }}|D �cg c].  }t        j
                  |«      rt        j                  |«      n|‘Œ0 }}|D �cg c].  }t        j
                  |«      rt        j                  |«      n|‘Œ0 }}| D �cg c].  }t        j
                  |«      rt        j                  |«      n|‘Œ0 } }t        j                  |d«       |€|n|dz  |z  }t        j                  | d||z  z
  «       t        j                  ||«       t        j                  ||d|z
  ¬«       t        j                  ||«       t        j                  |||d|z
  «       |�rk|D �cg c]  }t        j                  ||«      ‘Œ }}|D �cg c]  }t        j                  ||«      ‘Œ }}t        j                  |d«       t        j                  |d«       t        j                  |«       t        j                  |«       t        j                  ||«      }t        j                  |«       t        j                  |«       t        j                  |«      }t        j                   ||«      }t        j                  ||d|z
  ¬«       t        j                  |«      }t        j"                  |t        j                   ||«      «       t        j                  ||	«      }t        j                  |«       t        j$                  ||«      }|
r¯t        j                   ||«      }t'        ||«      D ��cg c]#  \  }}|dkD  j)                  |j*                  «      ‘Œ% }}}|D �cg c]  }|j-                  «       ‘Œ }}t        j.                  |d«       t        j"                  ||«       t        j                  ||«       t        j0                  | ||«       y |D �cg c]  }d|t3        |«      z  z
  ‘Œ }}|D �cg c]  }d|t3        |«      z  z
  ‘Œ }}|D �cg c]
  }||z  dz  ‘Œ }}|D �cg c]  }|dz  ‘Œ	 }}t        j                   ||«      }t        j                  ||d|z
  ¬«       t        j                  |«      }t        j"                  ||«       t        j$                  ||	«      }|
r¯t        j                   ||«      }t'        ||«      D ��cg c]#  \  }}|dkD  j)                  |j*                  «      ‘Œ% }}}|D �cg c]  }|j-                  «       ‘Œ }}t        j.                  |d«       t        j"                  ||«       t        j                  ||«       t        j"                  ||«       t        j0                  | ||«       y c c}w c c}w c c}w c c}w c c}w c c}w c c}}w c c}w c c}w c c}w c c}w c c}w c c}}w c c}w )Nr   r   rj   rk   rK   éÿÿÿÿro   )r@   r   r:   Ú_foreach_negÚtupleÚ
is_complexÚview_as_realÚ_foreach_add_Ú_foreach_mul_Ú_foreach_addcmul_rt   Ú_foreach_sub_Ú_foreach_neg_Ú_foreach_divÚ_foreach_reciprocal_Ú_foreach_sqrtÚ_foreach_mulÚ_foreach_div_Ú_foreach_addÚziprx   ry   r{   r`   Ú_foreach_addcdiv_r	   )r   rG   rH   rI   rJ   r6   r7   r   r   r   r   r   r   r8   Úxr�   r$   r‚   rƒ   r„   r†   Úexp_avg_sq_sqrtÚeps_over_step_sizer‡   ÚmasksÚmÚgÚ
mask_scaleÚbcs                                r!   rf   rf   4  s@  € ô" ˆ6ƒ{�aÒØáÜ! &¨+Ô6áÜ×"Ñ"¤5¨£<Ó0ˆàJOÖPÀQ¤e×&6Ñ&6°qÔ&9ŒU×Ñ Ô"¸qÑ@ÐP€EÐPØMUÖVÈ¬×)9Ñ)9¸!Ô)<”×"Ñ" 1Ô%À!ÑCÐV€HÐVØP[Ö\È1¬E×,<Ñ,<¸QÔ,?”5×%Ñ% aÔ(ÀQÑFÐ\€KÐ\ØKQÖRÀa¤u×'7Ñ'7¸Ô':Œe× Ñ  Ô#ÀÑAÐR€FÐRô 
×Ñ˜ QÔ'ð �^‰r¨¨q©°6Ñ)9€HÜ	×Ñ˜  X°Ñ%<Ñ <Ô=ô 
×Ñ˜ %Ô(Ü	×Ñ˜ %¨q°5©yÕ9ä	×Ñ˜ UÔ+Ü	×Ñ˜K¨°°q¸5±yÔAâà?JÖK°tœEŸI™I e¨TÕ2ÐKÐÐKØ?JÖK°tœEŸI™I e¨TÕ2ÐKÐÐKä×ÑÐ,¨aÔ0Ü×ÑÐ,¨aÔ0Ü×ÑÐ,Ô-Ü×ÑÐ,Ô-ô ×&Ñ&Ð'7¸Ó<ˆ	Ü×"Ñ" 9Ô-Ü×Ñ˜IÔ&ä %× 3Ñ 3Ð4DÓ EÐô ×%Ñ% h°Ó6ˆÜ×Ñ˜H e°1°u±9Õ=ä×-Ñ-¨kÓ:ˆÜ×ÑØÜ×ÑÐ4°iÓ@ô	
ô #×/Ñ/°	¸3Ó?ÐÜ×"Ñ"Ð#5Ô6Ü×"Ñ" ?Ð4FÓGˆáä×&Ñ& x°Ó7ˆEÜ58¸ÀÓ5F×G©T¨Q°�a˜!‘e—Z‘Z §¡Õ(ÐGˆEÑGØ,1Ö2 q˜!Ÿ&™&�(Ð2ˆJÐ2Ü×#Ñ# J°Ô5ä×Ñ  zÔ2Ü×Ñ ¨%Ô0ä×Ñ ¨°%Õ8àFQÖR¸d˜A ¬°DÓ)9Ñ 9Ó9ÐRÐÐRØFQÖR¸d˜A ¬°DÓ)9Ñ 9Ó9ÐRÐÐRà.>Ö?¨�b˜2‘g “^Ð?ˆ	Ð?à5EÖ F¨r  s£Ð FÐÐ Fô ×%Ñ% h°Ó6ˆÜ×Ñ˜H e°1°u±9Õ=ä×-Ñ-¨kÓ:ˆÜ×Ñ˜OÐ-BÔCÜ×"Ñ" ?°CÓ8ˆáä×&Ñ& x°Ó7ˆEÜ58¸ÀÓ5F×G©T¨Q°�a˜!‘e—Z‘Z §¡Õ(ÐGˆEÑGØ,1Ö2 q˜!Ÿ&™&�(Ð2ˆJÐ2Ü×#Ñ# J°Ô5ä×Ñ  zÔ2Ü×Ñ ¨%Ô0ô
 	×Ñ˜E 9Ô-Ü×Ñ ¨°%Õ8ùòy QùÚVùÚ\ùÚRùò$ LùÚKùó> HùÚ2ùò SùÚRùâ?ùâ Fùó HùÚ2sT   Á3W$Á;3W)Â43W.Ã-3W3Ç	W8Ç,W=Î(XÎ=XÐ4XÑXÑ0XÒXÔ,(X!ÕX')NF)rP   Útypingr   r   r   r:   r   Ú_helpersr   r	   r
   Ú_typesr   ÚoptimÚ	Optimizerr   rR   rQ   rC   rg   rf   © r"   r!   ú<module>rª      si  ðñ	÷ )Ñ (ã Ý ç IÑ IÝ ôKˆU�[‰[×"Ñ"ô Kðh #'Ø ñ9Ø�V‘ð9à�F‰|ð9ð �v‘,ð9ð ˜&‘\ð	9ð
 ˜&‘\ð9ð ˜$‘ð9ð ð9ð ð9ð ð9ð ð9ð ð9ð ð9ð ð9ð ð9ð  ˜‘ð!9ð" 
ó#9ðxR=Ø�V‘ðR=à�F‰|ðR=ð �v‘,ðR=ð ˜&‘\ð	R=ð
 ˜&‘\ðR=ð ðR=ð ðR=ð ðR=ð ðR=ð ðR=ð ðR=ð ðR=ð ðR=ð ˜‘óR=ðjv9Ø�V‘ðv9à�F‰|ðv9ð �v‘,ðv9ð ˜&‘\ð	v9ð
 ˜&‘\ðv9ð ðv9ð ðv9ð ðv9ð ðv9ð ðv9ð ðv9ð ðv9ð ðv9ð ˜‘ôv9r"   