Ë
    éÓ^j$D  ã            %       óÜ  — d Z ddlmZmZmZ ddlZddlmZ ddlmZ ddl	m
Z
mZmZ ddlmZ  G d	„ d
e«      Z	 	 d dee   dee   dee   dee   dee   dee   dee   dedededededededededee   ddf$d„Zdee   dee   dee   dee   dee   dee   dedededededededededee   f d„Zdee   dee   dee   dee   dee   dee   dedededededededededee   f d„Zy)!aM   AdamW Optimizer
Impl copied from PyTorch master

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Why Gradients Rapidly Increase Near the End of Training: https://arxiv.org/abs/2506.02285

NOTE: This impl has been deprecated in favour of torch.optim.AdamW and remains as a reference
é    )ÚListÚOptionalÚTupleN)ÚTensor)Ú	Optimizeré   )Ú_check_capturable_devicesÚ
_get_valueÚ_init_scalar)ÚParamsTc                   ó°   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 ddededeeef   dedededed	ed
edee   defˆ fd„Z	ˆ fd„Z
 ej                  «       dd„«       Zˆ xZS )ÚAdamWLegacya  Implements AdamW algorithm.

    NOTE: This impl has been deprecated in favour of torch.optim.AdamW and remains as a reference

    References:
        - Adam: A Method for Stochastic Optimization: https://arxiv.org/abs/1412.6980
        - Decoupled Weight Decay Regularization: https://arxiv.org/abs/1711.05101
        - On the Convergence of Adam and Beyond: https://openreview.net/forum?id=ryQu7f-RZ

    Args:
        params: iterable of parameters to optimize or dicts defining parameter groups
        lr: learning rate
        betas: coefficients used for computing running averages of gradient and its square
        eps: term added to the denominator to improve numerical stability
        weight_decay: weight decay coefficient
        amsgrad: whether to use the AMSGrad variant of this algorithm
            from the paper `On the Convergence of Adam and Beyond`
        caution: apply caution when using AdamW
        corrected_weight_decay: apply corrected weight decay (lr**2 / max_lr)
        maximize: maximize the params based on the objective, instead of minimizing
        foreach: whether foreach implementation of optimizer is used.
            If unspecified by the user (so foreach is None), we will try to use
            foreach over for-loop implementation on CUDA, since it is faster in general.
        capturable: whether this instance is safe to capture in a CUDA graph.
            Passing True can impair ungraphed performance, so if you don't intend to
            graph capture this instance, leave it False
    ÚparamsÚlrÚbetasÚepsÚweight_decayÚamsgradÚcautionÚcorrected_weight_decayÚmaximizeÚforeachÚ
capturablec                 óˆ  •— d|k  st        dj                  |«      «      ‚d|k  st        dj                  |«      «      ‚d|d   cxk  rdk  sn t        dj                  |d   «      «      ‚d|d   cxk  rdk  sn t        dj                  |d   «      «      ‚t        ||||||||
|	|¬	«
      }t        t        | �  ||«       y )
Ng        zInvalid learning rate: {}zInvalid epsilon value: {}r   ç      ð?z%Invalid beta parameter at index 0: {}r   z%Invalid beta parameter at index 1: {})
r   r   r   r   r   r   r   r   r   r   )Ú
ValueErrorÚformatÚdictÚsuperr   Ú__init__)Úselfr   r   r   r   r   r   r   r   r   r   r   ÚdefaultsÚ	__class__s                €ú[/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/optim/adamw.pyr    zAdamWLegacy.__init__1   sÖ   ø€ ð �bŠyÜÐ8×?Ñ?ÀÓCÓDÐDØ�cŠzÜÐ8×?Ñ?ÀÓDÓEÐEØ�e˜A‘hÔ$ Ô$ÜÐD×KÑKÈEÐRSÉHÓUÓVÐVØ�e˜A‘hÔ$ Ô$ÜÐD×KÑKÈEÐRSÉHÓUÓVÐVÜØØØØ%ØØØ#9ØØØ!ô
ˆô 	Œk˜4Ñ)¨&°(Õ;ó    c                 óÊ  •— t         t        | �  |«       | j                  D ]À  }|j	                  dd«       |j	                  dd«       |j	                  dd«       |j	                  dd «       |j	                  dd«       |j	                  dd«       |d   D ]J  }| j
                  j                  |i «      }|sŒ"d	|v sŒ't        |d	   |d   r|j                  nd
¬«      |d	<   ŒL ŒÂ y )Nr   Fr   r   r   r   r   r   ÚstepÚcpu©Údevice)	r   r   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetr   r*   )r!   r.   ÚgroupÚpÚp_stater#   s        €r$   r+   zAdamWLegacy.__setstate__U   sÞ   ø€ ÜŒk˜4Ñ-¨eÔ4Ø×&Ñ&ò 	ˆEØ×Ñ˜Y¨Ô.Ø×Ñ˜Y¨Ô.Ø×ÑÐ5°uÔ=Ø×Ñ˜Y¨Ô-Ø×Ñ˜Z¨Ô/Ø×Ñ˜\¨5Ô1Ø˜8‘_ò �ØŸ*™*Ÿ.™.¨¨BÓ/�Ú˜v¨Ò0Ü&2Ø ™Ø+0°Ò+>˜qŸxšxÀEô'�G˜F’Oññ	r%   c                 ó¼  — t        | d«      r| j                  «        nt        | d«      r| j                  «        d}|�$t        j                  «       5   |«       }ddd«       | j
                  D �]Ý  }g }g }g }g }g }g }	|d   \  }
}|d   }|d   D �]s  }|j                  €Œ|j                  |«       |j                  j                  rt        d«      ‚|j                  |j                  «       | j                  |   }t        |«      dk(  ršt        |d	   r|j                  nd
¬«      |d<   t        j                  |t        j                  ¬«      |d<   t        j                  |t        j                  ¬«      |d<   |r(t        j                  |t        j                  ¬«      |d<   |j                  |d   «       |j                  |d   «       |r!|j                  |j!                  dd«      «       |	j                  |d   «       �Œv t#        ||||||	|d   ||
||d   |d   |d   |d   |d   |d	   |d   r| j$                  d   nd¬«       �Œà |S # 1 sw Y   �ŒùxY w)z¶Performs a single optimization step.

        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Ú'_accelerator_graph_capture_health_checkÚ _cuda_graph_capture_health_checkNr   r   r   z'AdamW does not support sparse gradientsr   r   r(   r)   r'   )Úmemory_formatÚexp_avgÚ
exp_avg_sqÚmax_exp_avg_sqr   r   r   r   r   r   r   )r   r   Úbeta1Úbeta2r   r   r   r   r   r   Úmax_lr)Úhasattrr4   r5   ÚtorchÚenable_gradr,   ÚgradÚappendÚ	is_sparseÚRuntimeErrorr.   Úlenr   r*   Ú
zeros_likeÚpreserve_formatr/   Úadamwr"   )r!   ÚclosureÚlossr0   Úparams_with_gradÚgradsÚexp_avgsÚexp_avg_sqsÚmax_exp_avg_sqsÚstate_stepsr:   r;   r   r1   r.   s                  r$   r'   zAdamWLegacy.stepf   si  € ô �4ÐBÔCØ×8Ñ8Õ:Ü�TÐ=Ô>Ø×1Ñ1Ô3àˆØÐÜ×"Ñ"Ó$ñ !Ù“y�÷!ð ×&Ñ&ó 7	ˆEØ!ÐØˆEØˆHØˆKØ ˆOØˆKØ  ™>‰LˆE�5Ø˜IÑ&ˆGà˜8‘_ó 2�Ø—6‘6�>ØØ ×'Ñ'¨Ô*Ø—6‘6×#Ò#Ü&Ð'PÓQÐQØ—‘˜QŸV™VÔ$àŸ
™
 1™�ô �u“: ’?Ü$0ÀEÈ,ÒDW¸¿ºÐ]bÔ$c�E˜&‘Mä',×'7Ñ'7¸Ì×I^ÑI^Ô'_�E˜)Ñ$ä*/×*:Ñ*:¸1ÌE×LaÑLaÔ*b�E˜,Ñ'Ùä27×2BÑ2BÀ1ÔTY×TiÑTiÔ2j˜Ð.Ñ/à—‘  iÑ 0Ô1Ø×"Ñ" 5¨Ñ#6Ô7ÙØ#×*Ñ*¨5¯9©9Ð5EÀtÓ+LÔMØ×"Ñ" 5¨¡=Ö1ð32ô6 Ø ØØØØØØ˜iÑ(ØØØØ˜‘;Ø" >Ñ2Ø˜%‘LØ˜iÑ(Ø˜zÑ*Ø  Ñ.Ø.3Ð4LÒ.M�t—}‘} TÒ*ÐSW÷#ðK7	ðr ˆ÷y!ñ !ús   ÁIÉI)
çü©ñÒMbP?)gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?g:Œ0âŽyE>g{®Gáz„?FFFFNF©N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úfloatr   Úboolr   r    r+   r>   Úno_gradr'   Ú__classcell__)r#   s   @r$   r   r      sÊ   ø„ ñð> Ø)5ØØ"&Ø!Ø!Ø+0Ø"Ø&*Ø$ñ"<àð"<ð ð"<ð ˜ ˜Ñ&ð	"<ð
 ð"<ð  ð"<ð ð"<ð ð"<ð %)ð"<ð ð"<ð ˜d‘^ð"<ð õ"<ôHð" €U‡]�]ƒ_òJó ôJr%   r   r   rK   rL   rM   rN   rO   r   r   r   r:   r;   r   r   r   r   r   r<   Úreturnc       	         ó˜  — t        d„ |D «       «      st        d«      ‚|€U	 | xs4 dt        j                  j                  j
                  j                  «       v }|rt        j                  |«      r|sd}|r%t        j                  j                  «       st        }nt        } || |||||||	|
|||||||¬«       y# t        $ r d}Y ŒSw xY w)zgFunctional API that performs AdamW algorithm computation.
      See AdamWLegacy class for details.
    c              3   óP   K  — | ]  }t        |t        j                  «      –— Œ  y ­wrQ   )Ú
isinstancer>   r   )Ú.0Úts     r$   ú	<genexpr>zadamw.<locals>.<genexpr>Ì   s   è ø€ Ò@¨qŒz˜!œUŸ\™\×*Ñ@ùs   ‚$&zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNÚScalarF)
r   r:   r;   r   r   r   r   r   r   r<   )ÚallrC   r>   ÚopsÚatenÚ_foreach_maximum_Ú	overloadsÚ	is_tensorÚ	ExceptionÚjitÚis_scriptingÚ_multi_tensor_adamwÚ_single_tensor_adamw)r   rK   rL   rM   rN   rO   r   r   r   r:   r;   r   r   r   r   r   r<   Úfuncs                     r$   rG   rG   ´   sÙ   € ô0 Ñ@°KÔ@Ô@Üð!ó"ð 	"ð €ð	à!�kÒ] X´·±·±×1QÑ1Q×1[Ñ1[Ó1]Ð%]ˆGáœ5Ÿ?™?¨2Ô.±zØ�ñ ”u—y‘y×-Ñ-Ô/Ü"‰ä#ˆáØØØØØØØØØØØ!ØØØØØö!øô ò 	ØŠGð	ús   ¡AB; Â;C	ÃC	c       
         óò  — |rt        | |«       t        | «      D �]Z  \  }}|s||   n||    }||   }||   }||   }|dz  }|€|	n|	dz  |z  }|j                  d||
z  z
  «       |j                  |«      j                  |d|z
  ¬«       |j                  |«      j	                  ||d|z
  ¬«       |r ||   }t        j                  |||¬«       |}n|}|rì|}dt        j                  ||«      z
  }dt        j                  ||«      z
  }|	|z  }|j                  «       }|j                  «       }|j                  «       ||z  z  j                  ||z  «      }|rU||z  dkD  j                  |j                  «      } | j                  | j                  «       j                  d¬	«      «       || z  }|j                  ||«       �Œ¨t!        |«      }d||z  z
  }d||z  z
  }|	|z  }|d
z  }|j                  «       |z  j                  |«      }|rU||z  dkD  j                  |j                  «      } | j                  | j                  «       j                  d¬	«      «       || z  }|j                  ||| ¬«       �Œ] y )Nr   é   r   ©Úalpha)Úvalue)Úoutr   rP   )Úminç      à?)r	   Ú	enumerateÚmul_Úadd_Úaddcmul_r>   ÚmaxÚpowÚnegÚsqrtÚtoÚdtypeÚdiv_ÚmeanÚclamp_Úaddcdiv_r
   )!r   rK   rL   rM   rN   rO   r   r:   r;   r   r   r   r   r   r   r<   ÚiÚparamr@   r7   r8   Ústep_tÚwd_scaler9   Ú
denom_baser'   Úbias_correction1Úbias_correction2Ú	step_sizeÚstep_size_negÚbias_correction2_sqrtÚdenomÚmasks!                                    r$   rl   rl   ô   s�  € ñ& Ü! &¨+Ô6ä˜fÓ%ó ?=‰ˆˆ5Ù'ˆu�QŠx¨e°A©h¨YˆØ˜1‘+ˆØ  ‘^ˆ
Ø˜Q‘ˆð 	�!‰ˆð  ˜‘2¨R°1©W°vÑ-=ˆØ�
‰
�2˜ <Ñ/Ñ/Ô0ð 	�‰�UÓ× Ñ  ¨Q°©YÐ Ô7Ø�‰˜Ó×'Ñ'¨¨d¸!¸e¹)Ð'ÔDáØ,¨QÑ/ˆNä�I‰I�n j°nÕEØ'‰Jà#ˆJáØˆDð  !¤5§9¡9¨U°DÓ#9Ñ9ÐØ ¤5§9¡9¨U°DÓ#9Ñ9ÐàÐ-Ñ-ˆIØ%ŸM™M›OˆMà$4×$9Ñ$9Ó$;Ð!à—_‘_Ó&Ð*?À-Ñ*OÑP×VÑVÐWZÐ]jÑWjÓkˆEáð   $™¨Ñ*×.Ñ.¨t¯z©zÓ:�Ø—	‘	˜$Ÿ)™)›+×,Ñ,°Ð,Ó6Ô7Ø! D™.�à�N‰N˜7 EÖ*ä˜fÓ%ˆDØ  5¨D¡=Ñ0ÐØ  5¨D¡=Ñ0ÐØÐ-Ñ-ˆIØ$4¸Ñ$;Ð!à—_‘_Ó&Ð)>Ñ>×DÑDÀSÓIˆEáà $™¨Ñ*×.Ñ.¨t¯z©zÓ:�Ø—	‘	˜$Ÿ)™)›+×,Ñ,°Ð,Ó6Ô7Ø! D™.�à�N‰N˜7 E°)°ˆNÖ<ñ?=r%   c       
         óD  — t        | «      dk(  ry |rt        | |«       |rt        j                  t	        |«      «      }|D �cg c].  }t        j
                  |«      rt        j                  |«      n|‘Œ0 }}|D �cg c].  }t        j
                  |«      rt        j                  |«      n|‘Œ0 }}|D �cg c].  }t        j
                  |«      rt        j                  |«      n|‘Œ0 }}| D �cg c].  }t        j
                  |«      rt        j                  |«      n|‘Œ0 } }t        j                  |d«       |€|	n|	dz  |z  }t        j                  | d||
z  z
  «       t        j                  ||«       t        j                  ||d|z
  ¬«       t        j                  ||«       t        j                  |||d|z
  «       |�r¡|D �cg c]  }t        j                  ||«      ‘Œ }}|D �cg c]  }t        j                  ||«      ‘Œ }}t        j                  |d«       t        j                  |d«       t        j                  |«       t        j                  |«       t        j                  ||	«      }t        j                  |«       t        j                  |«       t        j                  |«      }|re|D �cg c].  }t        j
                  |«      rt        j                  |«      n|‘Œ0 }}t        j                   ||«       t        j                  |«      }nt        j                  |«      }t        j"                  |t        j$                  ||«      «       t        j                  ||«      }t        j                  |«       t        j&                  ||«      }|r¯t        j$                  ||«      }t)        ||«      D ��cg c]#  \  }}|dkD  j+                  |j,                  «      ‘Œ% }}}|D �cg c]  }|j/                  «       ‘Œ }}t        j                   |d«       t        j"                  ||«       t        j$                  ||«      }t        j0                  | ||«       y |D �cg c]  }d|t3        |«      z  z
  ‘Œ }}|D �cg c]  }d|t3        |«      z  z
  ‘Œ }}|D �cg c]
  }|	|z  dz  ‘Œ }}|D �cg c]  }|dz  ‘Œ	 }}|re|D �cg c].  }t        j
                  |«      rt        j                  |«      n|‘Œ0 }}t        j                   ||«       t        j                  |«      }nt        j                  |«      }t        j"                  ||«       t        j                  ||«       |r¯t        j$                  ||«      }t)        ||«      D ��cg c]#  \  }}|dkD  j+                  |j,                  «      ‘Œ% }}}|D �cg c]  }|j/                  «       ‘Œ }}t        j                   |d«       t        j"                  ||«       t        j$                  ||«      }t        j"                  ||«       t        j0                  | ||«       y c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}}w c c}w c c}w c c}w c c}w c c}w c c}w c c}}w c c}w )Nr   r   ro   rp   rP   éÿÿÿÿru   )rD   r	   r>   Ú_foreach_negÚtupleÚ
is_complexÚview_as_realÚ_foreach_add_Ú_foreach_mul_Ú_foreach_addcmul_r{   Ú_foreach_sub_Ú_foreach_neg_Ú_foreach_divÚ_foreach_reciprocal_Ú_foreach_sqrtre   Ú_foreach_div_Ú_foreach_mulÚ_foreach_addÚzipr~   r   r�   Ú_foreach_addcdiv_r
   )r   rK   rL   rM   rN   rO   r   r:   r;   r   r   r   r   r   r   r<   Úxr‡   r'   r‰   rŠ   r‹   r�   rˆ   Úeps_over_step_sizerŽ   ÚmasksÚmÚgÚ
mask_scaleÚbcs                                  r$   rk   rk   L  s¤  € ô& ˆ6ƒ{�aÒØáÜ! &¨+Ô6áÜ×"Ñ"¤5¨£<Ó0ˆàJOÖPÀQ¤e×&6Ñ&6°qÔ&9ŒU×Ñ Ô"¸qÑ@ÐP€EÐPØMUÖVÈ¬×)9Ñ)9¸!Ô)<”×"Ñ" 1Ô%À!ÑCÐV€HÐVØP[Ö\È1¬E×,<Ñ,<¸QÔ,?”5×%Ñ% aÔ(ÀQÑFÐ\€KÐ\ØKQÖRÀa¤u×'7Ñ'7¸Ô':Œe× Ñ  Ô#ÀÑAÐR€FÐRô 
×Ñ˜ QÔ'ð �^‰r¨¨q©°6Ñ)9€HÜ	×Ñ˜  X°Ñ%<Ñ <Ô=ô 
×Ñ˜ %Ô(Ü	×Ñ˜ %¨q°5©yÕ9ä	×Ñ˜ UÔ+Ü	×Ñ˜K¨°°q¸5±yÔAâà?JÖK°tœEŸI™I e¨TÕ2ÐKÐÐKØ?JÖK°tœEŸI™I e¨TÕ2ÐKÐÐKä×ÑÐ,¨aÔ0Ü×ÑÐ,¨aÔ0Ü×ÑÐ,Ô-Ü×ÑÐ,Ô-ô ×&Ñ&Ð'7¸Ó<ˆ	Ü×"Ñ" 9Ô-Ü×Ñ˜IÔ&ä %× 3Ñ 3Ð4DÓ EÐáà\kÖlÐWX¼×8HÑ8HÈÔ8Kœu×1Ñ1°!Ô4ÐQRÑRÐlˆOÐlÜ×#Ñ# O°[ÔAÜ×,Ñ,¨_Ó=‰Jä×,Ñ,¨[Ó9ˆJä×ÑØÜ×ÑÐ4°iÓ@ô	
ô #×/Ñ/°	¸3Ó?ÐÜ×"Ñ"Ð#5Ô6Ü×"Ñ" :Ð/AÓBˆáä×&Ñ& x°Ó7ˆEÜ58¸ÀÓ5F×G©T¨Q°�a˜!‘e—Z‘Z §¡Õ(ÐGˆEÑGØ,1Ö2 q˜!Ÿ&™&�(Ð2ˆJÐ2Ü×#Ñ# J°Ô5ä×Ñ  zÔ2Ü×)Ñ)¨(°EÓ:ˆHä×Ñ ¨°%Õ8àFQÖR¸d˜A ¬°DÓ)9Ñ 9Ó9ÐRÐÐRØFQÖR¸d˜A ¬°DÓ)9Ñ 9Ó9ÐRÐÐRà.>Ö?¨�b˜2‘g “^Ð?ˆ	Ð?à5EÖ F¨r  s£Ð FÐÐ Fáà\kÖlÐWX¼×8HÑ8HÈÔ8Kœu×1Ñ1°!Ô4ÐQRÑRÐlˆOÐlÜ×#Ñ# O°[ÔAÜ×'Ñ'¨Ó8‰Eä×'Ñ'¨Ó4ˆEä×Ñ˜EÐ#8Ô9Ü×Ñ˜E 3Ô'áä×&Ñ& x°Ó7ˆEÜ58¸ÀÓ5F×G©T¨Q°�a˜!‘e—Z‘Z §¡Õ(ÐGˆEÑGØ,1Ö2 q˜!Ÿ&™&�(Ð2ˆJÐ2Ü×#Ñ# J°Ô5ä×Ñ  zÔ2Ü×)Ñ)¨(°EÓ:ˆHô
 	×Ñ˜E 9Ô-Ü×Ñ ¨°%Õ8ùòC QùÚVùÚ\ùÚRùò& LùÚKùò  mùó" HùÚ2ùò SùÚRùâ?ùâ Fùò mùó HùÚ2s`   Á3YÁ;3YÂ43YÃ-3YÇ	Y$Ç,Y)Ê<3Y.Ï(Y3Ï3Y9Ñ*Y>ÒZÒ&ZÒ;ZÓ3ZÖ(Z×Z)NF)rU   Útypingr   r   r   r>   r   Útorch.optim.optimizerr   Ú_helpersr	   r
   r   Ú_typesr   r   rW   rV   rG   rl   rk   © r%   r$   ú<module>r¯      s®  ðñ÷ )Ñ (ã Ý Ý +ç IÑ IÝ ô]�)ô ]ðN #'Ø ñ=Ø�V‘ð=à�F‰|ð=ð �v‘,ð=ð ˜&‘\ð	=ð
 ˜f™ð=ð ˜&‘\ð=ð ˜$‘ð=ð ð=ð ð=ð ð=ð ð=ð ð=ð ð=ð ð=ð  ð!=ð" ð#=ð$ ˜‘ð%=ð& 
ó'=ð@U=Ø�V‘ðU=à�F‰|ðU=ð �v‘,ðU=ð ˜&‘\ð	U=ð
 ˜f™ðU=ð ˜&‘\ðU=ð ðU=ð ðU=ð ðU=ð ðU=ð ðU=ð ðU=ð ðU=ð ðU=ð  ð!U=ð" ˜‘ó#U=ðp}9Ø�V‘ð}9à�F‰|ð}9ð �v‘,ð}9ð ˜&‘\ð	}9ð
 ˜f™ð}9ð ˜&‘\ð}9ð ð}9ð ð}9ð ð}9ð ð}9ð ð}9ð ð}9ð ð}9ð ð}9ð  ð!}9ð" ˜‘ô#}9r%   