Ë
    éÓ^j+  ã                   óf   — d Z ddlmZmZ ddlZddlmZ ddlmZm	Z	m
Z
mZ ddlmZ  G d„ d	e«      Zy)
a   PyTorch Lamb optimizer w/ behaviour similar to NVIDIA FusedLamb

This optimizer code was adapted from the following (starting with latest)
* https://github.com/HabanaAI/Model-References/blob/2b435114fe8e31f159b1d3063b8280ae37af7423/PyTorch/nlp/bert/pretraining/lamb.py
* https://github.com/NVIDIA/DeepLearningExamples/blob/master/PyTorch/LanguageModeling/Transformer-XL/pytorch/lamb.py
* https://github.com/cybertronai/pytorch-lamb

Use FusedLamb if you can (GPU). The reason for including this variant of Lamb is to have a version that is
similar in behaviour to APEX FusedLamb if you aren't using NVIDIA GPUs or cannot install/use APEX.

In addition to some cleanup, this Lamb impl has been modified to support PyTorch XLA and has been tested on TPU.

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Why Gradients Rapidly Increase Near the End of Training: https://arxiv.org/abs/2506.02285

Original copyrights for above sources are below.

Modifications Copyright 2021 Ross Wightman
é    )ÚOptionalÚTupleN)Ú	Optimizeré   )Ú_add_scaled_Ú
_get_valueÚ_init_scalarÚ_validate_scalar)ÚParamsTc                   óÂ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 ddedededeeef   dededed	ee   d
ededededefˆ fd„Z	ˆ fd„Z
d„ Z ej                  «       dd„«       Zˆ xZS )ÚLambaû  Implements a pure pytorch variant of FuseLAMB (NvLamb variant) optimizer from apex.optimizers.FusedLAMB
    reference: https://github.com/NVIDIA/DeepLearningExamples/blob/master/PyTorch/LanguageModeling/Transformer-XL/pytorch/lamb.py

    LAMB was proposed in:
    - Large Batch Optimization for Deep Learning - Training BERT in 76 minutes:  https://arxiv.org/abs/1904.00962
    - On the Convergence of Adam and Beyond: https://openreview.net/forum?id=ryQu7f-RZ

    Args:
        params: Iterable of parameters to optimize or dicts defining parameter groups.
        lr: Learning rate
        betas: Coefficients used for computing running averages of gradient and its norm.
        eps: Term added to the denominator to improve numerical stability.
        weight_decay: Weight decay
        grad_averaging: Whether apply (1-beta2) to grad when calculating running averages of gradient.
        max_grad_norm: Value used to clip global grad norm.
        trust_clip: Enable LAMBC trust ratio clipping.
        always_adapt: Apply adaptive learning rate to 0.0 weight decay parameter.
        caution: Apply caution.
        decoupled: apply decoupled weight decay
        corrected_weight_decay: apply corrected weight decay (lr**2 / max_lr) when using decoupled_decay
    ÚparamsÚlrÚbias_correctionÚbetasÚepsÚweight_decayÚgrad_averagingÚmax_grad_normÚ
trust_clipÚalways_adaptÚcautionÚdecoupled_decayÚcorrected_weight_decayc                 óœ   •— t        d|«       t        d|«       t        d|«       t        ||||||||	|
|||¬«      }t        ‰| �  ||«       y )Nzlearning rateÚepsilonr   )r   r   r   r   r   r   r   r   r   r   r   r   )r
   ÚdictÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   r   r   r   r   r   r   ÚdefaultsÚ	__class__s                  €úZ/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/optim/lamb.pyr   zLamb.__init__Z   sb   ø€ ô  	˜¨"Ô-Ü˜ CÔ(Ü˜¨Ô6ÜØØ+ØØØ%Ø)Ø'Ø!Ø%ØØ+Ø#9ô
ˆô 	‰Ñ˜ Õ*ó    c                 óâ   •— t         ‰| �  |«       | j                  D ]P  }|j                  dd«       |j                  dd«       |j                  dd«       d|v sŒ>t	        |d   d¬«      |d<   ŒR y )Nr   Fr   r   ÚstepÚcpu©Údevice)r   Ú__setstate__Úparam_groupsÚ
setdefaultr	   )r    ÚstateÚgroupr"   s      €r#   r*   zLamb.__setstate__}   st   ø€ Ü‰Ñ˜UÔ#Ø×&Ñ&ò 	JˆEØ×Ñ˜Y¨Ô.Ø×ÑÐ.°Ô6Ø×ÑÐ5°uÔ=Ø˜ŠÜ ,¨U°6©]À5Ô I��f’ñ	Jr$   c                 ó®  — | j                   d   }|€y g }| j                  D ]j  }|d   D ]`  }|j                  €Œ|j                  }|j                  rt	        d«      ‚|j                  t        j                  j                  |«      «       Œb Œl t        j                  j                  t        j                  |«      «      }||z  j                  d¬«      }|S )Nr   r   zDLamb does not support sparse gradients, consider SparseAdam instead.ç      ð?©Úmin)r!   r+   ÚgradÚ	is_sparseÚRuntimeErrorÚappendÚtorchÚlinalgÚvector_normÚstackÚclamp_)r    r   Únormsr.   Úpr3   Úglobal_normÚclip_global_norms           r#   Ú_get_clip_grad_normzLamb._get_clip_grad_norm†   sÇ   € ØŸ™ oÑ6ˆØÐ ØàˆØ×&Ñ&ò 	=ˆEØ˜8‘_ò =�Ø—6‘6�>ØØ—v‘v�Ø—>’>Ü&Ð'mÓnÐnØ—‘œUŸ\™\×5Ñ5°dÓ;Õ<ñ=ð	=ô —l‘l×.Ñ.¬u¯{©{¸5Ó/AÓBˆØ'¨-Ñ7×?Ñ?ÀCÐ?ÓHÐØÐr$   c           
      ó¾  — d}|�$t        j                  «       5   |«       }ddd«       | j                  «       }| j                  D �]  }|d   rdnd}|d   \  }}|d   rdnd}|rd|z
  nd}	d|vrt	        d	¬
«      |d<   |d   j                  d«       t        d„ |d   D «       «      sŒdt        |d   «      }
|rd||
z  z
  }d||
z  z
  }nd\  }}|d   D �]u  }|j                  €Œ|j                  }|�|j                  |«       | j                  |   }t        |«      dk(  r0t        j                  |«      |d<   t        j                  |«      |d<   |d   |d   }}|j                  |«      j                  ||	¬«       |j                  |«      j                  ||d|z
  ¬«       |j                  «       |dz  z  j                  |d   «      }||z  j                  |«      }|d   ra||z  dkD  j!                  |j"                  «      }|j                  |j%                  «       j'                  d¬«      «       |j                  |«       |d   }|dk7  rZ|j)                  dd«      r5|d   r|d   dz  | j*                  d   z  }n|d   }t-        ||| |z  «       n|j                  ||¬«       |dk7  s|d   r†|j/                  d«      }|j/                  d«      }||z  }t        j0                  |dkD  t        j0                  |dkD  |d«      d«      }|d   rt        j2                  |d¬ «      }|j                  |«       t-        |||d    «       �Œx �Œ
 |S # 1 sw Y   �Œ3xY w)!zµPerforms a single optimization step.
        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r0   r&   r'   r(   c              3   ó8   K  — | ]  }|j                   d u–— Œ y ­w©N)r3   )Ú.0r=   s     r#   ú	<genexpr>zLamb.step.<locals>.<genexpr>²   s   è ø€ ÒC¨a�q—v‘v TÔ)ÑCùs   ‚r   )r0   r0   Úexp_avgÚ
exp_avg_sq)Úalpha)Úvalueg      à?r   r   çü©ñÒMbP?r1   r   r   Fr   r   é   r   g       @r   )Úmax)r7   Úenable_gradr@   r+   r	   Úadd_Úanyr   r3   Údiv_r-   ÚlenÚ
zeros_likeÚmul_Úaddcmul_ÚsqrtÚtoÚdtypeÚmeanr;   Úgetr!   r   ÚnormÚwhereÚclamp)r    ÚclosureÚlossÚclip_grad_normr.   r   Úbeta1Úbeta2r   Úbeta3r&   Úbias_correction1Úbias_correction2r=   r3   r-   rF   rG   ÚdenomÚupdateÚmaskr   Úwd_scaleÚw_normÚg_normÚtrust_ratios                             r#   r&   z	Lamb.step˜   s†  € ð ˆØÐÜ×"Ñ"Ó$ñ !Ù“y�÷!ð ×1Ñ1Ó3ˆà×&Ñ&ó T	6ˆEØ#(Ð):Ò#;™aÀˆOØ  ™>‰LˆE�5Ø"'Ð(8Ò"9™Q¸qˆNÙ!/�A˜’I°SˆEð ˜UÑ"Ü ,°EÔ :��f‘Ø�&‰M×Ñ˜qÔ!äÑC°5¸±?ÔCÔCØä˜e F™mÓ,ˆDáØ#$ u°¡}Ñ#4Ð Ø#$ u°¡}Ñ#4Ñ à5=Ñ2Ð Ð"2à˜8‘_ó =6�Ø—6‘6�>ØØ—v‘v�à!Ð-Ø—I‘I˜nÔ-àŸ
™
 1™�ô �u“: ’?ä',×'7Ñ'7¸Ó':�E˜)Ñ$ä*/×*:Ñ*:¸1Ó*=�E˜,Ñ'à&+¨IÑ&6¸¸lÑ8K˜�ð —‘˜UÓ#×(Ñ(¨°UÐ(Ô;Ø—‘ Ó&×/Ñ/°°dÀ!ÀeÁ)Ð/ÔLà#Ÿ™Ó*Ð.>À#Ñ.EÑF×LÑLÈUÐSXÉ\ÓZ�Ø!Ð$4Ñ4×:Ñ:¸5ÓA�à˜Ò#à" T™M¨AÑ-×1Ñ1°$·*±*Ó=�DØ—I‘I˜dŸi™i›k×0Ñ0°TÐ0Ó:Ô;Ø—K‘K Ô%à$ ^Ñ4�Ø 1Ò$Ø—y‘yÐ!2°EÔ:Ø Ð!9Ò:Ø',¨T¡{°aÑ'7¸$¿-¹-ÈÑ:MÑ'M™Hà',¨T¡{˜HÜ$ Q¨¨H¨9°|Ñ+CÕDàŸ™ A¨\˜Ô:à 1Ò$¨¨nÒ(=ð ŸV™V C›[�FØ#Ÿ[™[¨Ó-�FØ"(¨6¡/�Kô #(§+¡+Ø ™
ÜŸ™ F¨Q¡J°¸SÓAØó#�Kð
 ˜\Ò*ä&+§k¡k°+À3Ô&G˜Ø—K‘K Ô,ä˜Q ¨¨t©¨Ö5ò{=6ð/T	6ðl ˆ÷w!ñ !ús   ™MÍM)rJ   T)gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?g�íµ ÷Æ°>g{®Gáz„?Tr0   FFFFFrC   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚfloatÚboolr   r   r   r*   r@   r7   Úno_gradr&   Ú__classcell__)r"   s   @r#   r   r   C   sê   ø„ ñð2 Ø$(Ø)5ØØ"&Ø#'Ø-0Ø$Ø!&Ø!Ø$)Ø+0ñ!+àð!+ð ð!+ð "ð	!+ð
 ˜ ˜Ñ&ð!+ð ð!+ð  ð!+ð !ð!+ð $ E™?ð!+ð ð!+ð ð!+ð ð!+ð "ð!+ð %)õ!+ôFJò ð$ €U‡]�]ƒ_òcó ôcr$   r   )ro   Útypingr   r   r7   Útorch.optimr   Ú_helpersr   r   r	   r
   Ú_typesr   r   © r$   r#   ú<module>ry      s-   ðñ÷r #ã Ý !ç NÓ NÝ ôyˆ9õ yr$   