Ë
    éÓ^j.  ã                   ó2   — d Z ddlZddlmZ  G d„ de«      Zy)an   PyTorch LARS / LARC Optimizer

An implementation of LARS (SGD) + LARC in PyTorch

Based on:
  * PyTorch SGD: https://github.com/pytorch/pytorch/blob/1.7/torch/optim/sgd.py#L100
  * NVIDIA APEX LARC: https://github.com/NVIDIA/apex/blob/master/apex/parallel/LARC.py

Additional cleanup and modifications to properly support PyTorch XLA.

Copyright 2021 Ross Wightman
é    N)Ú	Optimizerc                   óp   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 dˆ fd„	Zˆ fd„Z ej                  «       dd„«       Zˆ xZ	S )ÚLarsa»   LARS for PyTorch
    
    Paper: `Large batch training of Convolutional Networks` - https://arxiv.org/pdf/1708.03888.pdf

    Args:
        params (iterable): iterable of parameters to optimize or dicts defining parameter groups.
        lr (float, optional): learning rate (default: 1.0).
        momentum (float, optional): momentum factor (default: 0)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        dampening (float, optional): dampening for momentum (default: 0)
        nesterov (bool, optional): enables Nesterov momentum (default: False)
        trust_coeff (float): trust coefficient for computing adaptive lr / trust_ratio (default: 0.001)
        eps (float): eps for division denominator (default: 1e-8)
        trust_clip (bool): enable LARC trust ratio clipping (default: False)
        always_adapt (bool): always apply LARS LR adapt, otherwise only when group weight_decay != 0 (default: False)
    c                 óî   •— |dk  rt        d|› �«      ‚|dk  rt        d|› �«      ‚|dk  rt        d|› �«      ‚|r|dk  s|dk7  rt        d«      ‚t        ||||||||	|
¬«	      }t        ‰| �  ||«       y )Ng        zInvalid learning rate: zInvalid momentum value: zInvalid weight_decay value: r   z8Nesterov momentum requires a momentum and zero dampening)	ÚlrÚmomentumÚ	dampeningÚweight_decayÚnesterovÚtrust_coeffÚepsÚ
trust_clipÚalways_adapt)Ú
ValueErrorÚdictÚsuperÚ__init__)ÚselfÚparamsr   r   r	   r
   r   r   r   r   r   ÚdefaultsÚ	__class__s               €úZ/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/optim/lars.pyr   zLars.__init__#   s¥   ø€ ð �Š8ÜÐ6°r°dÐ;Ó<Ð<Ø�cŠ>ÜÐ7¸°zÐBÓCÐCØ˜#ÒÜÐ;¸L¸>ÐJÓKÐKÙ˜ Qš¨)°qª.ÜÐWÓXÐXäØØØØ%ØØ#ØØ!Ø%ô

ˆô 	‰Ñ˜ Õ*ó    c                 ój   •— t         ‰| �  |«       | j                  D ]  }|j                  dd«       Œ y )Nr   F)r   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroupr   s      €r   r   zLars.__setstate__F   s5   ø€ Ü‰Ñ˜UÔ#Ø×&Ñ&ò 	0ˆEØ×Ñ˜Z¨Õ/ñ	0r   c           
      óº  — d}|�$t        j                  «       5   |«       }ddd«       | j                  D �]•  }|d   }|d   }|d   }|d   }|d   }|d   }	|d   D �]k  }
|
j                  €Œ|
j                  }|d	k7  s|d
   r«|
j	                  d«      }|j	                  d«      }||z  |||z  z   |	z   z  }t        j
                  |d	kD  t        j
                  |d	kD  |d«      d«      }|d   rt        j                  ||d   z  d¬«      }|j                  |
|¬«       |j                  |«       |d	k7  r~| j                  |
   }d|vr)t        j                  |«      j                  «       x}|d<   n*|d   }|j                  |«      j                  |d|z
  ¬«       |r|j                  ||¬«      }n|}|
j                  ||d    ¬«       �Œn �Œ˜ |S # 1 sw Y   �Œ±xY w)z¡Performs a single optimization step.

        Args:
            closure (callable, optional): A closure that reevaluates the model and returns the loss.
        Nr
   r   r	   r   r   r   r   r   r   g       @ç      ð?r   r   )Úmax)ÚalphaÚmomentum_buffer)ÚtorchÚenable_gradr   ÚgradÚnormÚwhereÚclampÚadd_Úmul_r   ÚcloneÚdetachÚadd)r   ÚclosureÚlossr   r
   r   r	   r   r   r   Úpr'   Úw_normÚg_normÚtrust_ratioÚparam_stateÚbufs                    r   Ústepz	Lars.stepK   s  € ð ˆØÐÜ×"Ñ"Ó$ñ !Ù“y�÷!ð ×&Ñ&ó ,	1ˆEØ  Ñ0ˆLØ˜ZÑ(ˆHØ˜kÑ*ˆIØ˜ZÑ(ˆHØ Ñ.ˆKØ˜‘,ˆCà˜8‘_ó $1�Ø—6‘6�>ØØ—v‘v�ð   1Ò$¨¨nÒ(=ØŸV™V C›[�FØ!ŸY™Y s›^�FØ"-°Ñ"6¸&À6ÈLÑCXÑ:XÐ[^Ñ:^Ñ"_�Kô #(§+¡+Ø ™
ÜŸ™ F¨Q¡J°¸SÓAØó#�Kð
 ˜\Ò*Ü&+§k¡k°+ÀÀdÁÑ2KÐQTÔ&U˜Ø—I‘I˜a |�IÔ4Ø—I‘I˜kÔ*ð ˜q’=Ø"&§*¡*¨Q¡-�KØ(°Ñ;Ü?D¿{¹{È4Ó?P×?WÑ?WÓ?YÐY˜˜kÐ*;Ò<à)Ð*;Ñ<˜ØŸ™ Ó*×/Ñ/°¸BÀ¹NÐ/ÔKÙØ#Ÿx™x¨°8˜xÓ<™à"˜à—‘�t E¨$¡K <�Ö0òI$1ð,	1ð\ ˆ÷c!ñ !ús   ™GÇG)	r!   r   r   r   Fgü©ñÒMbP?g:Œ0âŽyE>FF)N)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r%   Úno_gradr8   Ú__classcell__)r   s   @r   r   r      sL   ø„ ñð( ØØØØØØØØõ!+ôF0ð
 €U‡]�]ƒ_ò9ó ô9r   r   )r<   r%   Útorch.optim.optimizerr   r   © r   r   ú<module>rA      s    ðñó Ý +ôtˆ9õ tr   