Ë
    éÓ^j'#  ã                   óþ  — d Z ddlmZmZmZ ddlZddlmZ ddlm	Z	m
Z
 ddlmZ  G d„ d	e«      Z	 	 ddd
œdeej                     deej                     deej                     dedededededededee   fd„Zdeej                     deej                     deej                     dededededededee   fd„Zdeej                     deej                     deej                     dededededededee   fd„Zy)ah   Lion Optimizer
Paper: `Symbolic Discovery of Optimization Algorithms` - https://arxiv.org/abs/2302.06675
Original Impl: https://github.com/google/automl/tree/master/lion

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Why Gradients Rapidly Increase Near the End of Training: https://arxiv.org/abs/2506.02285
é    )ÚListÚOptionalÚTupleN)Ú	Optimizeré   )Ú_add_scaled_Ú_validate_scalar)ÚParamsTc                   óž   ‡ — e Zd ZdZ	 	 	 	 	 	 	 ddededeeef   dedededed	ee   fˆ fd
„Z	ˆ fd„Z
 ej                  «       dd„«       Zˆ xZS )ÚLionzImplements Lion algorithm.ÚparamsÚlrÚbetasÚweight_decayÚcautionÚcorrected_weight_decayÚmaximizeÚforeachc	           	      ó  •— t        d|«       d|d   cxk  rdk  sn t        dj                  |d   «      «      ‚d|d   cxk  rdk  sn t        dj                  |d   «      «      ‚t        |||||||¬«      }	t        ‰
| �  ||	«       y	)
a¶  Initialize the hyperparameters.

        Args:
            params: iterable of parameters to optimize or dicts defining parameter groups
            lr: learning rate
            betas: coefficients used for computing running averages of gradient and its square
            weight_decay: weight decay coefficient
            caution: apply caution
            corrected_weight_decay: apply corrected weight decay (lr**2 / max_lr)
        zlearning rateç        r   g      ð?z%Invalid beta parameter at index 0: {}r   z%Invalid beta parameter at index 1: {})r   r   r   r   r   r   r   N)r	   Ú
ValueErrorÚformatÚdictÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   r   ÚdefaultsÚ	__class__s             €úZ/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/optim/lion.pyr   zLion.__init__#   s™   ø€ ô, 	˜¨"Ô-Ø�e˜A‘hÔ$ Ô$ÜÐD×KÑKÈEÐRSÉHÓUÓVÐVØ�e˜A‘hÔ$ Ô$ÜÐD×KÑKÈEÐRSÉHÓUÓVÐVÜØØØ%ØØ#9ØØô
ˆô 	‰Ñ˜ Õ*ó    c                 óÖ   •— t         ‰| �  |«       | j                  D ]J  }|j                  dd«       |j                  dd«       |j                  dd«       |j                  dd «       ŒL y )Nr   Fr   r   r   )r   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroupr   s      €r   r"   zLion.__setstate__I   sf   ø€ Ü‰Ñ˜UÔ#Ø×&Ñ&ò 	.ˆEØ×Ñ˜Y¨Ô.Ø×ÑÐ5°uÔ=Ø×Ñ˜Z¨Ô/Ø×Ñ˜Y¨Õ-ñ		.r    c                 óž  — d}|�$t        j                  «       5   |«       }ddd«       | j                  D �]  }g }g }g }|d   \  }}|d   D ]µ  }	|	j                  €Œ|j	                  |	«       |	j                  j
                  rt        d«      ‚|j	                  |	j                  «       | j                  |	   }
t        |
«      dk(  r(t        j                  |	t         j                  ¬«      |
d<   |j	                  |
d   «       Œ· t        ||||||d   |d	   |d
   |d   |d   |d   r| j                  d   nd¬«       �Œ
 |S # 1 sw Y   �Œ#xY w)z´Performs a single optimization step.

        Args:
            closure: A closure that reevaluates the model and returns the loss.

        Returns:
            the loss.
        Nr   r   z&Lion does not support sparse gradientsr   )Úmemory_formatÚexp_avgr   r   r   r   r   r   )Úbeta1Úbeta2r   r   r   r   r   Úmax_lr)ÚtorchÚenable_gradr#   ÚgradÚappendÚ	is_sparseÚRuntimeErrorr%   ÚlenÚ
zeros_likeÚpreserve_formatÚlionr   )r   ÚclosureÚlossr&   Úparams_with_gradÚgradsÚexp_avgsr*   r+   Úpr%   s              r   Ústepz	Lion.stepQ   si  € ð ˆØÐÜ×"Ñ"Ó$ñ !Ù“y�÷!ð ×&Ñ&ó "	ˆEØ!ÐØˆEØˆHØ  ™>‰LˆE�5à˜8‘_ò 2�Ø—6‘6�>ØØ ×'Ñ'¨Ô*Ø—6‘6×#Ò#Ü&Ð'OÓPÐPØ—‘˜QŸV™VÔ$àŸ
™
 1™�ô �u“: ’?Ü',×'7Ñ'7¸Ì×I^ÑI^Ô'_�E˜)Ñ$à—‘  iÑ 0Õ1ð2ô  Ø ØØØØØ˜‘;Ø" >Ñ2Ø˜iÑ(Ø˜zÑ*Ø˜iÑ(Ø.3Ð4LÒ.M�t—}‘} TÒ*ÐSW÷ð-"	ðH ˆ÷O!ñ !ús   ™EÅE)g-Cëâ6?)gÍÌÌÌÌÌì?g®Gáz®ï?r   FFFN)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r
   Úfloatr   Úboolr   r   r"   r-   Úno_gradr=   Ú__classcell__)r   s   @r   r   r       sŸ   ø„ Ù%ð
 Ø)4Ø"%Ø!Ø+0Ø"Ø&*ñ$+àð$+ð ð$+ð ˜ ˜Ñ&ð	$+ð
  ð$+ð ð$+ð %)ð$+ð ð$+ð ˜d‘^õ$+ôL.ð €U‡]�]ƒ_ò2ó ô2r    r   )r,   r   r:   r;   r   r   r*   r+   r   r   r   r,   c                ó¤  — |€S	 |	 xs4 dt         j                  j                  j                  j	                  «       v }|rt        j
                  |«      rd}|r)t         j                  j                  «       rt        d«      ‚|r%t         j                  j                  «       st        }nt        } || |||||||	||
¬«
       y# t        $ r d}Y Œxw xY w)z=Functional API that performs Lion algorithm computation.
    NÚScalarFz6torch.jit.script not supported with foreach optimizers)r*   r+   r   r   r   r   r,   )r-   ÚopsÚatenÚ_foreach_maximum_Ú	overloadsÚ	is_tensorÚ	ExceptionÚjitÚis_scriptingr2   Ú_multi_tensor_lionÚ_single_tensor_lion)r   r:   r;   r   r   r*   r+   r   r   r   r,   Úfuncs               r   r6   r6   ‡   sÁ   € ð$ €ð	à!�kÒ] X´·±·±×1QÑ1Q×1[Ñ1[Ó1]Ð%]ˆGÙœ5Ÿ?™?¨2Ô.Ø�ñ ”5—9‘9×)Ñ)Ô+ÜÐSÓTÐTá”u—y‘y×-Ñ-Ô/Ü!‰ä"ˆáØØØØØØØ!ØØØöøô ò 	ØŠGð	ús   „AC ÃCÃCc                ó²  — t        | «      D �]H  \  }
}|s||
   n||
    }||
   }t        j                  |«      r?t        j                  |«      }t        j                  |«      }t        j                  |«      }|	€|n|dz  |	z  }|j	                  d||z  z
  «       |j                  |«      j                  |d|z
  ¬«      j                  «       }|ra||z  dkD  j                  |j                  «      }|j                  |j                  «       j                  d¬«      «       |j	                  |«       t        ||| «       |j                  |d|z
  «       �ŒK y )Né   r   ©Úalphar   çü©ñÒMbP?)Úmin)Ú	enumerater-   Ú
is_complexÚview_as_realÚmul_ÚmulÚadd_Úsign_ÚtoÚdtypeÚdiv_ÚmeanÚclamp_r   Úlerp_)r   r:   r;   r*   r+   r   r   r   r   r,   ÚiÚparamr/   r)   Úwd_scaleÚupdateÚmasks                    r   rQ   rQ   ¸   s=  € ô ˜fÓ%ó '‰ˆˆ5Ù'ˆu�QŠx¨e°A©h¨YˆØ˜1‘+ˆä×Ñ˜EÔ"Ü×%Ñ% dÓ+ˆDÜ×(Ñ(¨Ó1ˆGÜ×&Ñ& uÓ-ˆEð  ˜‘2¨R°1©W°vÑ-=ˆØ�
‰
�1�x ,Ñ.Ñ.Ô/ð —‘˜UÓ#×(Ñ(¨°Q¸±YÐ(Ó?×EÑEÓGˆáà˜T‘M AÑ%×)Ñ)¨$¯*©*Ó5ˆDØ�I‰I�d—i‘i“k×(Ñ(¨TÐ(Ó2Ô3Ø�K‰K˜Ôä�U˜F R CÔ(ð 	�‰�d˜A ™IÖ&ñ3'r    c                óØ  — t        | «      dk(  ry |rt        j                  t        |«      «      }|D �
cg c].  }
t        j                  |
«      rt        j
                  |
«      n|
‘Œ0 }}
|D �
cg c].  }
t        j                  |
«      rt        j
                  |
«      n|
‘Œ0 }}
| D �
cg c].  }
t        j                  |
«      rt        j
                  |
«      n|
‘Œ0 } }
|	€|n|dz  |	z  }t        j                  | d||z  z
  «       t        j                  ||«      }t        j                  ||d|z
  ¬«       |D �cg c]  }|j                  «       ‘Œ }}|r¯t        j                  ||«      }t        ||«      D ��cg c]#  \  }}|dkD  j                  |j                  «      ‘Œ% }}}|D �cg c]  }|j                  «       ‘Œ }}t        j                  |d«       t        j                  ||«       t        j                  ||«       t        j                  | || ¬«       t        j                  ||«       t        j                  ||d|z
  ¬«       y c c}
w c c}
w c c}
w c c}w c c}}w c c}w )Nr   rT   r   rU   rW   )r3   r-   Ú_foreach_negÚtuplerZ   r[   Ú_foreach_mul_Ú_foreach_mulÚ_foreach_add_r_   Úzipr`   ra   rc   rJ   Ú_foreach_div_)r   r:   r;   r*   r+   r   r   r   r   r,   Úxrh   ÚupdatesÚuÚmasksÚmÚgÚ
mask_scales                     r   rP   rP   á   s  € ô ˆ6ƒ{�aÒØáÜ×"Ñ"¤5¨£<Ó0ˆàJOÖPÀQ¤e×&6Ñ&6°qÔ&9ŒU×Ñ Ô"¸qÑ@ÐP€EÐPØMUÖVÈ¬×)9Ñ)9¸!Ô)<”×"Ñ" 1Ô%À!ÑCÐV€HÐVØKQÖRÀa¤u×'7Ñ'7¸Ô':Œe× Ñ  Ô#ÀÑAÐR€FÐRð �^‰r¨¨q©°6Ñ)9€HÜ	×Ñ˜  H¨|Ñ$;Ñ ;Ô<ô × Ñ  ¨5Ó1€GÜ	×Ñ˜ ¨a°%©iÕ8Ø")Ö*˜Qˆq�w‰w�yÐ*€GÐ*áä×"Ñ" 7¨EÓ2ˆÜ14°U¸EÓ1B×C©¨¨A�!�a‘%—‘˜AŸG™GÕ$ÐCˆÑCØ(-Ö. 1�a—f‘f•hÐ.ˆ
Ð.Ü×Ñ 
¨DÔ1Ü×Ñ˜E :Ô.Ü×Ñ˜G UÔ+ä	×Ñ˜ °¨sÕ3ô 
×Ñ˜ %Ô(Ü	×Ñ˜ %¨q°5©yÖ9ùò5 QùÚVùÚRùò +ùó
 DùÚ.s#   ´3IÁ-3IÂ&3IÄ8IÅ8(I!Æ'I')FN)rA   Útypingr   r   r   r-   Útorch.optim.optimizerr   Ú_helpersr   r	   Ú_typesr
   r   ÚTensorrC   rB   r6   rQ   rP   © r    r   ú<module>r€      s×  ðñ÷, )Ñ (ã Ý +ç 4Ý ôdˆ9ô dðZ Øð.ð #'ò.Ø�U—\‘\Ñ"ð.à�E—L‘LÑ!ð.ð �u—|‘|Ñ$ð.ð ð.ð ð.ð ð.ð ð.ð ð.ð ð.ð ð.ð ˜‘ó.ðb&'Ø�U—\‘\Ñ"ð&'à�E—L‘LÑ!ð&'ð �u—|‘|Ñ$ð&'ð
 ð&'ð ð&'ð ð&'ð ð&'ð ð&'ð ð&'ð ˜‘ó&'ðR-:Ø�U—\‘\Ñ"ð-:à�E—L‘LÑ!ð-:ð �u—|‘|Ñ$ð-:ð
 ð-:ð ð-:ð ð-:ð ð-:ð ð-:ð ð-:ð ˜‘ô-:r    