Ë
    éÓ^jí-  ã            "       óP  — d Z ddlZddlmZmZmZ ddlZddlmZ ddlm	Z	  G d„ de
«      Z G d„ d	e	«      Zd
ee   dee   dee   dee   dee   dee   dededededededededededef"d„Zd
ee   dee   dee   dee   dee   dee   dededededededededededef"d„Zy)zë Adan Optimizer

Adan: Adaptive Nesterov Momentum Algorithm for Faster Optimizing Deep Models[J]. arXiv preprint arXiv:2208.06677, 2022.
    https://arxiv.org/abs/2208.06677

Implementation adapted from https://github.com/sail-sg/Adan
é    N)ÚListÚOptionalÚTuple)ÚTensor)Ú	Optimizerc                   ó    — e Zd ZdZdZd„ Zd„ Zy)ÚMultiTensorApplyFc                 ó‚   — 	 dt         _        || _        y # t        $ r }dt         _        |t         _        Y d }~y d }~ww xY w)NTF)r	   Ú	availableÚ
chunk_sizeÚImportErrorÚ
import_err)Úselfr   Úerrs      úZ/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/optim/adan.pyÚ__init__zMultiTensorApply.__init__"   s<   € ð	.Ø)-ÔÔ&Ø(ˆD�OøÜò 	.Ø).ÔÔ&Ø*-Ô×'Ñ'ûð	.ús   ‚ •	>ž9¹>c                 ó,   —  || j                   ||g|¢­Ž S ©N)r   )r   ÚopÚnoop_flag_bufferÚtensor_listsÚargss        r   Ú__call__zMultiTensorApply.__call__*   s   € Ù�$—/‘/Ð#3°\ÐIÀDÒIÐIó    N)Ú__name__Ú
__module__Ú__qualname__r   Úwarnedr   r   © r   r   r	   r	      s   „ Ø€IØ€Fò.óJr   r	   c                   óÊ   ‡ — e Zd ZdZ	 	 	 	 	 	 	 ddedeeeef   dededededee   fˆ fd	„Zˆ fd
„Z	 e
j                  «       d„ «       Z e
j                  «       dd„«       Zˆ xZS )ÚAdanaØ   Implements a pytorch variant of Adan.

    Adan was proposed in Adan: Adaptive Nesterov Momentum Algorithm for Faster Optimizing Deep Models
    https://arxiv.org/abs/2208.06677

    Arguments:
        params: Iterable of parameters to optimize or dicts defining parameter groups.
        lr: Learning rate.
        betas: Coefficients used for first- and second-order moments.
        eps: Term added to the denominator to improve numerical stability.
        weight_decay: Decoupled weight decay (L2 penalty)
        no_prox: How to perform the weight decay
        caution: Enable caution from 'Cautious Optimizers'
        foreach: If True would use torch._foreach implementation. Faster but uses slightly more memory.
    ÚlrÚbetasÚepsÚweight_decayÚno_proxÚcautionÚforeachc	           	      óÔ  •— d|k  st        dj                  |«      «      ‚d|k  st        dj                  |«      «      ‚d|d   cxk  rdk  sn t        dj                  |d   «      «      ‚d|d   cxk  rdk  sn t        dj                  |d   «      «      ‚d|d	   cxk  rdk  sn t        d
j                  |d	   «      «      ‚t        |||||||¬«      }	t        ‰
| �  ||	«       y )Nç        zInvalid learning rate: {}zInvalid epsilon value: {}r   ç      ð?z%Invalid beta parameter at index 0: {}é   z%Invalid beta parameter at index 1: {}é   z%Invalid beta parameter at index 2: {})r"   r#   r$   r%   r&   r'   r(   )Ú
ValueErrorÚformatÚdictÚsuperr   )r   Úparamsr"   r#   r$   r%   r&   r'   r(   ÚdefaultsÚ	__class__s             €r   r   zAdan.__init__?   sø   ø€ ð �bŠyÜÐ8×?Ñ?ÀÓCÓDÐDØ�cŠzÜÐ8×?Ñ?ÀÓDÓEÐEØ�e˜A‘hÔ$ Ô$ÜÐD×KÑKÈEÐRSÉHÓUÓVÐVØ�e˜A‘hÔ$ Ô$ÜÐD×KÑKÈEÐRSÉHÓUÓVÐVØ�e˜A‘hÔ$ Ô$ÜÐD×KÑKÈEÐRSÉHÓUÓVÐVäØØØØ%ØØØô
ˆô 	‰Ñ˜ Õ*r   c                 ó–   •— t         t        | �  |«       | j                  D ]&  }|j	                  dd«       |j	                  dd«       Œ( y )Nr&   Fr'   )r1   r!   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroupr4   s      €r   r6   zAdan.__setstate___   sG   ø€ ÜŒd�DÑ& uÔ-Ø×&Ñ&ò 	/ˆEØ×Ñ˜Y¨Ô.Ø×Ñ˜Y¨Õ.ñ	/r   c                 ó  — | j                   D ]u  }d|d<   |d   D ]f  }|j                  sŒ| j                  |   }t        j                  |«      |d<   t        j                  |«      |d<   t        j                  |«      |d<   Œh Œw y )Nr   Ústepr2   Úexp_avgÚ
exp_avg_sqÚexp_avg_diff)r7   Úrequires_gradr9   ÚtorchÚ
zeros_like)r   r:   Úpr9   s       r   Úrestart_optzAdan.restart_opte   sŒ   € à×&Ñ&ò 	@ˆEØˆE�&‰MØ˜8‘_ò 
@�Ø—?“?Ø ŸJ™J q™M�Eô (-×'7Ñ'7¸Ó':�E˜)Ñ$ä*/×*:Ñ*:¸1Ó*=�E˜,Ñ'ä,1×,<Ñ,<¸QÓ,?�E˜.Ò)ñ
@ñ	@r   c                 óÊ  — d}|�$t        j                  «       5   |«       }ddd«       	 dt         j                  j                  j                  j                  «       v }| j                  D �]Ö  }g }g }g }g }g }	g }
|d   \  }}}d|v r|dxx   dz  cc<   nd|d<   d||d   z  z
  }d||d   z  z
  }d||d   z  z
  }|d   D �]  }|j                  €Œ|j                  |«       |j                  |j                  «       | j                  |   }t        |«      d	k(  rHt        j                  |«      |d
<   t        j                  |«      |d<   t        j                  |«      |d<   d|vs|d   dk(  r|j                  j                  «        |d<   |j                  |d
   «       |j                  |d   «       |	j                  |d   «       |
j                  |d   «       �Œ |s�Œy|d   €|d    xs |}n|d   }|rt        }nt        } |||f|||	|
|||||t!        j"                  |«      |d   |d   |d   |d   |d   dœŽ �ŒÙ |S # 1 sw Y   �Œ'xY w# t        $ r d}Y �Œw xY w)z$Performs a single optimization step.NÚScalarFr#   r<   r,   r+   r2   r   r=   r>   r?   Úneg_pre_gradr(   r'   r"   r%   r$   r&   )Úexp_avgsÚexp_avg_sqsÚexp_avg_diffsÚneg_pre_gradsÚbeta1Úbeta2Úbeta3Úbias_correction1Úbias_correction2Úbias_correction3_sqrtr"   r%   r$   r&   r'   )rA   Úenable_gradÚopsÚatenÚ_foreach_maximum_Ú	overloadsÚ	Exceptionr7   ÚgradÚappendr9   ÚlenrB   ÚcloneÚ_multi_tensor_adanÚ_single_tensor_adanÚmathÚsqrt)r   ÚclosureÚlossÚhas_scalar_maximumr:   Úparams_with_gradÚgradsrH   rI   rJ   rK   rL   rM   rN   rO   rP   Úbias_correction3rC   r9   Úuse_foreachÚfuncs                        r   r<   z	Adan.stepu   sÉ  € ð ˆØÐÜ×"Ñ"Ó$ñ !Ù“y�÷!ð	'Ø!)¬U¯Y©Y¯^©^×-MÑ-M×-WÑ-WÓ-YÐ!YÐð ×&Ñ&ó G	ˆEØ!ÐØˆEØˆHØˆKØˆMØˆMà"'¨¡.ÑˆE�5˜%ð ˜‰Ø�f“ Ñ"”à !��f‘à" U¨e°F©mÑ%;Ñ;ÐØ" U¨e°F©mÑ%;Ñ;ÐØ" U¨e°F©mÑ%;Ñ;Ðà˜8‘_ó <�Ø—6‘6�>ØØ ×'Ñ'¨Ô*Ø—‘˜QŸV™VÔ$àŸ
™
 1™�Ü�u“: ’?Ü',×'7Ñ'7¸Ó':�E˜)Ñ$Ü*/×*:Ñ*:¸1Ó*=�E˜,Ñ'Ü,1×,<Ñ,<¸QÓ,?�E˜.Ñ)à!¨Ñ.°%¸±-À1Ò2DØ-.¯V©V¯\©\«^¨O�E˜.Ñ)à—‘  iÑ 0Ô1Ø×"Ñ" 5¨Ñ#6Ô7Ø×$Ñ$ U¨>Ñ%:Ô;Ø×$Ñ$ U¨>Ñ%:Ö;ð%<ñ( $Ùà�YÑÐ'Ø"'¨	Ñ"2Ð2ÒHÐ6H‘à# IÑ.�áÜ)‘ä*�áØ Øðð "Ø'Ø+Ø+ØØØØ!1Ø!1Ü&*§i¡iÐ0@Ó&AØ˜‘;Ø" >Ñ2Ø˜%‘LØ˜iÑ(Ø˜iÑ(õ#ðkG	ðR ˆ÷c!ñ !ûô
 ò 	'Ø!&Óð	'ús   ™Iª4I ÉIÉI"É!I")çü©ñÒMbP?)g\�Âõ(\ï?gq=
×£pí?g®Gáz®ï?g:Œ0âŽyE>r*   FFNr   )r   r   r   Ú__doc__Úfloatr   Úboolr   r   r6   rA   Úno_gradrD   r<   Ú__classcell__)r4   s   @r   r!   r!   .   sº   ø„ ñð$ Ø0BØØ"%Ø!Ø!Ø&*ñ+àð+ð ˜  uÐ,Ñ-ð+ð ð	+ð
  ð+ð ð+ð ð+ð ˜d‘^õ+ô@/ð €U‡]�]ƒ_ñ@ó ð@ð €U‡]�]ƒ_òUó ôUr   r!   r2   rd   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   r"   r%   r$   r&   r'   c                óú  — t        | «      D �]ì  \  }}||   }||   }||   }||   }||   }|j                  |«       |j                  |«      j                  |d|z
  ¬«       |j                  |«      j                  |d|z
  ¬«       |j                  |«      j                  |«       |j                  |«      j                  ||d|z
  ¬«       |j	                  «       |z  j                  |«      }||z  |
z  }||	z  }|rU||z  dkD  j                  |j                  «      }|j                  |j                  «       j                  d¬«      «       ||z  }|rB|j                  d||z  z
  «       |j                  ||| ¬«       |j                  ||| ¬«       nA|j                  ||| ¬«       |j                  ||| ¬«       |j                  d||z  z   «       |j                  «       j                  |d¬«       �Œï y )Nr,   ©Úalpha©Úvaluer   rh   )Úminç      ð¿)Ú	enumerateÚadd_Úmul_Úaddcmul_r_   ÚtoÚdtypeÚdiv_ÚmeanÚclamp_Úaddcdiv_Úzero_)r2   rd   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   r"   r%   r$   r&   r'   ÚiÚparamrX   r=   r>   r?   Úneg_grad_or_diffÚdenomÚstep_size_diffÚ	step_sizeÚmasks                               r   r]   r]   Î   sû  € ô( ˜fÓ%ó #8‰ˆˆ5Ø�Q‰xˆØ˜1‘+ˆØ  ‘^ˆ
Ø$ QÑ'ˆØ(¨Ñ+Ðð 	×Ñ˜dÔ#à�‰�UÓ× Ñ  ¨Q°©YÐ Ô7Ø×Ñ˜%Ó ×%Ñ%Ð&6¸aÀ%¹iÐ%ÔHà×Ñ˜eÓ$×)Ñ)¨$Ô/Ø�‰˜Ó×'Ñ'Ð(8Ð:JÐRSÐV[ÑR[Ð'Ô\à—‘Ó"Ð%:Ñ:×@Ñ@ÀÓEˆØ˜e™Ð&6Ñ6ˆØÐ)Ñ)ˆ	áà˜d‘N QÑ&×*Ñ*¨4¯:©:Ó6ˆDØ�I‰I�d—i‘i“k×(Ñ(¨TÐ(Ó2Ô3Ø ‘nˆGáØ�J‰J�q˜2 Ñ,Ñ,Ô-Ø�N‰N˜7 E°)°ˆNÔ<Ø�N‰N˜<¨°~°oˆNÕFà�N‰N˜7 E°)°ˆNÔ<Ø�N‰N˜<¨°~°oˆNÔFØ�J‰J�q˜2 Ñ,Ñ,Ô-à×ÑÓ ×%Ñ% d°$Ð%Ö7ñG#8r   c                ó‚  — t        | «      dk(  ry t        j                  ||«       t        j                  ||«       t        j                  ||d|z
  ¬«       t        j                  ||«       t        j                  ||d|z
  ¬«       t        j                  ||«       t        j                  ||«       t        j                  ||«       t        j                  |||d|z
  ¬«       t        j
                  |«      }t        j                  ||«       t        j                  ||«       ||z  |
z  }||	z  }|r¯t        j                  ||«      }t        ||«      D ��cg c]#  \  }}|dkD  j                  |j                  «      ‘Œ% }}}|D �cg c]  }|j                  «       ‘Œ }}t        j                  |d«       t        j                  ||«       t        j                  ||«      }|rQt        j                  | d||z  z
  «       t        j                  | ||| ¬«       t        j                  | ||| ¬«       nPt        j                  | ||| ¬«       t        j                  | ||| ¬«       t        j                  | d||z  z   «       t        j                  |«       t        j                  ||d¬«       y c c}}w c c}w )Nr   r,   ro   rq   rh   rt   )rZ   rA   Ú_foreach_add_Ú_foreach_mul_Ú_foreach_addcmul_Ú_foreach_sqrtÚ_foreach_div_Ú_foreach_mulÚzipry   rz   r|   rU   Ú_foreach_addcdiv_Ú_foreach_zero_)r2   rd   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   r"   r%   r$   r&   r'   rƒ   r„   r…   ÚmasksÚmÚgÚ
mask_scales                           r   r\   r\     sL  € ô( ˆ6ƒ{�aÒØô 
×Ñ˜ uÔ-ä	×Ñ˜ %Ô(Ü	×Ñ˜ %¨q°5©yÕ9ä	×Ñ˜ uÔ-Ü	×Ñ˜ }¸AÀ¹IÕFä	×Ñ˜ uÔ-Ü	×Ñ˜ uÔ-Ü	×Ñ˜ UÔ+Ü	×Ñ˜K¨¸ÈQÐQVÉYÕWä×Ñ Ó,€EÜ	×Ñ˜Ð4Ô5Ü	×Ñ˜˜sÔ#à˜%‘ZÐ"2Ñ2€NØÐ%Ñ%€Iáä×"Ñ" 8¨UÓ3ˆÜ14°U¸EÓ1B×C©¨¨A�!�a‘%—‘˜AŸG™GÕ$ÐCˆÑCØ(-Ö. 1�a—f‘f•hÐ.ˆ
Ð.Ü×Ñ 
¨DÔ1Ü×Ñ˜E :Ô.Ü×%Ñ% h°Ó6ˆáÜ×Ñ˜F A¨¨\Ñ(9Ñ$9Ô:Ü×Ñ ¨°%À	¸zÕJÜ×Ñ ¨°uÀ^ÀOÖTä×Ñ ¨°%À	¸zÕJÜ×Ñ ¨°uÀ^ÀOÕTÜ×Ñ˜F A¨¨\Ñ(9Ñ$9Ô:ä	×Ñ˜Ô'Ü	×Ñ˜ u°DÖ9ùó! DùÚ.s   Å(J6Æ
J<)ri   r^   Útypingr   r   r   rA   r   Útorch.optim.optimizerr   Úobjectr	   r!   rj   rk   r]   r\   r   r   r   ú<module>r˜      sÂ  ðñó* ß (Ñ (ã Ý Ý +ôJ�vô Jô ]ˆ9ô ]ð@78Ø�V‘ð78à�F‰|ð78ð �v‘,ð78ð ˜&‘\ð	78ð
 ˜F‘|ð78ð ˜F‘|ð78ð ð78ð ð78ð ð78ð  ð78ð  ð78ð  %ð78ð ð78ð ð78ð  ð!78ð" ð#78ð$ ó%78ðt?:Ø�V‘ð?:à�F‰|ð?:ð �v‘,ð?:ð ˜&‘\ð	?:ð
 ˜F‘|ð?:ð ˜F‘|ð?:ð ð?:ð ð?:ð ð?:ð  ð?:ð  ð?:ð  %ð?:ð ð?:ð ð?:ð  ð!?:ð" ð#?:ð$ ô%?:r   