Ë
    éÓ^j,  ã                   óª  — d Z ddlmZmZ ddlZddlmZ ddlmZ 	 ddlmZm	Z	 dZ
d	d
lmZ ddgZ G d„ de«      Z	 	 dddœdee   dee   deee      dedee   dedededededededee   fd„Zdee   dee   deee      dededededededededee   fd„Zdee   dee   deee      dededededededededee   fd„Zy# e$ r dZ
Y Œ«w xY w) a   SGD with decoupled weight-decay.

References for added functionality:
    Cautious Optimizers: https://arxiv.org/abs/2411.16085
    Why Gradients Rapidly Increase Near the End of Training: https://arxiv.org/abs/2506.02285

Hacked together by Ross Wightman
é    )ÚListÚOptionalN)ÚTensor)Ú	Optimizer)Ú_use_grad_for_differentiableÚ_default_to_fused_or_foreachTFé   )ÚParamsTÚSGDWÚsgdwc                   ó¬   ‡ — e Zd Z	 	 	 	 	 dddddddœdededededed	ed
edededee   defˆ fd„Zˆ fd„Zd„ Z	 e
j                  «       dd„«       Zˆ xZS )r   FN)ÚcautionÚcorrected_weight_decayÚmaximizeÚforeachÚdifferentiableÚparamsÚlrÚmomentumÚ	dampeningÚweight_decayÚnesterovr   r   r   r   r   c                óð   •— |dk  rt        d|› �«      ‚|dk  rt        d|› �«      ‚|dk  rt        d|› �«      ‚t        ||||||||	|
|¬«
      }|r|dk  s|dk7  rt        d«      ‚t        ‰| �  ||«       y )Nç        zInvalid learning rate: zInvalid momentum value: zInvalid weight_decay value: )
r   r   r   r   r   r   r   r   r   r   r   z8Nesterov momentum requires a momentum and zero dampening)Ú
ValueErrorÚdictÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   r   r   r   r   ÚdefaultsÚ	__class__s                €úZ/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/optim/sgdw.pyr   zSGDW.__init__   s¨   ø€ ð �Š8ÜÐ6°r°dÐ;Ó<Ð<Ø�cŠ>ÜÐ7¸°zÐBÓCÐCØ˜#ÒÜÐ;¸L¸>ÐJÓKÐKäØØØØ%ØØØ#9ØØØ)ô
ˆñ ˜ Qš¨)°qª.ÜÐWÓXÐXÜ‰Ñ˜ Õ*ó    c                 ó  •— t         ‰| �  |«       | j                  D ]n  }|j                  dd«       |j                  dd«       |j                  dd«       |j                  dd«       |j                  dd «       |j                  dd«       Œp y )Nr   Fr   r   r   r   r   )r   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroupr!   s      €r"   r%   zSGDW.__setstate__@   s‡   ø€ Ü‰Ñ˜UÔ#Ø×&Ñ&ò 	6ˆEØ×Ñ˜Y¨Ô.Ø×ÑÐ5°uÔ=Ø×Ñ˜Z¨Ô/Ø×Ñ˜Z¨Ô/Ø×Ñ˜Y¨Ô-Ø×ÑÐ-¨uÕ5ñ	6r#   c                 ó2  — d}|d   D ]Œ  }|j                   €Œ|j                  |«       |j                  |j                   «       |j                   j                  rd}| j                  |   }d|vr|j                  d «       Œy|j                  |d   «       ŒŽ |S )NFr   TÚmomentum_buffer)ÚgradÚappendÚ	is_sparser(   )r   r)   Úparams_with_gradÚgradsÚmomentum_buffer_listÚhas_sparse_gradÚpr(   s           r"   Ú_init_groupzSGDW._init_groupJ   s•   € Øˆà�x‘ò 	JˆAØ�v‰vÑ!Ø ×'Ñ'¨Ô*Ø—‘˜QŸV™VÔ$Ø—6‘6×#Ò#Ø&*�OàŸ
™
 1™�Ø$¨EÑ1Ø(×/Ñ/°Õ5à(×/Ñ/°Ð6GÑ0HÕIð	Jð Ðr#   c                 óœ  — d}|�$t        j                  «       5   |«       }ddd«       | j                  D ]ˆ  }g }g }g }| j                  ||||«      }t	        ||||d   |d   |d   |d   |d   |d   |d   ||d	   |d
   r| j
                  d   nd¬«       t        ||«      D ]  \  }}	| j                  |   }
|	|
d<   Œ ŒŠ |S # 1 sw Y   Œ¢xY w)z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r   r   r   r   )
r   r   r   r   r   r   r   r2   r   Úmax_lrr+   )ÚtorchÚenable_gradr&   r4   r   r    Úzipr(   )r   ÚclosureÚlossr)   r/   r0   r1   r2   r3   r+   r(   s              r"   Ústepz	SGDW.step^   s  € ð ˆØÐÜ×"Ñ"Ó$ñ !Ù“y�÷!ð ×&Ñ&ò 	;ˆEØ!ÐØˆEØ#%Ð à"×.Ñ.¨uÐ6FÈÐOcÓdˆOäØ ØØ$Ø" >Ñ2Ø˜zÑ*Ø˜‘;Ø Ñ,Ø˜zÑ*Ø˜iÑ(Ø˜zÑ*Ø /Ø˜iÑ(Ø.3Ð4LÒ.M�t—}‘} TÒ*ÐSWõô" '*Ð*:Ð<PÓ&Qò ;Ñ"��?ØŸ
™
 1™�Ø+:�Ð'Ò(ñ;ð1	;ð8 ˆ÷?!ð !ús   ™CÃC)çü©ñÒMbP?r   r   r   F©N)Ú__name__Ú
__module__Ú__qualname__r
   ÚfloatÚboolr   r   r%   r4   r7   Úno_gradr<   Ú__classcell__)r!   s   @r"   r   r      sÆ   ø„ ð Ø Ø!Ø"$Ø"ð$+ð "Ø+0Ø"Ø&*Ø#(ò$+àð$+ð ð$+ð ð	$+ð
 ð$+ð  ð$+ð ð$+ð ð$+ð %)ð$+ð ð$+ð ˜d‘^ð$+ð !õ$+ôL6òð( €U‡]�]ƒ_ò(ó ô(r#   )r6   r   r0   r1   r2   r   r   r   r   r   r   r   r   r6   c                ót  — t         rot        t        d«      r_|€2t        j                  j                  «       st        | dd¬«      \  }}nd}|r+t        j                  j                  «       rt        d«      ‚d}|r%t        j                  j                  «       st        }nt        } || |||||||	|
|||¬«       y)zlFunctional API that performs SGD algorithm computation.

    See :class:`~torch.optim.SGD` for details.
    Ú"_group_tensors_by_device_and_dtypeNF)r   Ú	use_fusedz6torch.jit.script not supported with foreach optimizers)	r   r   r   r   r   r   r2   r   r6   )
Úhas_recent_ptÚhasattrr   r7   ÚjitÚis_scriptingr   ÚRuntimeErrorÚ_multi_tensor_sgdwÚ_single_tensor_sgdw)r   r0   r1   r2   r   r   r   r   r   r   r   r   r6   Ú_Úfuncs                  r"   r   r   Š   s¬   € õ, œ¤Ð,PÔQØˆ?ô —9‘9×)Ñ)Ô+Ü9¸&ÐQVÐbgÔh‘
�‘7à�á”u—y‘y×-Ñ-Ô/ÜÐWÓXÐXàˆá”u—y‘y×-Ñ-Ô/Ü!‰ä"ˆáØØØØ!ØØØØØØ'ØØör#   c       	         ót  — t        | «      D �])  \  }}|	s||   n||    }|€|n|dz  |z  }|j                  d||z  z
  «       |dk7  rÚ||   }|€)t        j                  |«      j	                  «       }|||<   n%|j                  |«      j                  |d|z
  ¬«       |rk|r|j                  ||¬«      }||z  dkD  j                  |j                  «      }|j                  |j                  «       j                  d¬«      «       ||z  }n|r|j                  ||¬«      }n|}|j                  || ¬«       �Œ, y )Né   ç      ð?r   r	   ©Úalphar=   )Úmin)Ú	enumerateÚmul_r7   ÚcloneÚdetachÚadd_ÚaddÚtoÚdtypeÚdiv_ÚmeanÚclamp_)r   r0   r1   r   r   r   r   r   r   r   r2   r6   ÚiÚparamr,   Úwd_scaleÚbufÚmasks                     r"   rO   rO   Ã   s:  € ô ˜fÓ%ó $‰ˆˆ5Ù'ˆu�QŠx¨e°A©h¨Yˆà˜‘2¨R°1©W°vÑ-=ˆØ�
‰
�2˜ <Ñ/Ñ/Ô0à�qŠ=Ø& qÑ)ˆCàˆ{Ü—k‘k $Ó'×.Ñ.Ó0�Ø*-Ð$ QÒ'à—‘˜Ó"×'Ñ'¨°A¸	±MÐ'ÔBáÙØŸ(™( 3¨h˜(Ó7�Cà˜d™
 Q™×*Ñ*¨4¯:©:Ó6�Ø—	‘	˜$Ÿ)™)›+×,Ñ,°Ð,Ó6Ô7Ø˜T‘z‘áØŸ8™8 C¨x˜8Ó8‘Dà�Dà�
‰
�4 ˜sˆ
Ö#ñ9$r#   c       	         óÐ  — t        | «      dk(  ry t        j                  | ||gd¬«      }|j                  «       D �]ž  \  \  }}}}|
xr t	        d„ |D «       «      }|	rt        j                  |«      }|€|n|dz  |z  }t        j                  | d||z  z
  «       |dk7  �rég }d}t        t        |«      «      D ]  }||   €d} n|j                  ||   «       Œ! |r2t        j                  ||«       t        j                  ||d|z
  ¬	«       n’g }t        t        |«      «      D ]y  }||   €4t        j                  ||   «      j                  «       x}x||<   |||   <   n-||   }|j                  |«      j                  ||   d|z
  ¬	«       |j                  |«       Œ{ |rÊ|rt        j                  |||¬	«      }t        j                   ||«      }t#        ||«      D ��cg c]#  \  }}|dkD  j%                  |j&                  «      ‘Œ% }}}|D �cg c]  }|j)                  «       ‘Œ }}t        j*                  |d
«       t        j,                  ||«       t        j                   ||«      }n|rt        j                  |||¬	«       n|}|st        j                  ||| ¬	«       �Œlt        t        |«      «      D ]  }||   j                  ||   | ¬	«       Œ �Œ¡ y c c}}w c c}w )Nr   T)Úwith_indicesc              3   ó4   K  — | ]  }|j                   –— Œ y ­wr>   )r.   )Ú.0r,   s     r"   ú	<genexpr>z%_multi_tensor_sgdw.<locals>.<genexpr>  s   è ø€ Ò8aÈD¸¿½Ñ8aùs   ‚rS   rT   Fr	   rU   r=   )Úlenr   rG   ÚvaluesÚanyr7   Ú_foreach_negÚ_foreach_mul_Úranger-   Ú_foreach_add_rZ   r[   rY   r\   Ú_foreach_addÚ_foreach_mulr9   r^   r_   ra   Ú_foreach_maximum_Ú_foreach_div_)r   r0   r1   r   r   r   r   r   r   r   r2   r6   Úgrouped_tensorsÚdevice_paramsÚdevice_gradsÚdevice_momentum_buffer_listÚindicesÚdevice_has_sparse_gradre   ÚbufsÚall_states_with_momentum_bufferrc   rf   ÚmasksÚmÚgÚ
mask_scales                              r"   rN   rN   ñ   sò  € ô ˆ6ƒ{�aÒØä×BÑBØ	�Ð,Ð-¸DôB€OàQ`×QgÑQgÓQió 9BÑMÑ	Cˆ-˜Ð'BÀWØ!0Ò!a´SÑ8aÐT`Ô8aÓ5aÐáÜ ×-Ñ-¨lÓ;ˆLà˜‘2¨R°1©W°vÑ-=ˆÜ×Ñ˜F B¨°LÑ)@Ñ$@ÔAà�q‹=ØˆDà.2Ð+Üœ3Ð:Ó;Ó<ò @�Ø.¨qÑ1Ð9Ø6;Ð3Ùà—K‘KÐ ;¸AÑ >Õ?ð@ñ /Ü×#Ñ# D¨(Ô3Ü×#Ñ# D¨,¸aÀ)¹mÖLà�ÜœsÐ#>Ó?Ó@ò %�AØ2°1Ñ5Ð=ä!ŸK™K¨°Q©Ó8×?Ñ?ÓAðB˜ð BÐ9¸!Ñ<Ð?SÐT[Ð\]ÑT^Ò?_ð :¸!Ñ<˜ØŸ™ Ó*×/Ñ/°¸Q±ÀqÈ9Á}Ð/ÔUà—K‘K Õ$ð%ñ Ùä ×-Ñ-¨l¸DÈÔQ�Dä×*Ñ*¨4°Ó>�Ü9<¸UÀLÓ9Q×R±°°A˜!˜a™%Ÿ™ A§G¡GÕ,ÐR�ÑRØ05Ö6¨1˜aŸf™f�hÐ6�
Ð6Ü×'Ñ'¨
°DÔ9Ü×#Ñ# E¨:Ô6Ü$×1Ñ1°$¸Ó>‘áÜ×'Ñ'¨°dÀ(ÖKà#'�Lá%Ü×Ñ ¨|ÀBÀ3×Gô œ3˜}Ó-Ó.ò B�Ø˜aÑ ×%Ñ% l°1¡o¸b¸SÐ%ÕAòBñq9BùóR SùÚ6s   Ç!(KÈK#)NN)Ú__doc__Útypingr   r   r7   r   Útorch.optim.optimizerr   r   r   rI   ÚImportErrorÚ_typesr
   Ú__all__r   rC   rB   r   rO   rN   © r#   r"   ú<module>r‹      sC  ðñ÷ "ã Ý Ý +ðß`Ø€Mõ à�6Ð
€ônˆ9ô nðn !%Ø"&ð6ð  #'ò!6Ø�V‘ð6à�F‰|ð6ð # 8¨FÑ#3Ñ4ð6ð ð6ð ˜$‘ð6ð ð6ð ð6ð ð6ð ð6ð ð6ð ð6ð ð6ð  ˜‘ó!6ðr+$Ø�V‘ð+$à�F‰|ð+$ð # 8¨FÑ#3Ñ4ð+$ð
 ð+$ð ð+$ð ð+$ð ð+$ð ð+$ð ð+$ð ð+$ð ð+$ð ˜‘ó+$ð\MBØ�V‘ðMBà�F‰|ðMBð # 8¨FÑ#3Ñ4ðMBð
 ðMBð ðMBð ðMBð ðMBð ðMBð ðMBð ðMBð ðMBð ˜‘ôMBøðA ò Ø‚Mðús   œ
C ÃCÃC