Ë
    éÓ^jñ&  ã                   ór   — d Z ddlZddlmZmZ ddlZddlmZ  G d„ dej                  j                  «      Z
y)zí Adafactor Optimizer

Lifted from https://github.com/pytorch/fairseq/blob/master/fairseq/optim/adafactor.py

Modified by Ross Wightman to fix some issues with factorization dims for non nn.Linear layers

Original header/copyright below.
é    N)ÚOptionalÚTupleé   )ÚParamsTc                   óô   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 ddedee   dededededeeeef      d	ed
edede	defˆ fd„Z
ˆ fd„Zed„ «       Zedd„«       Zed„ «       Zd„ Z ej$                  «       dd„«       Zˆ xZS )Ú	AdafactoraÕ  Implements Adafactor algorithm.

    This implementation is based on: `Adafactor: Adaptive Learning Rates with Sublinear Memory Cost`
    (see https://arxiv.org/abs/1804.04235)

    Note that this optimizer internally adjusts the learning rate depending on the
    *scale_parameter*, *relative_step* and *warmup_init* options.

    To use a manual (external) learning rate schedule you should set `scale_parameter=False` and
    `relative_step=False`.

    Ags:
        params: iterable of parameters to optimize or dicts defining parameter groups
        lr: external learning rate
        eps: regularization constants for square gradient and parameter scale respectively
        eps_scale: regularization constants for parameter scale respectively
        clip_threshold: threshold of root-mean-square of final gradient update
        decay_rate: coefficient used to compute running averages of square gradient
        beta1: coefficient used for computing running averages of gradient
        weight_decay: weight decay
        scale_parameter: if True, learning rate is scaled by root-mean-square of parameter
        warmup_init: time-dependent learning rate computation depends on whether warm-up initialization is being used
    ÚparamsÚlrÚepsÚ	eps_scaleÚclip_thresholdÚ
decay_rateÚbetasÚweight_decayÚscale_parameterÚwarmup_initÚmin_dim_size_to_factorÚcautionc                 ó’   •— | }|
r|st        d«      ‚|€d n|d   }t        ||||||||	||
||¬«      }t        t        | �  ||«       y )Nz'warmup_init requires relative_step=Truer   )r
   r   r   r   r   Úbeta1r   r   Úrelative_stepr   r   r   )Ú
ValueErrorÚdictÚsuperr   Ú__init__)Úselfr	   r
   r   r   r   r   r   r   r   r   r   r   r   r   ÚdefaultsÚ	__class__s                   €ú_/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/optim/adafactor.pyr   zAdafactor.__init__.   sm   ø€ ð ˜ˆÙ™}ÜÐFÓGÐGà˜‘¨5°©8ˆÜØØØØ)Ø!ØØ%Ø+Ø'Ø#Ø#9Øô
ˆô 	Œi˜Ñ'¨°Õ9ó    c                 óŽ   •— t         ‰| �  |«       | j                  D ]&  }|j                  dd«       |j                  dd«       Œ( y )Nr   Fr   é   )r   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroupr   s      €r   r#   zAdafactor.__setstate__R   sF   ø€ Ü‰Ñ˜UÔ#Ø×&Ñ&ò 	;ˆEØ×Ñ˜Y¨Ô.Ø×ÑÐ5°rÕ:ñ	;r    c                 óÀ   — | d   rU| d   rd|d   z  nd}t        |dt        j                  |d   «      z  «      }d}| d   rt        | d   |d	   «      }||z  | d
<   | d
   S )Nr   r   g�íµ ÷Æ°>Ústepg{®Gáz„?ç      ð?r   r   ÚRMSr
   )ÚminÚmathÚsqrtÚmax)Úparam_groupÚparam_stateÚmin_stepÚlr_tÚparam_scales        r   Ú_get_lrzAdafactor._get_lrX   s€   € à�Ò'Ø5@ÀÒ5O�t˜k¨&Ñ1Ò1ÐUYˆHÜ�x ¤t§y¡y°¸VÑ1DÓ'EÑ!EÓFˆDØˆKØÐ,Ò-Ü! +¨kÑ":¸KÈÑ<NÓO�Ø $ {Ñ 2ˆK˜ÑØ˜4Ñ Ð r    c                 ó¦   — | d   d u}d }t        |«      }|dkD  r|d   |kD  r|d   |kD  rd}||fS |dk\  r|d   |kD  r|d   |kD  r
|dz
  |dz
  f}||fS )Nr   é   r   r   )r   r   éþÿÿÿéÿÿÿÿ)Úlen)r0   Úparam_shapeÚmin_size_to_factorÚuse_first_momentÚfactoredÚndims         r   Ú_get_optionszAdafactor._get_optionsc   s•   € à& wÑ/°tÐ;ÐØˆÜ�;Óˆð
 �!Š8˜ A™Ð);Ò;ÀÈAÁÐQcÒ@càˆHð
 Ð)Ð)Ð)ð	 �QŠY˜; r™?Ð-?Ò?ÀKÐPRÁOÐVhÒDhà˜a‘x ¨¡Ð)ˆHàÐ)Ð)Ð)r    c                 óL   — | j                  d«      | j                  «       dz  z  S )Nr7   g      à?)ÚnormÚnumel)Útensors    r   Ú_rmszAdafactor._rmsu   s    € à�{‰{˜1‹~ §¡£°3Ñ!6Ñ7Ð7r    c                 óÒ   — ||j                  |d¬«      z  j                  «       j                  |«      }|j                  |«      j                  «       }t	        j
                  ||«      S )NT)ÚdimÚkeepdim)ÚmeanÚrsqrt_Ú	unsqueezeÚrsqrtÚtorchÚmul)r   Úexp_avg_sq_rowÚexp_avg_sq_colÚdim_colÚdim_rowÚr_factorÚc_factors          r   Ú_approx_sq_gradzAdafactor._approx_sq_grady   s[   € à" ^×%8Ñ%8¸WÈdÐ%8Ó%SÑS×[Ñ[Ó]×gÑgÐhoÓpˆØ!×+Ñ+¨GÓ4×:Ñ:Ó<ˆÜ�y‰y˜ 8Ó,Ð,r    c                 óÖ	  — d}|�$t        j                  «       5   |«       }ddd«       | j                  D �]£  }|d   D �]—  }|j                  €Œ|j                  }|j                  t         j
                  t         j                  hv r|j                  «       }|j                  rt        d«      ‚| j                  |   }| j                  ||j                  |d   ¬«      \  }}t        |«      dk(  r¸d|d<   |rt        j                  |«      |d<   |�y|\  }	}
d	„ }t        j                   ||j                  |
«      «      j!                  |«      |d
<   t        j                   ||j                  |	«      «      j!                  |«      |d<   nt        j                  |«      |d<   d|d<   na|r|d   j!                  |«      |d<   |�/|d
   j!                  |«      |d
<   |d   j!                  |«      |d<   n|d   j!                  |«      |d<   |}|j                  t         j
                  t         j                  hv r|j                  «       }|dxx   dz  cc<   | j#                  |«      |d<   | j%                  ||«      }dt'        j(                  |d   |d   «      z
  }|dz  |d   z   }|�Ÿ|\  }	}
|d
   }|d   }|j+                  |«      j-                  |j/                  |
¬«      d|z
  ¬«       |j+                  |«      j-                  |j/                  |	¬«      d|z
  ¬«       | j1                  |||	|
«      }|j+                  |«       nI|d   }|j+                  |«      j-                  |d|z
  ¬«       |j3                  «       j+                  |«      }|j5                  | j#                  |«      |d   z  j7                  d¬«      «       |j+                  |«       |r�|d   }|j+                  |d   «      j-                  |d|d   z
  ¬«       |d   rV||z  dkD  j!                  |j                  «      }|j5                  |j/                  «       j7                  d¬«      «       ||z  }n|}|d   dk7  r|j-                  ||d    |z  ¬«       |j-                  | «       |j                  t         j
                  t         j                  hv s�Œ‡|j9                  |«       �Œš �Œ¦ |S # 1 sw Y   �Œ¿xY w)z¥Performs a single optimization step.
        Arguments:
            closure (callable, optional): A closure that reevaluates the model and returns the loss.
        Nr	   z,Adafactor does not support sparse gradients.r   )r<   r   r)   Úexp_avgc                 ó   — | d | | |dz   d  z   S )Nr   © )ÚshaperG   s     r   Ú_remove_dimz#Adafactor.step.<locals>._remove_dim¤   s   € Ø#(¨¨# ;°°s¸Q±w°x°Ñ#@Ð@r    rO   rP   Ú
exp_avg_sqr+   r   r*   r   r7   r   )rG   )Úalphar   )r,   r   r   çü©ñÒMbP?r   )rM   Úenable_gradr$   ÚgradÚdtypeÚfloat16Úbfloat16ÚfloatÚ	is_sparseÚRuntimeErrorr&   r@   rZ   r:   Ú
zeros_likeÚzerosÚtorE   r5   r-   ÚpowÚmul_Úadd_rI   rU   rL   Údiv_Úclamp_Úcopy_)r   ÚclosureÚlossr'   Úpr`   r&   Úfactored_dimsr=   rQ   rR   r[   Úp_fp32r3   Úbeta2tÚupdaterO   rP   r\   rW   Úmasks                        r   r)   zAdafactor.step   s¾  € ð ˆØÐÜ×"Ñ"Ó$ñ !Ù“y�÷!ð ×&Ñ&ó Y	$ˆEØ˜8‘_ó X$�Ø—6‘6�>ØØ—v‘v�Ø—:‘:¤%§-¡-´·±Ð!@Ñ@ØŸ:™:›<�DØ—>’>Ü&Ð'UÓVÐVàŸ
™
 1™�à26×2CÑ2CØØ—J‘JØ',Ð-EÑ'Fð 3Dó 3Ñ/�Ð/ô �u“: ’?Ø$%�E˜&‘Má'ä+0×+;Ñ+;¸DÓ+A˜˜iÑ(Ø$Ð0Ø+8Ñ(˜ òAä27·+±+¹kÈ$Ï*É*ÐV]Ó>^Ó2_×2bÑ2bÐcgÓ2h˜Ð.Ñ/Ü27·+±+¹kÈ$Ï*É*ÐV]Ó>^Ó2_×2bÑ2bÐcgÓ2h˜Ð.Ò/ä.3×.>Ñ.>¸tÓ.D˜˜lÑ+à#$�E˜%’Lá'Ø+0°Ñ+;×+>Ñ+>¸tÓ+D˜˜iÑ(Ø$Ð0Ø27Ð8HÑ2I×2LÑ2LÈTÓ2R˜Ð.Ñ/Ø27Ð8HÑ2I×2LÑ2LÈTÓ2R˜Ð.Ò/à.3°LÑ.A×.DÑ.DÀTÓ.J˜˜lÑ+à�Ø—7‘7œuŸ}™}¬e¯n©nÐ=Ñ=Ø#Ÿ\™\›^�Fà�f“ Ñ"“Ø#Ÿy™y¨Ó0��e‘Ø—|‘| E¨5Ó1�àœtŸx™x¨¨f©°u¸\Ñ7JÓKÑK�Ø ™ U¨5¡\Ñ1�Ø Ð,Ø'4Ñ$�G˜WØ%*Ð+;Ñ%<�NØ%*Ð+;Ñ%<�Nà"×'Ñ'¨Ó/×4Ñ4°V·[±[ÀW°[Ó5MÐUXÐ[aÑUaÐ4ÔbØ"×'Ñ'¨Ó/×4Ñ4°V·[±[ÀW°[Ó5MÐUXÐ[aÑUaÐ4Ôbð "×1Ñ1°.À.ÐRYÐ[bÓc�FØ—K‘K Õ%à!& |Ñ!4�Jà—O‘O FÓ+×0Ñ0°¸sÀV¹|Ð0ÔLØ'×-Ñ-Ó/×4Ñ4°TÓ:�Fà—‘˜TŸY™Y vÓ.°Ð7GÑ1HÑH×PÑPÐUXÐPÓYÔZØ—‘˜DÔ!á#Ø# IÑ.�GØ—L‘L  w¡Ó0×5Ñ5°fÀAÈÈgÉÑDVÐ5ÔWØ˜YÒ'à '¨$¡°Ñ 2×6Ñ6°t·z±zÓB˜ØŸ	™	 $§)¡)£+×"4Ñ"4¸Ð"4Ó">Ô?Ø!(¨4¡™à!(˜à˜Ñ(¨AÒ-Ø—K‘K ¨u°^Ñ/DÐ.DÀtÑ.K�KÔLà—‘˜V˜GÔ$Ø—7‘7œuŸ}™}¬e¯n©nÐ=Ó=Ø—G‘G˜F–OòqX$ðY	$ðv ˆ÷}!ñ !ús   ™SÓS()Ng ÂëþKH´9r^   r*   gš™™™™™é¿Ng        TFr"   F)r"   )N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   rd   r   ÚboolÚintr   r#   Ústaticmethodr5   r@   rE   rU   rM   Úno_gradr)   Ú__classcell__)r   s   @r   r   r      s  ø„ ñð6 #'ØØ#Ø$'Ø $Ø37Ø"%Ø$(Ø %Ø*,Ø!ñ":àð":ð ˜‘ð":ð ð	":ð
 ð":ð "ð":ð ð":ð ˜E %¨ ,Ñ/Ñ0ð":ð  ð":ð "ð":ð ð":ð %(ð":ð õ":ôH;ð ñ!ó ð!ð ò*ó ð*ð" ñ8ó ð8ò-ð €U‡]�]ƒ_òeó ôer    r   )r{   r-   Útypingr   r   rM   Ú_typesr   ÚoptimÚ	Optimizerr   rY   r    r   ú<module>r…      s0   ðñó ß "ã å ôP�—‘×%Ñ%õ Pr    