Ë
    ÛÓ^jÐ[  ã                  ó€  — U d Z ddlmZ ddlZddlZddlmZ ddlm	Z	 ddl
mZmZ ddlZddlmZ ddlmZ d	d
lmZ d	dlmZ d	dlmZ d	dlmZmZ d	dlmZmZmZmZmZm Z m!Z!m"Z"m#Z# erd	dl$m%Z% d	dlm&Z&  ejN                  e(«      Z)e	 G d„ d«      «       Z*d(d„Z+ee*ge,ee-e.ef   f   f   Z/d)d„Z0	 d*	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d+d„Z1d,d„Z2d,d„Z3d,d„Z4d,d„Z5d,d„Z6d,d„Z7d,d„Z8d,d„Z9d,d„Z:d,d„Z;d,d „Z<d,d!„Z=d,d"„Z>d,d#„Z?d,d$„Z@d,d%„ZAej„                  ej†                  ejˆ                  ejŠ                  ejŒ                  ejŽ                  ej�                  ej’                  ej”                  ej–                  ej˜                  ejš                  ejœ                  ejž                  ej                   gZQej¤                  ej¦                  ej¨                  ejª                  ej¬                  ej®                  gZXej²                  ej´                  gZ[ej¸                  ejº                  gZ^ej¾                  ejÀ                  ejÂ                  gZbejÆ                  e2ejÈ                  e3ejÊ                  e3ejÌ                  e4ejÎ                  e5ejÐ                  e7ejÒ                  e8ejÔ                  e9ejÖ                  e:ejØ                  e>ejÚ                  eAejÜ                  e=ejÞ                  e=ie-já                  eQe6«      ¥e-já                  eXe;«      ¥e-já                  e[e<«      ¥e-já                  e^e?«      ¥e-já                  ebe@«      ¥Zqd&erd'<   y)-z,
Optimizer utilities for the Trainer class.
é    )ÚannotationsN)ÚCallable)Ú	dataclass)ÚTYPE_CHECKINGÚAny)Úversion)Únné   )Ú	Adafactor)ÚLayerWiseDummyOptimizer)Úcheck_target_module_exists)ÚOptimizerNamesÚParallelMode)	Úis_apollo_torch_availableÚis_bitsandbytes_availableÚis_galore_torch_availableÚis_grokadamw_availableÚis_lomo_availableÚis_schedulefree_availableÚis_torch_optimi_availableÚis_torchao_availableÚ	strtobool)ÚPreTrainedModel)ÚTrainingArgumentsc                  óD   — e Zd ZU dZded<   ded<   ded<   ded<   d	ed
<   y)ÚOptimizerContextz0Context object passed to all optimizer handlers.r   ÚargszPreTrainedModel | NoneÚmodelúdict[str, Any]Úoptimizer_kwargsÚadam_kwargsúdict[str, str]Ú
optim_argsN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú__annotations__© ó    úi/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/trainer_optimizer.pyr   r   6   s#   … á:à
ÓØ!Ó!Ø$Ó$ØÓØÔr*   r   c                ó�   — | si S i }| j                  dd«      j                  d«      D ]  }|j                  d«      \  }}|||<   Œ |S )z8Parse optimizer arguments from a comma-separated string.ú Ú ú,ú=)ÚreplaceÚsplit)Úoptim_args_strr#   ÚmappingÚkeyÚvalues        r+   Ú_parse_optim_argsr7   A   sY   € áØˆ	Ø€JØ!×)Ñ)¨#¨rÓ2×8Ñ8¸Ó=ò  ˆØ—]‘] 3Ó'‰
ˆˆUØˆ
�3Šð ð Ðr*   c                ón   — t        | t        «      r%t        | t        j                  j
                  «      ryy)a4  
    Check if the returned value from a handler is a factory rather than an Optimizer class.

    Factory callables are used for complex optimizers like Muon or Dion that need to:
    - Split parameters between multiple internal optimizers
    - Handle complex sharding logic
    - Access the full model structure for parameter grouping

    Args:
        optimizer_cls_or_factory: The first element returned by an optimizer handler.

    Returns:
        `bool`: True if it's not an Optimizer class (i.e., likely a factory), False if it's an Optimizer class.
    FT)Ú
isinstanceÚtypeÚ
issubclassÚtorchÚoptimÚ	Optimizer)Úoptimizer_cls_or_factorys    r+   Úis_optimizer_factoryr@   P   s,   € ô  Ð*¬DÔ1´jÐAYÔ[`×[fÑ[f×[pÑ[pÔ6qØØr*   c                óŠ  ‡— |j                  «       j                  d«      }|r.| j                  t        j                  k(  r|rt        d|› d�«      ‚||   }| j                  €t        d|› d�«      ‚t        | j                  t        t        f«      st        d| j                  › �«      ‚|€t        d|› d�«      ‚t        | j                  t        «      xr | j                  j                  d	d
«      dk(  }	g }
|j                  «       D ]s  \  }}t        | j                  |d¬«      \  }}t        |t        j                   «      s |r|st"        j%                  |› d|› d�«       Œ[|s|	sŒ`|
j'                  |dz   «       Œu t)        |
«      dk(  rt        d|› d| j                  › d�«      ‚|j+                  «       D ��cg c]  \  }}||
v sŒ|‘Œ }}}|j+                  «       D ��cg c]  \  }}||
vsŒ|‘Œ }}}d|id|i|¥g}|r¡| j,                  dk7  rt        d|› d�«      ‚i Š|D ]  } |d|gigfi |¤Ž‰|<   Œ |D ]  } |d|gi|¥gfi |¤Ž‰|<   Œ ˆfd„}|j/                  «       D ]   }|j0                  sŒ|j3                  |«       Œ" t4        }|j7                  d‰i«       |j7                  d|i«       ||fS c c}}w c c}}w )z¶
    Helper function to set up low-rank optimizers like GaLore and Apollo.

    These optimizers apply low-rank projections to specific target modules (typically linear layers).
    Ú	layerwisezLayer-wise z" does not support DDP at this timez1You need to define `optim_target_modules` to use z optimizerszX`optim_target_modules` must be a list of strings, a regex string, or 'all-linear'. Got: z'You need to pass a model to initialize z optimizer.Ú_ú-z
all-linearT)Úreturn_is_regexz matched but ignored. z only supports linear layers.z.weightr   zNo target modules found for z (z).Úparamsr
   z
Layerwise z( does not support gradient accumulation!c                ól   •— | j                   �'‰|    j                  «        ‰|    j                  «        y y )N)ÚgradÚstepÚ	zero_grad)ÚparamÚoptimizer_dicts    €r+   Úoptimizer_hookz1_setup_low_rank_optimizer.<locals>.optimizer_hook®   s4   ø€ Ø�z‰zÐ%Ø˜uÑ%×*Ñ*Ô,Ø˜uÑ%×/Ñ/Õ1ð &r*   rL   )ÚlowerÚendswithÚparallel_moder   ÚDISTRIBUTEDÚNotImplementedErrorÚoptim_target_modulesÚ
ValueErrorr9   ÚlistÚstrÚ	TypeErrorr1   Únamed_modulesr   r	   ÚLinearÚloggerÚwarningÚappendÚlenÚnamed_parametersÚgradient_accumulation_stepsÚ
parametersÚrequires_gradÚ"register_post_accumulate_grad_hookr   Úupdate)r   r   Úoptimizer_nameÚoptimizer_mappingÚoptim_kwargsr    Úis_layerwise_supportedÚis_layerwiseÚoptimizer_clsÚ
all_linearÚtarget_params_namesÚmodule_nameÚmoduleÚtarget_module_existsÚis_regexÚnÚpÚtarget_paramsÚnon_target_paramsÚparam_groupsrK   rM   rL   s                         @r+   Ú_setup_low_rank_optimizerru   e   sI  ø€ ð "×'Ñ'Ó)×2Ñ2°;Ó?€LÙ˜×*Ñ*¬l×.FÑ.FÒFÑKaÜ! K°Ð/?Ð?aÐ"bÓcÐcà% nÑ5€Mà× Ñ Ð(ÜÐLÈ^ÐL\Ð\gÐhÓiÐiä�d×/Ñ/´$¼°Ô=ÜðØ×-Ñ-Ð.ð0ó
ð 	
ð
 €}ÜÐBÀ>ÐBRÐR]Ð^Ó_Ð_ô 	�4×,Ñ,¬cÓ2Òr°t×7PÑ7P×7XÑ7XÐY\Ð^aÓ7bÐfrÑ7rð ð ÐØ$×2Ñ2Ó4ò <Ñˆ�VÜ)CØ×%Ñ% {ÀDô*
Ñ&Ð˜hô ˜&¤"§)¡)Ô,Ù#©HÜ—‘ + Ð.DÀ^ÐDTÐTqÐrÔsØá#©JØà×"Ñ" ;°Ñ#:Õ;ð<ô ÐÓ 1Ò$ÜÐ7¸Ð7GÀrÈ$×JcÑJcÐIdÐdfÐgÓhÐhà#(×#9Ñ#9Ó#;×X™4˜1˜a¸qÐDWÒ?W’QÐX€MÑXØ',×'=Ñ'=Ó'?×`™t˜q !À1ÐL_ÒC_šÐ`ÐÑ`ð 
Ð$Ð%Ø	�=Ð1 LÐ1ð€Lñ
 Ø×+Ñ+¨qÒ0Ü˜z¨.Ð)9Ð9aÐbÓcÐcàˆØ&ò 	]ˆEÙ$1°H¸u¸gÐ3FÐ2GÑ$\ÐK[Ñ$\ˆN˜5Ò!ð	]à"ò 	mˆEÙ$1°H¸u¸gÐ3VÈÐ3VÐ2WÑ$lÐ[kÑ$lˆN˜5Ò!ð	mô	2ð
 ×%Ñ%Ó'ò 	IˆEØ×"Ó"Ø×8Ñ8¸ÕHð	Iô 0ˆØ×ÑÐ!1°>Ð BÔCà×Ñ˜X |Ð4Ô5ØÐ*Ð*Ð*ùó? YùÛ`s   Æ:J9ÇJ9Ç!J?Ç.J?c                ób   — | j                   j                  dddœ«       t        | j                   fS )zGet Adafactor optimizer.F©Úscale_parameterÚrelative_step)r    rc   r   )Úctxs    r+   Ú_get_adafactorr{   Ã   s,   € à×Ñ×Ñ°EÈEÑ RÔSÜ�c×*Ñ*Ð*Ð*r*   c                óü   — ddl m} | j                  j                  | j                  «       | j
                  j                  t        j                  k(  r| j                  j                  ddi«       || j                  fS )z/Get PyTorch AdamW optimizer (regular or fused).r   ©ÚAdamWÚfusedT)	Útorch.optimr~   r    rc   r!   r   r=   r   ÚADAMW_TORCH_FUSED©rz   r~   s     r+   Ú_get_adamw_torchrƒ   É   s\   € å!à×Ñ×Ñ §¡Ô0Ø
‡x�x‡~�~œ×9Ñ9Ò9Ø×Ñ×#Ñ# W¨d OÔ4Ø�#×&Ñ&Ð&Ð&r*   c                ó¦   — 	 ddl m} | j                  j                  | j                  «       || j                  fS # t
        $ r t        d«      ‚w xY w)z'Get Torch XLA syncfree AdamW optimizer.r   r}   z7Trainer failed to import syncfree AdamW from torch_xla.)Útorch_xla.amp.syncfreer~   r    rc   r!   ÚImportErrorrT   r‚   s     r+   Ú_get_adamw_torch_xlar‡   Ó   sQ   € ðTÝ0à×Ñ×#Ñ# C§O¡OÔ4Ø�c×*Ñ*Ð*Ð*øÜò TÜÐRÓSÐSðTúó	   ‚8; »Ac                ó¦   — 	 ddl m} | j                  j                  | j                  «       || j                  fS # t
        $ r t        d«      ‚w xY w)zGet NPU Fused AdamW optimizer.r   )ÚNpuFusedAdamWz3Trainer failed to import FusedAdamW from torch_npu.)Útorch_npu.optimrŠ   r    rc   r!   r†   rT   )rz   rŠ   s     r+   Ú_get_adamw_torch_npu_fusedrŒ   Þ   sQ   € ðPÝ1à×Ñ×#Ñ# C§O¡OÔ4Ø˜c×2Ñ2Ð2Ð2øÜò PÜÐNÓOÐOðPúrˆ   c                óŒ  — t        «       st        d«      ‚ddlm}m}m} | j                  j                  }d|v }d|v rdnd}d}| j                  }d	|v r|}�n�d
|v r2|}d| j                  j                  | j                  j                  fi}�ngd|v r|}| j                  }�nSd|v �rNddlm}	 |	}t        | j                  j                  d| j                  j                  «      «      t        | j                  j                  d| j                  j                  «      «      t        | j                  j                  dd«      «      ft        | j                  j                  dd«      «      t        | j                  j                  d| j                  j                  «      «      dœ}d| j                  v rt!        | j                  d   «      |d<   d| j                  v rt!        | j                  d   «      |d<   d|i}
d|vr||
d<   | j"                  j%                  |«       | j"                  j%                  |
«       || j"                  fS )z;Get bitsandbytes optimizer (AdamW, Lion, RMSprop variants).ziYou need to install `bitsandbytes` in order to use bitsandbytes optimizers: `pip install -U bitsandbytes`r   )r~   ÚLionÚRMSpropÚpagedÚ8bité   é    NÚadamÚlionÚbetasÚrmspropÚademamix)ÚAdEMAMixÚbeta1Úbeta2Úbeta3g§èH.ÿï?Úalphag      @Úeps)r–   r�   rž   Út_alphaÚt_beta3Ú
optim_bitsÚis_paged)r   r†   Úbitsandbytes.optimr~   rŽ   r�   r   r=   r!   Ú
adam_beta1Ú
adam_beta2r#   r™   ÚfloatÚgetÚadam_epsilonÚintr    rc   )rz   r~   rŽ   r�   Ú
optim_namer¢   r¡   ri   Úadditional_optim_kwargsr™   Ú
bnb_kwargss              r+   Ú_get_bitsandbytes_optimizerr­   é   s  € ä$Ô&ÜØwó
ð 	
÷ 8Ñ7à—‘—‘€JØ˜*Ð$€HØ 
Ñ*‘°€JØ€MØ!Ÿo™oÐà�ÑØŠØ	�:Ñ	ØˆØ#*¨S¯X©X×-@Ñ-@À#Ç(Á(×BUÑBUÐ,VÐ"WÒØ	�jÑ	 ØˆØ"%§.¡.ÒØ	�zÒ	!Ý/à ˆô �c—n‘n×(Ñ(¨°#·(±(×2EÑ2EÓFÓGÜ�c—n‘n×(Ñ(¨°#·(±(×2EÑ2EÓFÓGÜ�c—n‘n×(Ñ(¨°&Ó9Ó:ðô
 ˜3Ÿ>™>×-Ñ-¨g°sÓ;Ó<Ü˜Ÿ™×+Ñ+¨E°3·8±8×3HÑ3HÓIÓJñ#
Ðð ˜Ÿ™Ñ&Ü14°S·^±^ÀIÑ5NÓ1OÐ# IÑ.Ø˜Ÿ™Ñ&Ü14°S·^±^ÀIÑ5NÓ1OÐ# IÑ.à 
Ð+€JØ˜
Ñ"Ø!)ˆ
�:Ñà×Ñ×ÑÐ 7Ô8Ø×Ñ×Ñ 
Ô+Ø˜#×.Ñ.Ð.Ð.r*   c                ó  — 	 ddl m} | j                  j                  | j                  «       | j                  j                  t        | j                  j                  dd«      «      t        t        | j                  j                  dd«      «      t        t        | j                  j                  dd«      «      t        t        | j                  j                  dd	«      «      d
œ«       || j                  fS # t        $ r t        d«      ‚w xY w)z!Get AnyPrecision AdamW optimizer.r   )ÚAnyPrecisionAdamWÚuse_kahan_summationÚFalseÚmomentum_dtypeÚfloat32Úvariance_dtypeÚcompensation_buffer_dtypeÚbfloat16)r°   r²   r´   rµ   z4Please install https://github.com/pytorch/torchdistx)Útorchdistx.optimizersr¯   r    rc   r!   r   r#   r§   Úgetattrr<   r†   rT   )rz   r¯   s     r+   Ú_get_adamw_anyprecisionr¹     sà   € ðQÝ;à×Ñ×#Ñ# C§O¡OÔ4Ø×Ñ×#Ñ#ä'0°·±×1CÑ1CÐDYÐ[bÓ1cÓ'dÜ")¬%°·±×1CÑ1CÐDTÐV_Ó1`Ó"aÜ")¬%°·±×1CÑ1CÐDTÐV_Ó1`Ó"aÜ-4Ü˜3Ÿ>™>×-Ñ-Ð.IÈ:ÓVó.ñ	ô		
ð ! #×"6Ñ"6Ð6Ð6øÜò QÜÐOÓPÐPðQús   ‚C3C6 Ã6Dc                óH  — | j                   j                  «       }| j                  radD ],  }|| j                  v sŒt        | j                  |   «      ||<   Œ. d| j                  v r"| j                  d   j	                  «       dv |d<   t
        j                  j                  |fS )zGet SGD optimizer.)ÚmomentumÚ	dampeningÚweight_decayÚnesterov©ÚtrueÚ1Úyes)r    Úcopyr#   r¦   rN   r<   r=   ÚSGD©rz   Úkwargsr5   s      r+   Ú_get_sgdrÇ   0  s’   € à×!Ñ!×&Ñ&Ó(€FØ
‡~‚~Ø<ò 	9ˆCØ�c—n‘nÒ$Ü# C§N¡N°3Ñ$7Ó8��s’ð	9ð ˜Ÿ™Ñ'Ø!$§¡°
Ñ!;×!AÑ!AÓ!CÐG[Ð![ˆF�:ÑÜ�;‰;�?‰?˜FÐ"Ð"r*   c                óè   — | j                   j                  «       }| j                  r1dD ],  }|| j                  v sŒt        | j                  |   «      ||<   Œ. t        j
                  j                  |fS )zGet Adagrad optimizer.)Úlr_decayr½   rž   )r    rÃ   r#   r¦   r<   r=   ÚAdagradrÅ   s      r+   Ú_get_adagradrË   <  sf   € à×!Ñ!×&Ñ&Ó(€FØ
‡~‚~Ø6ò 	9ˆCØ�c—n‘nÒ$Ü# C§N¡N°3Ñ$7Ó8��s’ð	9ô �;‰;×Ñ Ð&Ð&r*   c                óH  — | j                   j                  «       }| j                  radD ],  }|| j                  v sŒt        | j                  |   «      ||<   Œ. d| j                  v r"| j                  d   j	                  «       dv |d<   t
        j                  j                  |fS )zGet RMSprop optimizer.)r»   r�   rž   r½   Úcenteredr¿   )r    rÃ   r#   r¦   rN   r<   r=   r�   rÅ   s      r+   Ú_get_rmsproprÎ   F  s”   € à×!Ñ!×&Ñ&Ó(€FØ
‡~‚~Ø?ò 	9ˆCØ�c—n‘nÒ$Ü# C§N¡N°3Ñ$7Ó8��s’ð	9ð ˜Ÿ™Ñ'Ø!$§¡°
Ñ!;×!AÑ!AÓ!CÐG[Ð![ˆF�:ÑÜ�;‰;×Ñ Ð&Ð&r*   c                ó   — t        «       st        d«      ‚ddlm}m}m} t        j                  |t        j                  |t        j                  |t        j                  |t        j                  |t        j                  |i}t        | j                  j                  dd«      «      t        | j                  j                  dd«      «      t!        | j                  j                  dd	«      «      | j                  j                  d
d«      dœ}t#        | j$                  | j&                  | j$                  j(                  ||| j*                  «      \  }}| j$                  j(                  t        j                  k(  r|j-                  dddœ«       ||fS )zGet GaLore optimizer.z�You need to install `galore_torch` in order to use GaLore optimizers. Install it with `pip install git+https://github.com/jiaweizzhao/GaLore`r   )ÚGaLoreAdafactorÚGaLoreAdamWÚGaLoreAdamW8bitÚranké€   Úupdate_proj_gapéÈ   Úscaleg      Ð?Ú	proj_typeÚstd)rÓ   rÕ   r×   rØ   Frw   )r   r†   Úgalore_torchrÐ   rÑ   rÒ   r   ÚGALORE_ADAMWÚGALORE_ADAMW_8BITÚGALORE_ADAFACTORÚGALORE_ADAMW_LAYERWISEÚGALORE_ADAMW_8BIT_LAYERWISEÚGALORE_ADAFACTOR_LAYERWISEr©   r#   Úpopr¦   ru   r   r   r=   r    rc   )rz   rÐ   rÑ   rÒ   re   Úgalore_optim_kwargsri   r    s           r+   Ú_get_galore_optimizerrã   R  sJ  € ä$Ô&ÜðVó
ð 	
÷ KÑJô 	×#Ñ# [Ü×(Ñ(¨/Ü×'Ñ'¨Ü×-Ñ-¨{Ü×2Ñ2°OÜ×1Ñ1°?ðÐô �C—N‘N×&Ñ& v¨sÓ3Ó4Ü˜sŸ~™~×1Ñ1Ð2CÀSÓIÓJÜ�s—~‘~×)Ñ)¨'°4Ó8Ó9Ø—^‘^×'Ñ'¨°UÓ;ñ	Ðô '@Ø�‰�#—)‘)˜SŸX™XŸ^™^Ð->Ð@SÐUX×UiÑUió'Ñ#€MÐ#ð ‡x�x‡~�~œ×8Ñ8Ò8Ø×Ñ°EÈEÑ RÔSØÐ*Ð*Ð*r*   c           
     ó¶  — t        «       st        d«      ‚ddlm} t        j
                  |t        j                  |i}t        | j                  j                  dd«      «      | j                  j                  dd«      | j                  j                  dd	«      t        | j                  j                  d
d«      «      t        | j                  j                  dd«      «      | j                  j                  dd«      dœ}|j                  | j                  «       t        | j                  | j                  | j                  j                   ||| j"                  «      S )zGet Apollo optimizer.zŒYou need to install `apollo_torch` in order to use APOLLO optimizers. Install it with `pip install git+https://github.com/zhuhanqing/APOLLO`r   )ÚAPOLLOAdamWrÓ   rÔ   ÚprojÚrandomÚ
scale_typeÚchannelrÕ   rÖ   r×   ç      ð?rØ   rÙ   )rÓ   ræ   rè   rÕ   r×   rØ   )r   r†   Úapollo_torchrå   r   ÚAPOLLO_ADAMWÚAPOLLO_ADAMW_LAYERWISEr©   r#   rá   r¦   rc   r!   ru   r   r   r=   r    )rz   rå   re   Úapollo_optim_kwargss       r+   Ú_get_apollo_optimizerrï   s  s  € ä$Ô&ÜðUó
ð 	
õ )ô 	×#Ñ# [Ü×-Ñ-¨{ðÐô �C—N‘N×&Ñ& v¨sÓ3Ó4Ø—‘×"Ñ" 6¨8Ó4Ø—n‘n×(Ñ(¨°yÓAÜ˜sŸ~™~×1Ñ1Ð2CÀSÓIÓJÜ�s—~‘~×)Ñ)¨'°3Ó7Ó8Ø—^‘^×'Ñ'¨°UÓ;ñÐð ×Ñ˜sŸ™Ô/ä$Ø�‰�#—)‘)˜SŸX™XŸ^™^Ð->Ð@SÐUX×UiÑUióð r*   c                ó  — t        «       st        d«      ‚| j                  €t        d«      ‚ddlm}m} d| j                  j                  v r|n|}| j                  j                  d| j                  i«       || j                  fS )zGet LOMO optimizer.zjYou need to install `lomo_optim` in order to use LOMO optimizers. Install it with `pip install lomo-optim`zMYou need to pass a `model` in order to correctly initialize a LOMO optimizer.r   )ÚAdaLomoÚLomoÚadar   )r   r†   r   rT   Ú
lomo_optimrñ   rò   r   r=   r    rc   )rz   rñ   rò   ri   s       r+   Ú_get_lomo_optimizerrõ   �  sy   € äÔÜð7ó
ð 	
ð
 ‡y�yÐÜÐhÓiÐiç(à$¨¯©¯©Ñ6‘G¸D€MØ×Ñ×Ñ ¨#¯)©)Ð 4Ô5Ø˜#×.Ñ.Ð.Ð.r*   c                óô  — t        «       st        d«      ‚ddlm} | j                  j                  t        | j                  j                  dd«      «      t        | j                  j                  dd«      «      t        | j                  j                  dd	«      «      t        | j                  j                  d
d	«      «      t        | j                  j                  dd«      «      dœ«       || j                  fS )zGet GrokAdamW optimizer.z5Please install grokadamw with `pip install grokadamw`r   )Ú	GrokAdamWÚ
alpha_initg\�Âõ(\ï?Úlambç       @Úgammagš™™™™™¹?Úgrokking_signal_decay_rateÚgradient_clippingrê   )rø   rù   rû   rü   rý   )	r   rT   Ú	grokadamwr÷   r    rc   r¦   r#   r§   )rz   r÷   s     r+   Ú_get_grokadamwrÿ   ¢  sÃ   € ä!Ô#ÜÐPÓQÐQå#à×Ñ×Ñä §¡× 2Ñ 2°<ÀÓ FÓGÜ˜#Ÿ.™.×,Ñ,¨V°SÓ9Ó:Ü˜3Ÿ>™>×-Ñ-¨g°sÓ;Ó<Ü*/°·±×0BÑ0BÐC_ÐadÓ0eÓ*fÜ!& s§~¡~×'9Ñ'9Ð:MÈsÓ'SÓ!Tñ	
ôð �c×*Ñ*Ð*Ð*r*   c           	     óˆ  — t        «       rHt        j                  t        j                  j                  d«      «      t        j                  d«      k  rt        d«      ‚t        j                  t        j                  j                  d«      «      t        j                  d«      k  rt        d«      ‚t        j                  t        j                  j                  d«      «      t        j                  d«      k\  r	dd	lm}m} ndd	l	m}m} | j                  j                  t        j                  k(  r|}n|}| j                  j                  | j                   j#                  d
d«      t%        | j                   j#                  dd«      «      dœ«       | j                  j                  | j&                  «       || j                  fS )z%Get TorchAO 4-bit or 8-bit optimizer.Útorchaoz0.4.0z®You need to have `torchao>=0.4.0` in order to use torch 4-bit optimizers. Install it with `pip install torchao` or follow the instructions here: https://github.com/pytorch/aor<   z2.4z¹You need to have `torch>2.4` in order to use torch 4-bit optimizers. Install it with `pip install --upgrade torch` it is available on pipy. Otherwise, you need to install torch nightly.z0.11.0r   )Ú	AdamW4bitÚ	AdamW8bitÚ
block_sizeé   Úbf16_stochastic_roundr±   )r  r  )r   r   ÚparseÚ	importlibÚmetadatar†   Útorchao.optimr  r  Útorchao.prototype.low_bit_optimr   r=   r   ÚADAMW_TORCH_4BITr    rc   r#   r§   r   r!   )rz   r  r  ri   s       r+   Ú_get_torchao_optimizerr  µ  sP  € äÔ!¤W§]¡]´9×3EÑ3E×3MÑ3MÈiÓ3XÓ%YÔ\c×\iÑ\iÐjqÓ\rÒ%rÜð,ó
ð 	
ô
 ‡}�}”Y×'Ñ'×/Ñ/°Ó8Ó9¼W¿]¹]È5Ó=QÒQÜð<ó
ð 	
ô ‡}�}”Y×'Ñ'×/Ñ/°	Ó:Ó;¼w¿}¹}ÈXÓ?VÒVß6Ð6çHà
‡x�x‡~�~œ×8Ñ8Ò8Ø!‰à!ˆà×Ñ×ÑàŸ.™.×,Ñ,¨\¸3Ó?Ü%.¨s¯~©~×/AÑ/AÐBYÐ[bÓ/cÓ%dñ	
ôð ×Ñ×Ñ §¡Ô0Ø˜#×.Ñ.Ð.Ð.r*   c           	     ó0  — t        «       st        d«      ‚ddlm}m} i }d}| j
                  j                  t        j                  k(  r-t        d«      st        d«      ‚ddlm	} |}| j                  }d}nk| j
                  j                  t        j                  k(  r|}| j                  }n5| j
                  j                  t        j                  k(  r|}nt        d	«      ‚| j
                  j                  |d
<   |r| j
                  j                  |d<   |j!                  t#        | j$                  j'                  dd«      «      t#        | j$                  j'                  dd«      «      dœ«       | j(                  j!                  |«       || j(                  fS )zGet ScheduleFree optimizer.zwYou need to install `schedulefree` in order to use schedulefree optimizers. Install it with `pip install schedulefree.`r   )ÚAdamWScheduleFreeÚSGDScheduleFreeTz1.4.0z‚You need to install `schedulefree>=1.4.0` in order to use RAdamScheduleFree optimizer. Install it with `pip install schedulefree.`)ÚRAdamScheduleFreeFzInvalid schedulefree optimizerr½   Úwarmup_stepsÚweight_lr_powerrú   Úrg        )r  r  )r   r†   Úschedulefreer  r  r   r=   r   ÚSCHEDULE_FREE_RADAMr  r!   ÚSCHEDULE_FREE_ADAMWÚSCHEDULE_FREE_SGDrT   r½   r  rc   r¦   r#   r§   r    )rz   r  r  r«   Úrequire_warmupr  ri   s          r+   Ú_get_schedule_free_optimizerr  Ø  s[  € ä$Ô&Üð:ó
ð 	
÷ @à ÐØ€Nà
‡x�x‡~�~œ×;Ñ;Ò;Ü(¨Ô1Üð>óð õ 	3à)ˆØ"%§/¡/ÐØ‰Ø	�‰�‰œ>×=Ñ=Ò	=Ø)ˆØ"%§/¡/ÑØ	�‰�‰œ>×;Ñ;Ò	;Ø'‰äÐ9Ó:Ð:à.1¯h©h×.CÑ.CÐ˜NÑ+ÙØ25·(±(×2GÑ2GÐ Ñ/Ø×"Ñ"ä$ S§^¡^×%7Ñ%7Ð8IÈ3Ó%OÓPÜ�s—~‘~×)Ñ)¨#¨sÓ3Ó4ñ	
ôð ×Ñ×ÑÐ 7Ô8Ø˜#×.Ñ.Ð.Ð.r*   c                ó  — t        «       st        d«      ‚ddlm} | j                  j                  dd«      }|�t        |«      }| j                  j                  dd«      }|�t        |«      }| j                  j                  | j                  d<   t        | j                  j                  dd	«      «      ||d
œ}| j                  j                  | j                  «       | j                  j                  |«       || j                  fS )z,Get StableAdamW optimizer from torch-optimi.zwYou need to install `torch-optimi` in order to use stable_adamw optimizers. Install it with `pip install torch-optimi`.r   )ÚStableAdamWÚmax_lrNÚ	kahan_sumr½   Údecouple_lrF)r  r  r  )r   r†   Úoptimir  r#   rá   r¦   Úboolr   r½   r!   r    rc   )rz   r  r  r  Ústable_adamw_kwargss        r+   Ú_get_stable_adamwr#    sè   € ä$Ô&Üð:ó
ð 	
õ #à�^‰^×Ñ ¨$Ó/€FØÐÜ�v“ˆà—‘×"Ñ" ;°Ó5€IØÐÜ˜“Oˆ	à&)§h¡h×&;Ñ&;€C‡O�O�NÑ#ä˜CŸN™N×.Ñ.¨}¸eÓDÓEØØñÐð ×Ñ×Ñ §¡Ô0Ø×Ñ×ÑÐ 3Ô4Ø˜×,Ñ,Ð,Ð,r*   zdict[str, OptimizerHandler]Ú_OPTIMIZER_HANDLERS)r3   z
str | NoneÚreturnr"   )r?   r   r%  r!  )T)r   r   r   r   rd   rV   re   r   rf   r   r    r   rg   r!  r%  útuple[Any, dict[str, Any]])rz   r   r%  r&  )sr'   Ú
__future__r   Úimportlib.metadatar  ÚloggingÚcollections.abcr   Údataclassesr   Útypingr   r   r<   Ú	packagingr   r	   Úoptimizationr   Útrainer_pt_utilsr   Útrainer_utilsr   Útraining_argsr   r   Úutilsr   r   r   r   r   r   r   r   r   Úmodeling_utilsr   r   Ú	getLoggerr$   rZ   r   r7   ÚtupleÚdictrV   ÚOptimizerHandlerr@   ru   r{   rƒ   r‡   rŒ   r­   r¹   rÇ   rË   rÎ   rã   rï   rõ   rÿ   r  r  r#  Ú	ADAMW_BNBÚ
ADAMW_8BITÚPAGED_ADAMWÚPAGED_ADAMW_8BITÚADEMAMIXÚADEMAMIX_8BITÚPAGED_ADEMAMIXÚPAGED_ADEMAMIX_8BITÚLIONÚ	LION_8BITÚ
PAGED_LIONÚPAGED_LION_8BITÚRMSPROP_BNBÚRMSPROP_8BITÚRMSPROP_32BITÚ_BITSANDBYTES_OPTIMIZERSrÛ   rÜ   rÝ   rÞ   rß   rà   Ú_GALORE_OPTIMIZERSrì   rí   Ú_APOLLO_OPTIMIZERSr  ÚADAMW_TORCH_8BITÚ_TORCHAO_OPTIMIZERSr  r  r  Ú_SCHEDULE_FREE_OPTIMIZERSÚ	ADAFACTORÚADAMW_TORCHr�   ÚADAMW_TORCH_XLAÚADAMW_TORCH_NPU_FUSEDÚADAMW_ANYPRECISIONrÄ   ÚADAGRADÚRMSPROPÚ	GROKADAMWÚSTABLE_ADAMWÚLOMOÚADALOMOÚfromkeysr$  r(   r)   r*   r+   ú<module>rY     s¬  ðòõ #ã Û Ý $Ý !ß %ã Ý Ý å #Ý 5Ý 5ß 7÷
÷ 
õ 
ñ Ý/Ý0à	ˆ×	Ñ	˜8Ó	$€ð ÷ð ó ðóð Ð-Ð.°°c¸4ÀÀSÀ¹>Ð6IÑ0JÐJÑKÐ óð8 $(ðV+Ø
ðV+àðV+ð ðV+ð &ð	V+ð
 !ðV+ð %ðV+ð !ðV+ð  óV+ó|+ó'óTóPó//ódQó*	#ó'ó	'ó+óBó:/ó$+ó& /óF)/óX-ðD ×ÑØ×ÑØ×ÑØ×#Ñ#Ø×ÑØ× Ñ Ø×!Ñ!Ø×&Ñ&Ø×ÑØ×ÑØ×ÑØ×"Ñ"Ø×ÑØ×ÑØ× Ñ ðÐ ð& ×ÑØ×$Ñ$Ø×#Ñ#Ø×)Ñ)Ø×.Ñ.Ø×-Ñ-ðÐ ð ×ÑØ×)Ñ)ðÐ ð ×#Ñ#Ø×#Ñ#ðÐ ð ×&Ñ&Ø×&Ñ&Ø×$Ñ$ðÐ ð ×Ñ˜nØ×ÑÐ 0Ø×$Ñ$Ð&6Ø×"Ñ"Ð$8Ø×(Ñ(Ð*DØ×%Ñ%Ð'>Ø×Ñ˜Ø×Ñ˜LØ×Ñ˜LØ×Ñ˜nØ×ÑÐ!2Ø×ÑÐ,Ø×ÑÐ/ð4ð ‡m�mÐ,Ð.IÓJð4ð ‡m�mÐ&Ð(=Ó>ð4ð  ‡m�mÐ&Ð(=Ó>ð!4ð" ‡m�mÐ'Ð)?Ó@ð#4ð$ ‡m�mÐ-Ð/KÓLð%4Ð Ð0ô r*   