Ë
    éÓ^j2'  ã                   ó6   — d dl Z d dlZd dlmZ  G d„ de«      Zy)é    N)Ú	Optimizerc                   óž   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 dˆ fd„	Zˆ fd„Z ej                  «       d„ «       Z ej                  «       dd„«       Z	ˆ xZ
S )Ú	AdaBeliefa‹  Implements AdaBelief algorithm. Modified from Adam in PyTorch

    Arguments:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-16)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        amsgrad (boolean, optional): whether to use the AMSGrad variant of this
            algorithm from the paper `On the Convergence of Adam and Beyond`_
            (default: False)
        decoupled_decay (boolean, optional): (default: True) If set as True, then
            the optimizer uses decoupled weight decay as in AdamW
        fixed_decay (boolean, optional): (default: False) This is used when weight_decouple
            is set as True.
            When fixed_decay == True, the weight decay is performed as
            $W_{new} = W_{old} - W_{old} \times decay$.
            When fixed_decay == False, the weight decay is performed as
            $W_{new} = W_{old} - W_{old} \times decay \times lr$. Note that in this case, the
            weight decay ratio decreases with learning rate (lr).
        rectify (boolean, optional): (default: True) If set as True, then perform the rectified
            update similar to RAdam
        degenerated_to_sgd (boolean, optional) (default:True) If set as True, then perform SGD update
            when variance of gradient is high
    reference: AdaBelief Optimizer, adapting stepsizes by the belief in observed gradients, NeurIPS 2020

    For a complete table of recommended hyperparameters, see https://github.com/juntang-zhuang/Adabelief-Optimizer'
    For example train/args for EfficientNet see these gists
      - link to train_script: https://gist.github.com/juntang-zhuang/0a501dd51c02278d952cf159bc233037
      - link to args.yaml: https://gist.github.com/juntang-zhuang/517ce3c27022b908bb93f78e4f786dc3
    c                 óÊ  •— d|k  st        dj                  |«      «      ‚d|k  st        dj                  |«      «      ‚d|d   cxk  rdk  sn t        dj                  |d   «      «      ‚d|d   cxk  rdk  sn t        dj                  |d   «      «      ‚t        |t        t        f«      rht        |«      dkD  rZt        |d   t        «      rG|D ]B  }d	|v sŒ|d	   d   |d   k7  s|d	   d   |d   k7  sŒ%t        d
«      D �cg c]  }g d¢‘Œ c}|d<   ŒD t        ||||||
||	|t        d
«      D �cg c]  }g d¢‘Œ c}¬«
      }t        t        | �+  ||«       y c c}w c c}w )Ng        zInvalid learning rate: {}zInvalid epsilon value: {}r   ç      ð?z%Invalid beta parameter at index 0: {}é   z%Invalid beta parameter at index 1: {}Úbetasé
   )NNNÚbuffer)
Úlrr	   ÚepsÚweight_decayÚamsgradÚdegenerated_to_sgdÚdecoupled_decayÚrectifyÚfixed_decayr   )Ú
ValueErrorÚformatÚ
isinstanceÚlistÚtupleÚlenÚdictÚrangeÚsuperr   Ú__init__)ÚselfÚparamsr   r	   r   r   r   r   r   r   r   ÚparamÚ_ÚdefaultsÚ	__class__s                 €ú_/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/optim/adabelief.pyr   zAdaBelief.__init__*   s{  ø€ ð �bŠyÜÐ8×?Ñ?ÀÓCÓDÐDØ�cŠzÜÐ8×?Ñ?ÀÓDÓEÐEØ�e˜A‘hÔ$ Ô$ÜÐD×KÑKÈEÐRSÉHÓUÓVÐVØ�e˜A‘hÔ$ Ô$ÜÐD×KÑKÈEÐRSÉHÓUÓVÐVä�fœt¤U˜mÔ,´°V³¸q²ÄZÐPVÐWXÑPYÔ[_ÔE`Øò N�Ø˜eÒ#¨¨w©¸Ñ):¸eÀA¹hÒ)FÈ%ÐPWÉ.ÐYZÑJ[Ð_dÐefÑ_gÓJgÜCHÈÃ9Ö&M¸aÔ'9Ò&M�E˜(’OðNô ØØØØ%ØØ1Ø+ØØ#Ü05°b³	Ö:¨1Ô&Ò:ô
ˆô 	Œi˜Ñ'¨°Õ9ùò 'Nùò ;s   ÄEÄ3E c                 ór   •— t         t        | �  |«       | j                  D ]  }|j	                  dd«       Œ y )Nr   F)r   r   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroupr#   s      €r$   r&   zAdaBelief.__setstate__S   s7   ø€ ÜŒi˜Ñ+¨EÔ2Ø×&Ñ&ò 	/ˆEØ×Ñ˜Y¨Õ.ñ	/ó    c                 ó  — | j                   D ]p  }|d   D ]f  }| j                  |   }|d   }d|d<   t        j                  |«      |d<   t        j                  |«      |d<   |sŒOt        j                  |«      |d<   Œh Œr y )Nr   r   r   ÚstepÚexp_avgÚexp_avg_varÚmax_exp_avg_var)r'   r)   ÚtorchÚ
zeros_like)r   r*   Úpr)   r   s        r$   ÚresetzAdaBelief.resetX   s•   € à×&Ñ&ò 	CˆEØ˜8‘_ò C�ØŸ
™
 1™�Ø 	Ñ*�ð !"��f‘ä#(×#3Ñ#3°AÓ#6��iÑ ô (-×'7Ñ'7¸Ó':��mÑ$Úä/4×/?Ñ/?ÀÓ/B�EÐ+Ò,ñCñ	Cr+   c           	      ó$	  — d}|�$t        j                  «       5   |«       }ddd«       | j                  D �]J  }|d   D �]>  }|j                  €Œ|j                  }|j                  t         j
                  t         j                  hv r|j                  «       }|j                  rt        d«      ‚|}|j                  t         j
                  t         j                  hv r|j                  «       }|d   }|d   \  }}	| j                  |   }
t        |
«      dk(  rOd|
d<   t        j                  |«      |
d<   t        j                  |«      |
d	<   |rt        j                  |«      |
d
<   |d   r;|d   s|j                  d|d   |d   z  z
  «       n6|j                  d|d   z
  «       n|d   dk7  r|j                  ||d   ¬«       |
d   |
d	   }}|
dxx   dz  cc<   d||
d   z  z
  }d|	|
d   z  z
  }|j                  |«      j                  |d|z
  ¬«       ||z
  }|j                  |	«      j                  ||d|	z
  ¬«       |rh|
d
   }t        j                   ||j                  |d   «      |¬«       |j#                  «       t%        j"                  |«      z  j                  |d   «      }nJ|j                  |d   «      j#                  «       t%        j"                  |«      z  j                  |d   «      }|d   s|d   |z  }|j'                  ||| ¬«       �n&|d   t)        |
d   dz  «         }|
d   |d   k(  r|d   |d   }}n—|
d   |d<   |	|
d   z  }dd|	z
  z  dz
  }|d|
d   z  |z  d|z
  z  z
  }||d<   |dk\  rCt%        j"                  d|z
  |dz
  z  |dz
  z  |dz
  z  |z  |z  |dz
  z  «      d||
d   z  z
  z  }n|d   rdd||
d   z  z
  z  }nd}||d<   |dk\  r>|j#                  «       j                  |d   «      }|j'                  ||| |d   z  ¬«       n|dkD  r|j                  || |d   z  ¬«       |j                  t         j
                  t         j                  hv s�Œ.|j+                  |«       �ŒA �ŒM |S # 1 sw Y   �ŒfxY w)zµPerforms a single optimization step.
        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   zOAdaBelief does not support sparse gradients, please consider SparseAdam insteadr   r	   r   r-   r.   r/   r0   r   r   r   r   r   )Úalphar   )Úvaluer   )Úoutr   r   r
   é   é   é   r   éÿÿÿÿ)r1   Úenable_gradr'   ÚgradÚdtypeÚfloat16Úbfloat16ÚfloatÚ	is_sparseÚRuntimeErrorr)   r   r2   Úmul_Úadd_Úaddcmul_ÚmaxÚsqrtÚmathÚaddcdiv_ÚintÚcopy_)r   ÚclosureÚlossr*   r3   r>   Úp_fp32r   Úbeta1Úbeta2r)   r.   r/   Úbias_correction1Úbias_correction2Úgrad_residualr0   ÚdenomÚ	step_sizeÚbufferedÚnum_smaÚbeta2_tÚnum_sma_maxs                          r$   r-   zAdaBelief.stepj   s  € ð ˆØÐÜ×"Ñ"Ó$ñ !Ù“y�÷!ð ×&Ñ&ó b	$ˆEØ˜8‘_ó a$�Ø—6‘6�>ØØ—v‘v�Ø—:‘:¤%§-¡-´·±Ð!@Ñ@ØŸ:™:›<�DØ—>’>Ü&Øiókð kð �Ø—7‘7œuŸ}™}¬e¯n©nÐ=Ñ=Ø#Ÿ\™\›^�Fà 	Ñ*�Ø$ W™~‘��uØŸ
™
 1™�ä�u“: ’?Ø$%�E˜&‘Mä',×'7Ñ'7¸Ó'?�E˜)Ñ$ä+0×+;Ñ+;¸FÓ+C�E˜-Ñ(Ùä38×3CÑ3CÀFÓ3K˜Ð/Ñ0ð Ð*Ò+Ø  Ò/ØŸ™ C¨%°©+¸¸nÑ8MÑ*MÑ$MÕNàŸ™ C¨%°Ñ*?Ñ$?Õ@à˜^Ñ,°Ò1ØŸ	™	 &°°nÑ0E˜	ÔFð (-¨YÑ'7¸¸}Ñ9M˜�à�f“ Ñ"“Ø#$ u°°f±Ñ'=Ñ#=Ð Ø#$ u°°f±Ñ'=Ñ#=Ð ð —‘˜UÓ#×(Ñ(¨°Q¸±YÐ(Ô?Ø $ w¡�Ø× Ñ  Ó'×0Ñ0°ÀÐUVÐY^ÑU^Ð0Ô_áØ&+Ð,=Ñ&>�Oä—I‘I˜o¨{×/?Ñ/?ÀÀeÁÓ/MÐSbÕcð -×1Ñ1Ó3´d·i±iÐ@PÓ6QÑQ×WÑWÐX]Ð^cÑXdÓe‘Eà(×-Ñ-¨e°E©lÓ;×@Ñ@ÓBÄTÇYÁYÐO_ÓE`Ñ`×fÑfÐglÐmrÑgsÓt�Eð ˜YÒ'à % d¡Ð.>Ñ >�IØ—O‘O G¨U¸9¸*�OÖEð  % X™¬s°5¸±=À2Ñ3EÓ/FÑG�HØ˜V‘}¨°©Ò3Ø-5°a©[¸(À1¹+ ™à&+¨F¡m˜ ™Ø"'¨5°©=Ñ"8˜Ø&'¨1¨u©9¡o¸Ñ&9˜Ø"-°°E¸&±MÑ0AÀGÑ0KÈqÐSZÉ{Ñ0[Ñ"[˜Ø&-˜ ™ð # aš<Ü(,¯	©	Ø!" W¡Ø!(¨1¡ñ!.Ø1<¸q±ñ!Bà!(¨1¡ñ!.à07ñ!8ð !,ñ!,ð 0;¸Q©ñ!@ó)Að EFÈÐQVÐW]ÑQ^ÑH^ÑD^ñ	)`™Ið
 #Ð#7Ò8Ø(+¨q°5¸EÀ&¹MÑ3IÑ/IÑ(J™Ià(*˜IØ&/˜ ™à !’|Ø +× 0Ñ 0Ó 2× 7Ñ 7¸¸e¹Ó E˜ØŸ™¨°¸y¸jÈ5ÐQUÉ;Ñ>V˜ÕWØ" QšØŸ™ G°I°:ÀÀdÁÑ3K˜ÔLà—7‘7œuŸ}™}¬e¯n©nÐ=Ó=Ø—G‘G˜F–OòCa$ðb	$ðH ˆ÷O!ñ !ús   ™RÒR)	gü©ñÒMbP?)gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?g¼‰Ø—²Òœ<r   FTFTT)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r&   r1   Úno_gradr4   r-   Ú__classcell__)r#   s   @r$   r   r      sm   ø„ ñ!ðL ØØØØØ ØØØ#õ':ôR/ð
 €U‡]�]ƒ_ñCó ðCð" €U‡]�]ƒ_òoó ôor+   r   )rJ   r1   Útorch.optim.optimizerr   r   © r+   r$   ú<module>rd      s   ðÛ Û Ý +ôT�	õ Tr+   