
    ^j                         d Z ddlZddlZddlZddlZddlmZmZm	Z	 ddl
mZ  ej                  e      Z G d de      Zy)z Cosine Scheduler

Cosine LR schedule with warmup, cycle/restarts, noise, k-decay.

Hacked together by / Copyright 2021 Ross Wightman
    N)TupleListUnion   )	Schedulerc            #            e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddej
                  j                  dededededed	ed
ede	de	de
ee   eeef   edf   dedededede	ddf" fdZdedee   fdZddZ xZS )CosineLRSchedulera\  
    Cosine decay with restarts.
    This is described in the paper https://arxiv.org/abs/1608.03983.

    Inspiration from
    https://github.com/allenai/allennlp/blob/master/allennlp/training/learning_rate_schedulers/cosine.py

    k-decay option based on `k-decay: A New Method For Learning Rate Schedule` - https://arxiv.org/abs/2004.05909
    N	optimizer	t_initiallr_min	cycle_mulcycle_decaycycle_limitwarmup_twarmup_lr_initwarmup_prefixt_in_epochsnoise_range_t	noise_pct	noise_std
noise_seedk_decay
initializereturnc           
         t         |   |d|
|||||       |dkD  sJ |dk\  sJ |dk(  r|dk(  r|dk(  rt        j                  d       || _        || _        || _        || _        || _        || _	        || _
        |	| _        || _        | j                  rI| j                  D cg c]  }||z
  | j                  z   c}| _        t         | =  | j                         y | j                  D cg c]  }d c}| _        y c c}w c c}w )Nlr)param_group_fieldr   r   r   r   r   r   r   r   zjCosine annealing scheduler will have no effect on the learning rate since t_initial = t_mul = eta_mul = 1.)super__init___loggerwarningr   r   r   r   r   r   r   r   r   base_valueswarmup_stepsupdate_groups)selfr
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   v_	__class__s                      c/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/scheduler/cosine_lr.pyr   zCosineLRScheduler.__init__   s   & 	"#'!! 	 		
 1}}{{>i1n1AOO>? #"&& ,*==OSO_O_ `!!n"4!E `DG!$"5"56,0,<,< =q =D !a !>s   $C?-	Dtc                 (   || j                   k  r+| j                  D cg c]  }| j                  ||z  z    }}|S | j                  r|| j                   z
  }| j                  dk7  rt        j                  t        j                  d|| j                  z  d| j                  z
  z  z
  | j                              }| j                  |z  | j                  z  }|d| j                  |z  z
  d| j                  z
  z  | j                  z  z
  }n-|| j                  z  }| j                  }|| j                  |z  z
  }| j                  |z  }| j                  D cg c]  }||z  	 }	}| j                  }
|| j                  k  ra|	D cg c]T  }| j                  d|| j                  z
  z  dt        j                  t
        j                  ||
z  z  ||
z  z        z   z  z   V }}|S | j                  D cg c]  }| j                   }}|S c c}w c c}w c c}w c c}w )Nr   g      ?)r   r#   r   r   r   mathfloorlogr   r   r"   r   r   r   cospi)r%   r*   slrsit_it_currgammar&   lr_max_valuesklr_maxr'   s                r)   _get_lrzCosineLRScheduler._get_lrQ   s   t}}8<8I8IJ14&&Q.JCJ4 
1 !!%~~"JJtxxA,>!dnnBT,U(UW[WeWefgnn)DNN:a$..A"55!dnn:LMPTP^P^^^'nndnnq01$$)E040@0@A1QYAMAA4### #0 KK#$++)=">!dhhtwwY_cdYdOdgjnogoOoFpBp"qq  
 -1,<,<=qt{{==
5 K B
 >s   H H>AH
)Hc                 F   t        d|xs | j                        }| j                  dk(  r| j                  |z  }nLt	        t        j                  | j                   | j                  |z  dz
  z  d| j                  z
  z              }| j                  r|| j                  z   S |S )Nr         ?)	maxr   r   r   intr,   r-   r   r   )r%   cyclesr*   s      r)   get_cycle_lengthz"CosineLRScheduler.get_cycle_lengtho   s    Q2$"2"23>>S 'ADJJ$..F2JQ2NOSTW[WeWeSefghA$($6$6q4== =A=    )        r<   r<   r   r   rB   FTNgq=
ףp?r<   *   r<   T)r   )__name__
__module____qualname____doc__torchoptim	Optimizerr>   floatboolr   r   r   r   r:   r@   __classcell__)r(   s   @r)   r	   r	      s"    !!# $&"' $JN#"  ##1>{{,,1> 1> 	1>
 1> 1> 1> 1> "1>  1> 1> !cE#s(OS$!FG1> 1> 1> 1>  !1>" #1>$ 
%1>f e <>rA   r	   )rG   loggingr,   numpynprH   typingr   r   r   	schedulerr   	getLoggerrD   r    r	    rA   r)   <module>rU      sC        % %   '

H
%b>	 b>rA   