
    khf                     z    d Z ddlZddlZddlZddlZddlmZ ddlm	Z	  ej                  e      Z G d de	      Zy)z Cosine Scheduler

Cosine LR schedule with warmup, cycle/restarts, noise, k-decay.

Hacked together by / Copyright 2021 Ross Wightman
    N)List   )	Schedulerc                        e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddej
                  j                  dededededed	df fd
Z	ded	e
e   fdZddZ xZS )CosineLRSchedulera\  
    Cosine decay with restarts.
    This is described in the paper https://arxiv.org/abs/1608.03983.

    Inspiration from
    https://github.com/allenai/allennlp/blob/master/allennlp/training/learning_rate_schedulers/cosine.py

    k-decay option based on `k-decay: A New Method For Learning Rate Schedule` - https://arxiv.org/abs/2004.05909
    N	optimizer	t_initiallr_min	cycle_mulcycle_decaycycle_limitreturnc           
         t         |   |d|
|||||       |dkD  sJ |dk\  sJ |dk(  r|dk(  r|dk(  rt        j                  d       || _        || _        || _        || _        || _        || _	        || _
        |	| _        || _        | j                  rI| j                  D cg c]  }||z
  | j                  z   c}| _        t         | =  | j                         y | j                  D cg c]  }d c}| _        y c c}w c c}w )Nlr)param_group_fieldt_in_epochsnoise_range_t	noise_pct	noise_std
noise_seed
initializer   r   zjCosine annealing scheduler will have no effect on the learning rate since t_initial = t_mul = eta_mul = 1.)super__init___loggerwarningr	   r
   r   r   r   warmup_twarmup_lr_initwarmup_prefixk_decaybase_valueswarmup_stepsupdate_groups)selfr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   v_	__class__s                      T/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/scheduler/cosine_lr.pyr   zCosineLRScheduler.__init__   s   & 	"#'!! 	 		
 1}}{{>i1n1AOO>? #"&& ,*==OSO_O_ `!!n"4!E `DG!$"5"56,0,<,< =q =D !a !>s   $C?-	Dtc                 (   || j                   k  r+| j                  D cg c]  }| j                  ||z  z    }}|S | j                  r|| j                   z
  }| j                  dk7  rt        j                  t        j                  d|| j                  z  d| j                  z
  z  z
  | j                              }| j                  |z  | j                  z  }|d| j                  |z  z
  d| j                  z
  z  | j                  z  z
  }n-|| j                  z  }| j                  }|| j                  |z  z
  }| j                  |z  }| j                  D cg c]  }||z  	 }	}| j                  }
|| j                  k  ra|	D cg c]T  }| j                  d|| j                  z
  z  dt        j                  t
        j                  ||
z  z  ||
z  z        z   z  z   V }}|S | j                  D cg c]  }| j                   }}|S c c}w c c}w c c}w c c}w )Nr   g      ?)r   r!   r   r   r   mathfloorlogr	   r   r    r   r   r
   cospi)r#   r(   slrsit_it_currgammar$   lr_max_valuesklr_maxr%   s                r'   _get_lrzCosineLRScheduler._get_lrQ   s   t}}8<8I8IJ14&&Q.JCJ4 
1 !!%~~"JJtxxA,>!dnnBT,U(UW[WeWefgnn)DNN:a$..A"55!dnn:LMPTP^P^^^'nndnnq01$$)E040@0@A1QYAMAA4### #0 KK#$++)=">!dhhtwwY_cdYdOdgjnogoOoFpBp"qq  
 -1,<,<=qt{{==
5 K B
 >s   H H>AH
)Hc                 F   t        d|xs | j                        }| j                  dk(  r| j                  |z  }nLt	        t        j                  | j                   | j                  |z  dz
  z  d| j                  z
  z              }| j                  r|| j                  z   S |S )Nr         ?)	maxr   r   r	   intr*   r+   r   r   )r#   cyclesr(   s      r'   get_cycle_lengthz"CosineLRScheduler.get_cycle_lengtho   s    Q2$"2"23>>S 'ADJJ$..F2JQ2NOSTW[WeWeSefghA$($6$6q4== =A=    )g        r:   r:   r   r   r   FTNgq=
ףp?r:   *   r:   T)r   )__name__
__module____qualname____doc__torchoptim	Optimizerr<   floatr   r   r8   r>   __classcell__)r&   s   @r'   r   r      s     !!# #1>{{,,1> 1> 	1>
 1> 1> 1>$ 
%1>f e <>r?   r   )rD   loggingr*   numpynprE   typingr   	schedulerr   	getLoggerrA   r   r    r?   r'   <module>rQ      s@           '

H
%b>	 b>r?   