
    kh                         d Z ddlZddlZddlZddlZddlZddlZddlmZm	Z	  ej                  e      Z G d d      Zy)z Checkpoint Saver

Track top-n training checkpoints and maintain recovery checkpoints on specified intervals.

Hacked together by / Copyright 2020 Ross Wightman
    N   )unwrap_modelget_state_dictc            
       Z    e Zd Zdddddddddef
dZd Zd	 Zdd
ZddZddZ	ddZ
d Zy)CheckpointSaverN
checkpointrecovery F
   c                 ~   || _         || _        || _        || _        || _        g | _        d | _        d | _        d| _        d| _	        d| _
        || _        |	| _        || _        || _        d| _        |
| _        |
rt"        j$                  nt"        j&                  | _        || _        || _        | j*                  dk\  sJ y )Nr
   Tz.pth.tarr   )model	optimizerargs	model_ema
amp_scalercheckpoint_files
best_epochbest_metriccurr_recovery_fileprev_recovery_filecan_hardlinkcheckpoint_dirrecovery_dirsave_prefixrecovery_prefix	extension
decreasingoperatorltgtcmpmax_history	unwrap_fn)selfr   r   r   r   r   checkpoint_prefixr   r   r   r   r"   r#   s                W/var/www/teggl/fontify/venv/lib/python3.12/site-packages/timm/utils/checkpoint_saver.py__init__zCheckpointSaver.__init__   s    " 
"	"$ !#"$"$  -(,.#$",8;;(++&"1$$$    c                     | j                   r5	 t        j                  j                  |      rt        j                  |       t        j                  ||       y # t
        t        f$ r}d| _         Y d }~2d }~ww xY wNF)r   ospathexistsunlinkOSErrorNotImplementedErrorreplacer$   srcdstes       r&   _replacezCheckpointSaver._replaceB   s^    *77>>#&IIcN 	

3 01 *$)!!*s   4A A9(A44A9c                 &   | j                   rL	 t        j                  j                  |      rt        j                  |       t        j
                  ||       y t        j                  ||       y # t        t        f$ r}d| _         Y d }~2d }~ww xY wr*   )
r   r+   r,   r-   r.   linkr/   r0   shutilcopy2r2   s       r&   
_duplicatezCheckpointSaver._duplicateK   so    *77>>#&IIcNS! 	S# 01 *$)!!*s   A
A0 0B?BBc                 P   |t        | j                        j                  j                         t	        | j                  | j
                        | j                  j                         dd}| j                  (| j                  j                  |d<   | j                  |d<   | j                  1| j                  j                         || j                  j                  <   | j                  #t	        | j                  | j
                        |d<   |||d<   t        j                  ||       y )N   )epocharch
state_dictr   versionr?   r   state_dict_emametric)typer   __name__lowerr   r#   r   r@   r   r   state_dict_keyr   torchsave)r$   	save_pathr>   rC   
save_states        r&   _savezCheckpointSaver._saveW   s    $--335(T^^D224

 99 !%Jv!%Jv??&9=9S9S9UJt556>>%+9$..$..+YJ'(#)Jx 

:y)r(   c                    t        t        | j                        |      }| j                  |z
  }|dk  st        | j                        |k  ry | j                  |d  }|D ]?  }	 t        j                  dj                  |             t        j                  |d          A | j                  d | | _        y # t        $ r.}t        j                  dj                  |             Y d }~d }~ww xY w)Nr   zCleaning checkpoint: {}z(Exception '{}' while deleting checkpoint)minlenr   r"   _loggerdebugformatr+   remove	Exceptionerror)r$   trimdelete_index	to_deletedr5   s         r&   _cleanup_checkpointsz$CheckpointSaver._cleanup_checkpointsj   s    3t,,-t4''$.!s4#8#89\I)),-8	 	TAT7>>qAB		!A$	T !% 5 5m| D  THOOPQRSSTs   "<B55	C,>$C''C,c                 z   |dk\  sJ t         j                  j                  | j                  d| j                  z         }t         j                  j                  | j                  d| j                  z         }| j                  |||       | j                  ||       | j                  r| j                  d   nd }t        | j                        | j                  k  s|| j                  ||d         rt        | j                        | j                  k\  r| j                  d       dj                  | j                  t        |      g      | j                  z   }t         j                  j                  | j                  |      }| j                  ||       | j                  j                  ||f       t!        | j                  d | j"                         | _        d	}| j                  D ]  }	|d
j%                  |	      z  } t&        j)                  |       || j*                  | j                  || j*                        rW|| _        || _        t         j                  j                  | j                  d| j                  z         }
| j                  ||
       | j*                  dS | j*                  | j,                  fS )Nr   tmplastr   -c                     | d   S )Nr    )xs    r&   <lambda>z1CheckpointSaver.save_checkpoint.<locals>.<lambda>   s
    ad r(   )keyreversezCurrent checkpoints:
z {}

model_best)NN)r+   r,   joinr   r   rL   r6   r   rO   r"   r!   rZ   r   strr;   appendsortedr   rR   rP   infor   r   )r$   r>   rC   tmp_save_pathlast_save_path
worst_filefilenamerJ   checkpoints_strcbest_save_paths              r&   save_checkpointzCheckpointSaver.save_checkpointx   s8   zzT%8%8%$..:PQd&9&96DNN;RS

=%0m^4262G2GT**2.T
%%&)9)99~xx
1.4(()T-=-==))!,xx!1!13u: >?$..PHT%8%8(CIOONI6!!(()V)<=$*%%" OO+%D! 7O** 57>>!#445LL)!t'7'7'?488FTXTdTdCe"'#) !#d.A.A<RVR`R`C`!a?#//7|`d>N>NPTP_P_=``r(   c                 >   |dk\  sJ t         j                  j                  | j                  d| j                  z         }| j                  ||       dj                  | j                  t        |      t        |      g      | j                  z   }t         j                  j                  | j                  |      }| j                  ||       t         j                  j                  | j                        rN	 t        j                  dj                  | j                               t        j                  | j                         | j"                  | _
        || _        y # t        $ r9}t        j!                  dj                  || j                               Y d }~Vd }~ww xY w)Nr   recovery_tmpr_   zCleaning recovery: {}z Exception '{}' while removing {})r+   r,   rg   r   r   rL   r   rh   r6   r-   r   rP   rQ   rR   rS   rT   rU   r   )r$   r>   	batch_idxrl   ro   rJ   r5   s          r&   save_recoveryzCheckpointSaver.save_recovery   s1   zzT%6%68WX

=%(88T113u:s9~NORVR`R``GGLL!2!2H=	mY/77>>$112e5<<T=T=TUV		$112 #'"9"9"+  e@GG4KbKbcddes   4AE 	F#/FFc                     t         j                  j                  | j                  | j                        }t        j
                  |dz   | j                  z         }t        |      }t        |      r|d   S dS )N*r   r
   )	r+   r,   rg   r   r   globr   rj   rO   )r$   recovery_pathfiless      r&   find_recoveryzCheckpointSaver.find_recovery   s\    T%6%68L8LM		-#->?uu:uQx-2-r(   )N)r   )rE   
__module____qualname__r   r'   r6   r;   rL   rZ   rs   rw   r}   ra   r(   r&   r   r      sN    
 *&")%V
*&E%aN,$.r(   r   )__doc__rz   loggingr   r+   r9   rH   r   r   r   	getLoggerrE   rP   r   ra   r(   r&   <module>r      sA       	   / '

H
%_. _.r(   