
    ^j                         d Z ddlZddlZddlZddlZddlZddlZddlmZm	Z	  ej                  e      Z G d d      Zy)z Checkpoint Saver

Track top-n training checkpoints and maintain recovery checkpoints on specified intervals.

Hacked together by / Copyright 2020 Ross Wightman
    N   )unwrap_modelget_state_dictc                   \    e Zd ZdddddddddedfdZd Zd	 Zdd
ZddZddZ	ddZ
d Zy)CheckpointSaverN
checkpointrecovery F
   c                    || _         || _        || _        || _        || _        || _        g | _        d | _        d | _        d| _	        d| _
        d| _        || _        |	| _        || _        || _        d| _        |
| _        |
rt$        j&                  nt$        j(                  | _        || _        || _        | j,                  dk\  sJ y )Nr
   Tz.pth.tarr   )modeltask	optimizerargs	model_ema
amp_scalercheckpoint_files
best_epochbest_metriccurr_recovery_fileprev_recovery_filecan_hardlinkcheckpoint_dirrecovery_dirsave_prefixrecovery_prefix	extension
decreasingoperatorltgtcmpmax_history	unwrap_fn)selfr   r   r   r   r   checkpoint_prefixr   r   r   r   r#   r$   r   s                 f/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/utils/checkpoint_saver.py__init__zCheckpointSaver.__init__   s    $ 
	"	"$ !#"$"$  -(,.#$",8;;(++&"1$$$    c                     | j                   r5	 t        j                  j                  |      rt        j                  |       t        j                  ||       y # t
        t        f$ r}d| _         Y d }~2d }~ww xY wNF)r   ospathexistsunlinkOSErrorNotImplementedErrorreplacer%   srcdstes       r'   _replacezCheckpointSaver._replaceD   s^    *77>>#&IIcN 	

3 01 *$)!!*s   4A A9(A44A9c                 &   | j                   rL	 t        j                  j                  |      rt        j                  |       t        j
                  ||       y t        j                  ||       y # t        t        f$ r}d| _         Y d }~2d }~ww xY wr+   )
r   r,   r-   r.   r/   linkr0   r1   shutilcopy2r3   s       r'   
_duplicatezCheckpointSaver._duplicateM   so    *77>>#&IIcNS! 	S# 01 *$)!!*s   A
A0 0B?BBc                 H   |t        | j                        j                  j                         | j                  j                         dd}| j                  (| j                  j                  |d<   | j                  |d<   | j                  1| j                  j                         || j                  j                  <   | j                  l|j                  | j                  j                  | j                               |j                  | j                  j                  d| j                               nRt        | j                  | j                        |d<   | j                  #t        | j                  | j                        |d	<   |||d
<   t        j                   ||       y )N   )epocharchr   versionr@   r   )r$   T)emar$   
state_dictstate_dict_emametric)typer   __name__lowerr   rC   r   r   state_dict_keyr   updateget_checkpoint_stater$   r   r   torchsave)r%   	save_pathr?   rE   
save_states        r'   _savezCheckpointSaver._saveY   sC   $--335224	

 99 !%Jv!%Jv??&9=9S9S9UJt55699 dii<<t~~<VWdii<<QUQ_Q_<`a'5djj$..'QJ|$~~)/=dnndnn/]
+,#)Jx 

:y)r)   c                    t        t        | j                        |      }| j                  |z
  }|dk  st        | j                        |k  ry | j                  |d  }|D ]?  }	 t        j                  dj                  |             t        j                  |d          A | j                  d | | _        y # t        $ r.}t        j                  dj                  |             Y d }~d }~ww xY w)Nr   zCleaning checkpoint: {}z(Exception '{}' while deleting checkpoint)minlenr   r#   _loggerdebugformatr,   remove	Exceptionerror)r%   trimdelete_index	to_deletedr6   s         r'   _cleanup_checkpointsz$CheckpointSaver._cleanup_checkpointsp   s    3t,,-t4''$.!s4#8#89\I)),-8	 	TAT7>>qAB		!A$	T !% 5 5m| D  THOOPQRSSTs   "<B55	C,>$C''C,c                 z   |dk\  sJ t         j                  j                  | j                  d| j                  z         }t         j                  j                  | j                  d| j                  z         }| j                  |||       | j                  ||       | j                  r| j                  d   nd }t        | j                        | j                  k  s|| j                  ||d         rt        | j                        | j                  k\  r| j                  d       dj                  | j                  t        |      g      | j                  z   }t         j                  j                  | j                  |      }| j                  ||       | j                  j                  ||f       t!        | j                  d | j"                         | _        d	}| j                  D ]  }	|d
j%                  |	      z  } t&        j)                  |       || j*                  | j                  || j*                        rW|| _        || _        t         j                  j                  | j                  d| j                  z         }
| j                  ||
       | j*                  dS | j*                  | j,                  fS )Nr   tmplastr   -c                     | d   S )Nr    )xs    r'   <lambda>z1CheckpointSaver.save_checkpoint.<locals>.<lambda>   s
    ad r)   )keyreversezCurrent checkpoints:
z {}

model_best)NN)r,   r-   joinr   r   rP   r7   r   rS   r#   r"   r^   r   strr<   appendsortedr   rV   rT   infor   r   )r%   r?   rE   tmp_save_pathlast_save_path
worst_filefilenamerN   checkpoints_strcbest_save_paths              r'   save_checkpointzCheckpointSaver.save_checkpoint~   s8   zzT%8%8%$..:PQd&9&96DNN;RS

=%0m^4262G2GT**2.T
%%&)9)99~xx
1.4(()T-=-==))!,xx!1!13u: >?$..PHT%8%8(CIOONI6!!(()V)<=$*%%" OO+%D! 7O** 57>>!#445LL)!t'7'7'?488FTXTdTdCe"'#) !#d.A.A<RVR`R`C`!a?#//7|`d>N>NPTP_P_=``r)   c                 >   |dk\  sJ t         j                  j                  | j                  d| j                  z         }| j                  ||       dj                  | j                  t        |      t        |      g      | j                  z   }t         j                  j                  | j                  |      }| j                  ||       t         j                  j                  | j                        rN	 t        j                  dj                  | j                               t        j                  | j                         | j"                  | _
        || _        y # t        $ r9}t        j!                  dj                  || j                               Y d }~Vd }~ww xY w)Nr   recovery_tmprc   zCleaning recovery: {}z Exception '{}' while removing {})r,   r-   rk   r   r   rP   r   rl   r7   r.   r   rT   rU   rV   rW   rX   rY   r   )r%   r?   	batch_idxrp   rs   rN   r6   s          r'   save_recoveryzCheckpointSaver.save_recovery   s1   zzT%6%68WX

=%(88T113u:s9~NORVR`R``GGLL!2!2H=	mY/77>>$112e5<<T=T=TUV		$112 #'"9"9"+  e@GG4KbKbcddes   4AE 	F#/FFc                     t         j                  j                  | j                  | j                        }t        j
                  |dz   | j                  z         }t        |      }t        |      r|d   S dS )N*r   r
   )	r,   r-   rk   r   r   globr   rn   rS   )r%   recovery_pathfiless      r'   find_recoveryzCheckpointSaver.find_recovery   s\    T%6%68L8LM		-#->?uu:uQx-2-r)   )N)r   )rG   
__module____qualname__r   r(   r7   r<   rP   r^   rw   r{   r   re   r)   r'   r   r      sQ    
 *&"+%Z
*.E%aN,$.r)   r   )__doc__r~   loggingr   r,   r:   rL   r   r   r   	getLoggerrG   rT   r   re   r)   r'   <module>r      sA       	   / '

H
%e. e.r)   