Ë
    éÓ^j  ã                   ó€   — d Z ddlZddlZddlZddlZddlZddlZddlmZm	Z	  ej                  e«      Z G d„ d«      Zy)z¡ Checkpoint Saver

Track top-n training checkpoints and maintain recovery checkpoints on specified intervals.

Hacked together by / Copyright 2020 Ross Wightman
é    Né   )Úunwrap_modelÚget_state_dictc                   ó\   — e Zd Zdddddddddedfd„Zd„ Zd	„ Zdd
„Zdd„Zdd„Z	dd„Z
d„ Zy)ÚCheckpointSaverNÚ
checkpointÚrecoveryÚ Fé
   c                 óŒ  — || _         || _        || _        || _        || _        || _        g | _        d | _        d | _        d| _	        d| _
        d| _        || _        |	| _        || _        || _        d| _        |
| _        |
rt$        j&                  nt$        j(                  | _        || _        || _        | j,                  dk\  sJ ‚y )Nr
   Tz.pth.tarr   )ÚmodelÚtaskÚ	optimizerÚargsÚ	model_emaÚ
amp_scalerÚcheckpoint_filesÚ
best_epochÚbest_metricÚcurr_recovery_fileÚprev_recovery_fileÚcan_hardlinkÚcheckpoint_dirÚrecovery_dirÚsave_prefixÚrecovery_prefixÚ	extensionÚ
decreasingÚoperatorÚltÚgtÚcmpÚmax_historyÚ	unwrap_fn)Úselfr   r   r   r   r   Úcheckpoint_prefixr   r   r   r   r#   r$   r   s                 úf/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/utils/checkpoint_saver.pyÚ__init__zCheckpointSaver.__init__   sÅ   € ð$ ˆŒ
ØˆŒ	Ø"ˆŒØˆŒ	Ø"ˆŒØ$ˆŒð !#ˆÔØˆŒØˆÔØ"$ˆÔØ"$ˆÔØ ˆÔð -ˆÔØ(ˆÔØ,ˆÔØ.ˆÔØ#ˆŒØ$ˆŒÙ",”8—;’;´(·+±+ˆŒØ&ˆÔØ"ˆŒØ×Ñ 1Ò$Ð$Ñ$ó    c                 óø   — | j                   r5	 t        j                  j                  |«      rt        j                  |«       t        j                  ||«       y # t
        t        f$ r}d| _         Y d }~Œ2d }~ww xY w©NF)r   ÚosÚpathÚexistsÚunlinkÚOSErrorÚNotImplementedErrorÚreplace©r%   ÚsrcÚdstÚes       r'   Ú_replacezCheckpointSaver._replaceD   s^   € Ø×Òð*Ü—7‘7—>‘> #Ô&Ü—I‘I˜c”Nô 	�
‰
�3˜Õøô Ô0Ð1ò *Ø$)�×!Ñ!ûð*ús   Ž4A ÁA9Á(A4Á4A9c                 ó&  — | j                   rL	 t        j                  j                  |«      rt        j                  |«       t        j
                  ||«       y t        j                  ||«       y # t        t        f$ r}d| _         Y d }~Œ2d }~ww xY wr+   )
r   r,   r-   r.   r/   Úlinkr0   r1   ÚshutilÚcopy2r3   s       r'   Ú
_duplicatezCheckpointSaver._duplicateM   so   € Ø×Òð*Ü—7‘7—>‘> #Ô&ä—I‘I˜c”NÜ—‘˜˜SÔ!Øô 	�‰�S˜#Õøô Ô0Ð1ò *Ø$)�×!Ñ!ûð*ús   ŽA
A0 Á0BÁ?BÂBc                 óH  — |t        | j                  «      j                  j                  «       | j                  j                  «       ddœ}| j                  �(| j                  j                  |d<   | j                  |d<   | j                  �1| j                  j                  «       || j                  j                  <   | j                  �l|j                  | j                  j                  | j                  ¬«      «       |j                  | j                  j                  d| j                  ¬«      «       nRt        | j                  | j                  «      |d<   | j                  �#t        | j                  | j                  «      |d	<   |�||d
<   t        j                   ||«       y )Né   )ÚepochÚarchr   Úversionr@   r   )r$   T)Úemar$   Ú
state_dictÚstate_dict_emaÚmetric)Útyper   Ú__name__Úlowerr   rC   r   r   Ústate_dict_keyr   ÚupdateÚget_checkpoint_stater$   r   r   ÚtorchÚsave)r%   Ú	save_pathr?   rE   Ú
save_states        r'   Ú_savezCheckpointSaver._saveY   sC  € àÜ˜Ÿ™Ó$×-Ñ-×3Ñ3Ó5ØŸ™×2Ñ2Ó4Øñ	
ˆ
ð �9‰9Ð Ø!%§¡§¡ˆJ�vÑØ!%§¡ˆJ�vÑØ�?‰?Ð&Ø9=¿¹×9SÑ9SÓ9UˆJ�t—‘×5Ñ5Ñ6Ø�9‰9Ð Ø×Ñ˜dŸi™i×<Ñ<ÀtÇ~Á~Ð<ÓVÔWØ×Ñ˜dŸi™i×<Ñ<ÀÐQU×Q_ÑQ_Ð<Ó`Õaä'5°d·j±jÀ$Ç.Á.Ó'QˆJ�|Ñ$Ø�~‰~Ð)Ü/=¸d¿n¹nÈdÏnÉnÓ/]�
Ð+Ñ,ØÐØ#)ˆJ�xÑ Ü�
‰
�:˜yÕ)r)   c                 óÞ  — t        t        | j                  «      |«      }| j                  |z
  }|dk  st        | j                  «      |k  ry | j                  |d  }|D ]?  }	 t        j                  dj                  |«      «       t        j                  |d   «       ŒA | j                  d | | _        y # t        $ r.}t        j                  dj                  |«      «       Y d }~Œ‰d }~ww xY w)Nr   zCleaning checkpoint: {}z(Exception '{}' while deleting checkpoint)ÚminÚlenr   r#   Ú_loggerÚdebugÚformatr,   ÚremoveÚ	ExceptionÚerror)r%   ÚtrimÚdelete_indexÚ	to_deleteÚdr6   s         r'   Ú_cleanup_checkpointsz$CheckpointSaver._cleanup_checkpointsp   sÝ   € Ü”3�t×,Ñ,Ó-¨tÓ4ˆØ×'Ñ'¨$Ñ.ˆØ˜!Òœs 4×#8Ñ#8Ó9¸\ÒIØØ×)Ñ)¨,¨-Ð8ˆ	Øò 	TˆAðTÜ—‘Ð7×>Ñ>¸qÓAÔBÜ—	‘	˜!˜A™$•ð	Tð !%× 5Ñ 5°m°|Ð DˆÕøô ò TÜ—‘ÐH×OÑOÐPQÓR×SÑSûðTús   Á"<B5Â5	C,Â>$C'Ã'C,c                 óz  — |dk\  sJ ‚t         j                  j                  | j                  d| j                  z   «      }t         j                  j                  | j                  d| j                  z   «      }| j                  |||«       | j                  ||«       | j                  r| j                  d   nd }t        | j                  «      | j                  k  s|�| j                  ||d   «      �r¥t        | j                  «      | j                  k\  r| j                  d«       dj                  | j                  t        |«      g«      | j                  z   }t         j                  j                  | j                  |«      }| j                  ||«       | j                  j                  ||f«       t!        | j                  d„ | j"                   ¬«      | _        d	}| j                  D ]  }	|d
j%                  |	«      z  }Œ t&        j)                  |«       |�| j*                  �| j                  || j*                  «      rW|| _        || _        t         j                  j                  | j                  d| j                  z   «      }
| j                  ||
«       | j*                  €dS | j*                  | j,                  fS )Nr   ÚtmpÚlastéÿÿÿÿr   ú-c                 ó   — | d   S )Nr   © )Úxs    r'   ú<lambda>z1CheckpointSaver.save_checkpoint.<locals>.<lambda>”   s
   € ˜a ™d€ r)   )ÚkeyÚreversezCurrent checkpoints:
z {}
Ú
model_best)NN)r,   r-   Újoinr   r   rP   r7   r   rS   r#   r"   r^   r   Ústrr<   ÚappendÚsortedr   rV   rT   Úinfor   r   )r%   r?   rE   Útmp_save_pathÚlast_save_pathÚ
worst_fileÚfilenamerN   Úcheckpoints_strÚcÚbest_save_paths              r'   Úsave_checkpointzCheckpointSaver.save_checkpoint~   s8  € Ø˜ŠzÐˆzÜŸ™Ÿ™ T×%8Ñ%8¸%À$Ç.Á.Ñ:PÓQˆÜŸ™Ÿ™ d×&9Ñ&9¸6ÀDÇNÁNÑ;RÓSˆØ�
‰
�= %¨Ô0Ø�‰�m ^Ô4à26×2GÒ2G�T×*Ñ*¨2Ò.ÈTˆ
ä�×%Ñ%Ó&¨×)9Ñ)9Ò9Øˆ~Ø�x‰x˜ 
¨1¡Õ.ä�4×(Ñ(Ó)¨T×-=Ñ-=Ò=Ø×)Ñ)¨!Ô,Ø—x‘x ×!1Ñ!1´3°u³:Ð >Ó?À$Ç.Á.ÑPˆHÜŸ™Ÿ™ T×%8Ñ%8¸(ÓCˆIØ�O‰O˜N¨IÔ6à×!Ñ!×(Ñ(¨)°VÐ)<Ô=Ü$*Ø×%Ñ%Ù"Ø ŸO™OÐ+ô%ˆDÔ!ð 7ˆOØ×*Ñ*ò 5�Ø 7§>¡>°!Ó#4Ñ4‘ð5ä�L‰L˜Ô)àÐ! t×'7Ñ'7Ð'?À4Ç8Á8ÈFÐTX×TdÑTdÔCeØ"'�”Ø#)�Ô Ü!#§¡§¡¨d×.AÑ.AÀ<ÐRV×R`ÑR`ÑC`Ó!a�Ø—‘ °Ô?à#×/Ñ/Ð7ˆ|Ð`¸d×>NÑ>NÐPT×P_ÑP_Ð=`Ð`r)   c                 ó>  — |dk\  sJ ‚t         j                  j                  | j                  d| j                  z   «      }| j                  ||«       dj                  | j                  t        |«      t        |«      g«      | j                  z   }t         j                  j                  | j                  |«      }| j                  ||«       t         j                  j                  | j                  «      rN	 t        j                  dj                  | j                  «      «       t        j                  | j                  «       | j"                  | _
        || _        y # t        $ r9}t        j!                  dj                  || j                  «      «       Y d }~ŒVd }~ww xY w)Nr   Úrecovery_tmprc   zCleaning recovery: {}z Exception '{}' while removing {})r,   r-   rk   r   r   rP   r   rl   r7   r.   r   rT   rU   rV   rW   rX   rY   r   )r%   r?   Ú	batch_idxrp   rs   rN   r6   s          r'   Úsave_recoveryzCheckpointSaver.save_recovery¥   s1  € Ø˜ŠzÐˆzÜŸ™Ÿ™ T×%6Ñ%6¸ÈÏÉÑ8WÓXˆØ�
‰
�= %Ô(à—8‘8˜T×1Ñ1´3°u³:¼sÀ9»~ÐNÓOÐRV×R`ÑR`Ñ`ˆÜ—G‘G—L‘L ×!2Ñ!2°HÓ=ˆ	Ø�‰�m YÔ/ä�7‰7�>‰>˜$×1Ñ1Ô2ðeÜ—‘Ð5×<Ñ<¸T×=TÑ=TÓUÔVÜ—	‘	˜$×1Ñ1Ô2ð #'×"9Ñ"9ˆÔØ"+ˆÕøô ò eÜ—‘Ð@×GÑGÈÈ4×KbÑKbÓc×dÑdûðeús   Ã4AE Å	FÅ#/FÆFc                 óî   — t         j                  j                  | j                  | j                  «      }t        j
                  |dz   | j                  z   «      }t        |«      }t        |«      r|d   S dS )NÚ*r   r
   )	r,   r-   rk   r   r   Úglobr   rn   rS   )r%   Úrecovery_pathÚfiless      r'   Úfind_recoveryzCheckpointSaver.find_recovery·   s\   € ÜŸ™Ÿ™ T×%6Ñ%6¸×8LÑ8LÓMˆÜ—	‘	˜-¨#Ñ-°·±Ñ>Ó?ˆÜ�u“ˆÜ˜uœ:ˆu�Q‰xÐ-¨2Ð-r)   )N)r   )rG   Ú
__module__Ú__qualname__r   r(   r7   r<   rP   r^   rw   r{   r�   re   r)   r'   r   r      sQ   „ ð
 ØØØ*Ø&ØØØØØ"Øó+%òZò
ó*ó.Eó%aóN,ó$.r)   r   )Ú__doc__r~   Úloggingr   r,   r:   rL   r   r   r   Ú	getLoggerrG   rT   r   re   r)   r'   ú<module>r‡      sA   ðñó Û Û Û 	Û ã ç /ð ˆ'×
Ñ
˜HÓ
%€÷e.ò e.r)   