Ë
    éÓ^jµ'  ã                   óv  — d Z ddlZddlZddlZddlZddlmZ ddlmZmZm	Z	m
Z
mZmZ ddlZddlmZ ddlmZ ddlmZ dd	lmZ  ej.                  e«      Zd
Z G d„ dej6                  «      Z G d„ d«      Zdej<                  de	de
e   fd„Z 	 	 	 	 ddee	   dee!   deeeee
f      de!fd„Z" G d„ de«      Z#y)aS   A dataset reader that reads tarfile based datasets

This reader can extract image samples from:
* a single tar of image files
* a folder of multiple tarfiles containing imagefiles
* a tar of tars containing image files

Labels are based on the combined folder and/or tar name structure.

Hacked together by / Copyright 2020 Ross Wightman
é    N)Úglob)ÚListÚTupleÚDictÚSetÚOptionalÚUnion)Únatural_keyé   )Úload_class_map)Úget_img_extensions)ÚReaderz_tarinfos.picklec                   ó2   ‡ — e Zd ZdZdhZdedefˆ fd„Zˆ xZS )Ú_TarInfoUnpicklera±  Restricted unpickler for the tarinfo cache file.

    The cache only ever contains plain containers (dict/list/str/int/None) plus
    ``tarfile.TarInfo`` instances. Built-in containers are handled by pickle opcodes
    and never reach ``find_class``, so allowing only ``tarfile.TarInfo`` is sufficient
    to load valid caches while blocking arbitrary code execution from a crafted cache
    file (CWE-502, GHSA-pqww-w348-m542).
    )ÚtarfileÚTarInfoÚmoduleÚnamec                 ó|   •— ||f| j                   v rt        ‰| �	  ||«      S t        j                  d|› d|› d�«      ‚)NzGlobal ú.z+ is not permitted in a tar info cache file.)Ú_allowed_globalsÚsuperÚ
find_classÚpickleÚUnpicklingError)Úselfr   r   Ú	__class__s      €úp/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/data/readers/reader_image_in_tar.pyr   z_TarInfoUnpickler.find_class*   sH   ø€ Ø�Dˆ>˜T×2Ñ2Ñ2Ü‘7Ñ% f¨dÓ3Ð3Ü×$Ñ$ w¨v¨h°a¸°vÐ=hÐ%iÓjÐjó    )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Ústrr   Ú__classcell__©r   s   @r   r   r      s/   ø„ ñð /Ð/Ððk ð k¨C÷ kñ kr   r   c                   óL   — e Zd Zddej                  dej
                  fd„Zd„ Zy)ÚTarStateNÚtfÚtic                 ó.   — || _         || _        i | _        y ©N)r)   r*   Úchildren)r   r)   r*   s      r   Ú__init__zTarState.__init__2   s   € Ø#%ˆŒØ#%ˆŒØ-/ˆ�r   c                 ó   — d | _         y r,   )r)   ©r   s    r   ÚresetzTarState.reset7   s	   € Øˆ�r   )NN)r    r!   r"   r   ÚTarFiler   r.   r1   © r   r   r(   r(   0   s"   „ ñ0˜7Ÿ?™?ð 0°w·±ó 0ó
r   r(   r)   Úparent_infoÚ
extensionsc                 óþ  — d}t        | «      D �]^  \  }}|j                  «       sŒt        j                  j	                  |j                  «      \  }}t        j                  j                  |«      \  }}	|	j                  «       }	|	dk(  rÅt        j                  | j                  |«      d¬«      5 }
t        |j                  t        j                  j                  |d   |«      |g g ¬«      }|t        |
||¬«      z  }t        j                  |› d|j                  › d	t!        |d
   «      › d�«       |d   j#                  |«       d d d «       �Œ@|	|v s�ŒF|d
   j#                  |«       |dz  }�Œa |S # 1 sw Y   �ŒnxY w)Nr   ú.tarúr|)ÚfileobjÚmodeÚpath©r   r;   r*   r-   Úsamples©r5   z"/?. Extracted child tarinfos from ú. r=   z images.r-   r   )Ú	enumerateÚisfileÚosr;   ÚsplitÚsplitextÚlowerr   ÚopenÚextractfileÚdictr   ÚjoinÚ_extract_tarinfoÚ_loggerÚdebugÚlenÚappend)r)   r4   r5   Úsample_countÚir*   ÚdirnameÚbasenamer   ÚextÚctfÚ
child_infos               r   rJ   rJ   ;   s\  € Ø€LÜ˜2“ó ‰ˆˆ2Ø�y‰yŒ{ØÜŸG™GŸM™M¨"¯'©'Ó2Ñˆ�Ü—G‘G×$Ñ$ XÓ.‰	ˆˆcØ�i‰i‹kˆØ�&Š=Ü—‘ b§n¡n°RÓ&8¸tÔDð ;ÈÜ!ØŸ™¤r§w¡w§|¡|°KÀÑ4GÈÓ'NÐSUÐ`bÐlnôp�
àÔ 0°°jÈZÔ XÑX�Ü—‘  Ð#EÀbÇgÁgÀYÈbÔQTÐU_Ð`iÑUjÓQkÐPlÐltÐuÔvØ˜JÑ'×.Ñ.¨zÔ:÷;ñ ;ð �JÓØ˜	Ñ"×)Ñ)¨"Ô-Ø˜AÑŠLðð  Ð÷;ñ ;ús   Â/BE2Å2E<	Úclass_name_to_idxÚcache_tarinfoÚsortc                 ó’  ‡‡$‡%‡&‡'— |st        d¬«      n
t        |«      }d}t        j                  j	                  | «      rt        j                  j                  | «      d   j                  «       dk(  sJ ‚| g}t        j                  j                  | «      \  } }t        j                  j                  |«      d   }d}n~| j                  t        j                  j                  «      j                  t        j                  j                  «      d   }t        t        j                  j                  | d«      d¬«      }t        |«      }t        |D �	cg c]!  }	t        j                  j                  |	«      ‘Œ# c}	«      }
|sJ d	| › d
�«       ‚t        j!                  d|
dz  d›d�«       t#        g ¬«      }d}|€	|
dkD  rdnd}|r,d|z   t$        z   }t        j                  j                  | |«      }t        j                  j'                  |«      ržt        j!                  d|› d�«       t)        |d«      5 }t+        |«      j-                  «       }d d d «       t/        |t"        «      rt/        |j1                  d«      t2        «      st5        d|› d�«      ‚t        |d   «      |k(  �sEJ d«       ‚t7        |«      D ]ê  \  }}|rdn>t        j                  j                  t        j                  j9                  |«      «      d   }t;        j(                  |d¬«      5 }t#        t        j                  j=                  || «      |d g g ¬«      }t?        |||¬«      }t        |d   «      }t        jA                  |› d|› d|› d |› d!|› d"�
«       d d d «       |d   jC                  «       Œì |rDt        j!                  d#|› d�«       t)        |d$«      5 }tE        jF                  ||«       d d d «       g Š'g Š&dŠ%‰€dŠ%g }dd%œd&„Š$ˆ$ˆ%ˆˆ&ˆ'fd'„}t        j!                  d(«       |d   D ]t  }|rd n|d)   }tI        «       }d}|d   D ]2  } |||¬*«      }|rtI        |d+   ¬,«      |jJ                  |d)   <   ||z  }Œ4 | |||¬*«      z  }|sŒb|jC                  ||f«       Œv ~‰%rBt3        tM        t        ‰&«      tN        ¬-«      «      }t7        |«      D ��ci c]  \  }}||“Œ
 c}}Št        j!                  d.«       tQ        ‰'‰&«      D � �!cg c]  \  } }!|!‰v sŒ| ‰|!   f‘Œ }"} }!|rtM        |"d/„ ¬-«      }"tQ        |"Ž \  Š'}#tS        jT                  ‰'«      Š'tS        jT                  |#«      }#t        j!                  d0t        ‰'«      › d1t        |«      › d2�«       ‰'|#‰|fS c c}	w # 1 sw Y   �ŒKxY w# 1 sw Y   �ŒxY w# 1 sw Y   �ŒÐxY wc c}}w c c}!} w )3NT)Úas_setFéÿÿÿÿr7   r   z*.tar)Ú	recursivez'No .tar files found at specified path (z).z	Scanning i   z.2fzMB of tar files...)ÚtartreesÚ l       
 Ú_z!Reading tar info from cache file r   Úrbr]   z'Invalid or corrupt tar info cache file z3Cached tartree len doesn't match number of tarfilesr8   )r:   r<   r>   r-   ú/z. Extracted tarinfos from r?   z children, z	 samples.zWriting tar info to cache file Úwb)Ú	leaf_onlyc                 ó8  — t        j                  j                  |Ž j                  t         j                  j                  «      }| r,|j                  t         j                  j                  «      d   S |j                  t         j                  j                  d«      S )Nr[   r_   )rB   r;   rI   ÚstripÚseprC   Úreplace)rc   r;   s     r   Ú_label_from_pathsz+extract_tarinfos.<locals>._label_from_paths�   s^   € Ü�w‰w�|‰|˜TÐ"×(Ñ(¬¯©¯©Ó5ˆÙ.7ˆt�z‰zœ"Ÿ'™'Ÿ+™+Ó& rÑ*Ð[¸T¿\¹\Ì"Ï'É'Ï+É+ÐWZÓ=[Ð[r   c                 óî   •— d}| d   D ]i  } ‰| d   t         j                  j                  |j                  «      «      }‰s|‰vrŒ=‰	j                  ||| d   f«       ‰j                  |«       |dz  }Œk |S )Nr   r=   r;   r*   r   )rB   r;   rQ   rN   )
ÚinfoÚfnÚaddedÚsÚlabelrh   Úbuild_class_maprV   Úlabelsr=   s
        €€€€€r   Ú_add_samplesz&extract_tarinfos.<locals>._add_samples”   s}   ø€ ØˆØ�i‘ò 	ˆAÙ% d¨6¡l´B·G±G·O±OÀAÇFÁFÓ4KÓLˆEÙ" uÐ4EÑ'EØØ�N‰N˜A˜r 4¨¡:Ð.Ô/Ø�M‰M˜%Ô Ø�Q‰J‰Eð	ð ˆr   z+Collecting samples and building tar states.r   )rk   r*   )r*   )Úkeyz$Mapping targets and sorting samples.c                 ó8   — t        | d   d   j                  «      S ©Nr   )r
   r;   )Úks    r   ú<lambda>z"extract_tarinfos.<locals>.<lambda>·   s   € ÌÐTUÐVWÑTXÐYZÑT[×T`ÑT`ÓHa€ r   zFinished processing z samples across z tar files.)+r   ÚsetrB   r;   rA   rD   rE   rC   re   rf   r   rI   rM   ÚsumÚgetsizerK   rj   rH   ÚCACHE_FILENAME_SUFFIXÚexistsrF   r   ÚloadÚ
isinstanceÚgetÚlistÚ
ValueErrorr@   rR   r   ÚrelpathrJ   rL   rN   r   Údumpr(   r-   Úsortedr
   ÚzipÚnpÚarray)(ÚrootrV   rW   r5   rX   Úroot_is_tarÚtar_filenamesÚ	root_nameÚnum_tarsÚfÚ	tar_bytesrj   Ú
cache_pathÚcache_filenameÚpfrP   rk   r;   r)   r4   Únum_samplesÚnum_childrenÚtarfilesrq   Útar_nameÚ	tar_stateÚparent_addedrU   Úchild_addedÚsorted_labelsÚidxÚcrm   ÚlÚsamples_and_targetsÚtargetsrh   ro   rp   r=   s(    `                                  @@@@r   Úextract_tarinfosrž   P   s\  ü€ ñ 9CÔ#¨4Õ0ÌÈJË€JØ€KÜ	‡w�w‡~�~�dÔÜ�w‰w×Ñ Ó% bÑ)×/Ñ/Ó1°VÒ;Ð;Ð;Ø˜ˆÜŸ'™'Ÿ-™-¨Ó-‰ˆˆiÜ—G‘G×$Ñ$ YÓ/°Ñ2ˆ	Ø‰à—J‘JœrŸw™wŸ{™{Ó+×1Ñ1´"·'±'·+±+Ó>¸rÑBˆ	ÜœRŸW™WŸ\™\¨$°Ó8ÀDÔIˆÜ�=Ó!€HÜ°Ö?¨A”R—W‘W—_‘_ QÕ'Ò?Ó@€IÙÐGÐ>¸t¸fÀBÐGÓGˆ8ä‡L�L�9˜Y wÑ.¨sÐ3Ð3EÐFÔGÜ˜Ô€DØ€JØÐØ )¨JÒ 6™¸EˆÙØ˜y™Ô+@Ñ@ˆÜ—W‘W—\‘\ $¨Ó7ˆ
Ü	‡w�w‡~�~�jÔ!Ü�‰Ð8¸¸ÀAÐFÔGÜ�*˜dÓ#ð 	0 rÜ$ RÓ(×-Ñ-Ó/ˆD÷	0ä˜$¤Ô%¬Z¸¿¹ÀÓ8LÌdÔ-SÜÐFÀzÀlÐRSÐTÓUÐUÜ�4˜
Ñ#Ó$¨Ó0ÐgÐ2gÓgÐ0ä˜}Ó-ò 	1‰EˆAˆrÙ$‘2¬"¯'©'×*:Ñ*:¼2¿7¹7×;KÑ;KÈBÓ;OÓ*PÐQRÑ*SˆDÜ—‘˜b tÔ,ð v°Ü"¬¯©¯©¸¸DÓ(AÈÐQUÐ`bÐlnÔo�Ü.¨r°;È:ÔV�Ü" ;¨zÑ#:Ó;�Ü—‘Ø�c˜˜8˜*Ð$>¸r¸dÀ"À\ÀNÐR]Ð^iÐ]jÐjsÐtôv÷	vð �Ñ×#Ñ# KÕ0ð	1ñ Ü�L‰LÐ:¸:¸,ÀaÐHÔIÜ�j $Ó'ð &¨2Ü—‘˜D "Ô%÷&ð €GØ€FØ€OØÐ Øˆð €Hà+/ô \÷	ð 	ô ‡L�LÐ>Ô@Ø˜JÑ'ò 3ˆá&‘4¨K¸Ñ,?ˆÜ“Jˆ	ØˆØ% jÑ1ò 	(ˆJÙ& z°hÔ?ˆKÙÜ9AÀZÐPTÑEUÔ9V�	×"Ñ" :¨fÑ#5Ñ6Ø˜KÑ'‰Lð		(ð
 	™ [°XÔ>Ñ>ˆÚØ�O‰O˜X yÐ1Õ2ð3ð 	áäœV¤C¨£K´[ÔAÓBˆÜ2;¸MÓ2J×K©¨¨Q˜Q ™VÓKÐä‡L�LÐ7Ô9ÜADÀWÈfÓAU×p¹¸¸AÐYZÐ^oÒYo˜AÐ0°Ñ3Ò4ÐpÐÑpÙÜ$Ð%8Ñ>aÔbÐÜÐ/Ð0Ñ€GˆWÜ�h‰h�wÓ€GÜ�h‰h�wÓ€GÜ‡L�LÐ'¬¨G« ~Ð5EÄcÈ(ÃmÀ_ÐT_Ð`ÔaØ�GÐ.°Ð8Ð8ùòs @÷	0ñ 	0ú÷vñ vú÷&ñ &üób Lùó qs=   Å&VÈ3VÌA/V#ÏV0Ó
V=Ó?WÔ	WÖV Ö#V-	Ö0V:c                   ó8   ‡ — e Zd ZdZdˆ fd„	Zd„ Zd„ Zdd„Zˆ xZS )ÚReaderImageInTarzI Multi-tarfile dataset reader where there is one .tar file per class
    c                 óº  •— t         ‰	| �  «        d }|rt        ||«      }|| _        t	        | j                  ||¬«      \  | _        | _        | _        }| j                  j                  «       D ��ci c]  \  }}||“Œ
 c}}| _	        t        |«      dk(  r$|d   d   €d| _        |d   d   | _        || _        y d| _        t        |«      | _        || _        y c c}}w )N)rV   rW   r   r   TF)r   r.   r   r‡   rž   r=   r�   rV   ÚitemsÚclass_idx_to_namerM   rˆ   r•   rH   Úcache_tarfiles)
r   r‡   Ú	class_mapr¤   rW   rV   r“   ru   Úvr   s
            €r   r.   zReaderImageInTar.__init__Ã   sÜ   ø€ Ü‰ÑÔà ÐÙÜ .¨y¸$Ó ?ÐØˆŒ	ÜGWØ�I‰IØ/Ø'ôH
ÑDˆŒ�d”l DÔ$:¸Hð
 48×3IÑ3I×3OÑ3OÓ3Q×!R©4¨1¨a ! Q¡$Ó!RˆÔÜˆx‹=˜AÒ (¨1¡+¨a¡.Ð"8Ø#ˆDÔØ% a™[¨™^ˆDŒNð -ˆÕð  %ˆDÔÜ! (›^ˆDŒNØ,ˆÕùó "Ss   Á0Cc                 ó,   — t        | j                  «      S r,   )rM   r=   r0   s    r   Ú__len__zReaderImageInTar.__len__Ø   s   € Ü�4—<‘<Ó Ð r   c                 óÆ  — | j                   |   }| j                  |   }|\  }}}|r*t        j                  j	                  | j
                  |«      n| j
                  }d }d }	| j                  r3| j                  r| j                  n| j                  |   }	|	j                  }|€(t        j                  |«      }| j                  r||	_	        |�„| j                  r#|	j                  |j                     j                  nd }
|
€Ot        j                  |j                  |«      ¬«      }
| j                  r|
|	j                  |j                     _	        |
}|j                  |«      |fS )N)r9   )r=   r�   rB   r;   rI   r‡   r¤   rˆ   r•   r)   r   rF   r-   r   rG   )r   ÚindexÚsampleÚtargetÚ	sample_tiÚ	parent_fnÚchild_tiÚ
parent_absr)   Úcache_staterT   s              r   Ú__getitem__zReaderImageInTar.__getitem__Û   s&  € Ø—‘˜eÑ$ˆØ—‘˜eÑ$ˆØ)/Ñ&ˆ	�9˜hÙ;D”R—W‘W—\‘\ $§)¡)¨YÔ7È$Ï)É)ˆ
àˆØˆØ×ÒØ,0×,<Ò,<˜$Ÿ.š.À$Ç.Á.ÐQZÑB[ˆKØ—‘ˆBØˆ:Ü—‘˜jÓ)ˆBØ×"Ò"Ø!#�”ØÐØ<@×<OÒ<O�+×&Ñ& x§}¡}Ñ5×8Ò8ÐUYˆCØˆ{Ü—l‘l¨2¯>©>¸(Ó+CÔD�Ø×&Ò&Ø=@�K×(Ñ(¨¯©Ñ7Ô:ØˆBà�~‰~˜iÓ(¨&Ð0Ð0r   c                 ó€   — | j                   |   d   j                  }|rt        j                  j	                  |«      }|S rt   )r=   r   rB   r;   rR   )r   rª   rR   ÚabsoluteÚfilenames        r   Ú	_filenamezReaderImageInTar._filenameô   s7   € Ø—<‘< Ñ& qÑ)×.Ñ.ˆÙÜ—w‘w×'Ñ'¨Ó1ˆHØˆr   )r^   TN)FF)	r    r!   r"   r#   r.   r¨   r²   r¶   r%   r&   s   @r   r    r    ¿   s   ø„ ñõ-ò*!ò1÷2r   r    )NNNT)$r#   ÚloggingrB   r   r   r   Útypingr   r   r   r   r   r	   Únumpyr…   Útimm.utils.miscr
   r¥   r   Úimg_extensionsr   Úreaderr   Ú	getLoggerr    rK   rz   Ú	Unpicklerr   r(   r2   r$   rJ   Úboolrž   r    r3   r   r   ú<module>rÀ      sì   ðñ
ó Û 	Û Û Ý ß :× :ã å 'å %Ý .Ý à
ˆ'×
Ñ
˜HÓ
%€Ø*Ð ôk˜×(Ñ(ô k÷"ñ ð˜Ÿ™ð °tð ÈÈSÉó ð. -1Ø(,Ø8<Øñl9à# D™>ðl9ð   ‘~ðl9ð ˜U 4¨°Ð#3Ñ4Ñ5ð	l9ð
 ól9ô^9�võ 9r   