
    ^j'                     v   d Z ddlZddlZddlZddlZddlmZ ddlmZmZm	Z	m
Z
mZmZ ddlZddlmZ ddlmZ ddlmZ dd	lmZ  ej.                  e      Zd
Z G d dej6                        Z G d d      Zdej<                  de	de
e   fdZ 	 	 	 	 ddee	   dee!   deeeee
f      de!fdZ" G d de      Z#y)aS   A dataset reader that reads tarfile based datasets

This reader can extract image samples from:
* a single tar of image files
* a folder of multiple tarfiles containing imagefiles
* a tar of tars containing image files

Labels are based on the combined folder and/or tar name structure.

Hacked together by / Copyright 2020 Ross Wightman
    N)glob)ListTupleDictSetOptionalUnion)natural_key   )load_class_map)get_img_extensions)Readerz_tarinfos.picklec                   2     e Zd ZdZdhZdedef fdZ xZS )_TarInfoUnpicklera  Restricted unpickler for the tarinfo cache file.

    The cache only ever contains plain containers (dict/list/str/int/None) plus
    ``tarfile.TarInfo`` instances. Built-in containers are handled by pickle opcodes
    and never reach ``find_class``, so allowing only ``tarfile.TarInfo`` is sufficient
    to load valid caches while blocking arbitrary code execution from a crafted cache
    file (CWE-502, GHSA-pqww-w348-m542).
    )tarfileTarInfomodulenamec                 |    ||f| j                   v rt        | 	  ||      S t        j                  d| d| d      )NzGlobal .z+ is not permitted in a tar info cache file.)_allowed_globalssuper
find_classpickleUnpicklingError)selfr   r   	__class__s      p/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/data/readers/reader_image_in_tar.pyr   z_TarInfoUnpickler.find_class*   sH    D>T2227%fd33$$wvhav=h%ijj    )__name__
__module____qualname____doc__r   strr   __classcell__r   s   @r   r   r      s/     //k kC k kr   r   c                   L    e Zd Zddej                  dej
                  fdZd Zy)TarStateNtftic                 .    || _         || _        i | _        y N)r)   r*   children)r   r)   r*   s      r   __init__zTarState.__init__2   s    #%#%-/r   c                     d | _         y r,   )r)   r   s    r   resetzTarState.reset7   s	    r   )NN)r    r!   r"   r   TarFiler   r.   r1    r   r   r(   r(   0   s"    07?? 0w 0
r   r(   r)   parent_info
extensionsc                    d}t        |       D ]^  \  }}|j                         st        j                  j	                  |j                        \  }}t        j                  j                  |      \  }}	|	j                         }	|	dk(  rt        j                  | j                  |      d      5 }
t        |j                  t        j                  j                  |d   |      |g g       }|t        |
||      z  }t        j                  | d|j                   d	t!        |d
          d       |d   j#                  |       d d d        @|	|v sF|d
   j#                  |       |dz  }a |S # 1 sw Y   nxY w)Nr   .tarr|)fileobjmodepathr   r;   r*   r-   samplesr5   z"/?. Extracted child tarinfos from . r=   z images.r-   r   )	enumerateisfileosr;   splitsplitextlowerr   openextractfiledictr   join_extract_tarinfo_loggerdebuglenappend)r)   r4   r5   sample_countir*   dirnamebasenamer   extctf
child_infos               r   rJ   rJ   ;   s\   L2 2yy{GGMM"''2GG$$X.	ciik&=bnnR&8tD ;!rww||K4G'NSU`blnp
 0jZ XX#EbggYbQTU_`iUjQkPlltuvJ'..z:; ; J	"))"-AL  ; ;s   /BE22E<	class_name_to_idxcache_tarinfosortc                   $%&' |st        d      n
t        |      }d}t        j                  j	                  |       rt        j                  j                  |       d   j                         dk(  sJ | g}t        j                  j                  |       \  } }t        j                  j                  |      d   }d}n~| j                  t        j                  j                        j                  t        j                  j                        d   }t        t        j                  j                  | d      d      }t        |      }t        |D 	cg c]!  }	t        j                  j                  |	      # c}	      }
|sJ d	|  d
       t        j!                  d|
dz  dd       t#        g       }d}|	|
dkD  rdnd}|r,d|z   t$        z   }t        j                  j                  | |      }t        j                  j'                  |      rt        j!                  d| d       t)        |d      5 }t+        |      j-                         }d d d        t/        |t"              rt/        |j1                  d      t2              st5        d| d      t        |d         |k(  sEJ d       t7        |      D ]  \  }}|rdn>t        j                  j                  t        j                  j9                  |            d   }t;        j(                  |d      5 }t#        t        j                  j=                  ||       |d g g       }t?        |||      }t        |d         }t        jA                  | d| d| d | d!| d"
       d d d        |d   jC                          |rDt        j!                  d#| d       t)        |d$      5 }tE        jF                  ||       d d d        g 'g &d%d%g }dd%d&$$%&'fd'}t        j!                  d(       |d   D ]t  }|rd n|d)   }tI               }d}|d   D ]2  } |||*      }|rtI        |d+   ,      |jJ                  |d)   <   ||z  }4 | |||*      z  }|sb|jC                  ||f       v ~%rBt3        tM        t        &      tN        -            }t7        |      D ci c]  \  }}||
 c}}t        j!                  d.       tQ        '&      D  !cg c]  \  } }!|!v s| |!   f }"} }!|rtM        |"d/ -      }"tQ        |" \  '}#tS        jT                  '      'tS        jT                  |#      }#t        j!                  d0t        '       d1t        |       d2       '|#|fS c c}	w # 1 sw Y   KxY w# 1 sw Y   xY w# 1 sw Y   xY wc c}}w c c}!} w )3NT)as_setFr7   r   z*.tar)	recursivez'No .tar files found at specified path (z).z	Scanning i   z.2fzMB of tar files...)tartrees l       
 _z!Reading tar info from cache file r   rbr]   z'Invalid or corrupt tar info cache file z3Cached tartree len doesn't match number of tarfilesr8   )r:   r<   r>   r-   /z. Extracted tarinfos from r?   z children, z	 samples.zWriting tar info to cache file wb)	leaf_onlyc                 8   t        j                  j                  | j                  t         j                  j                        }| r,|j                  t         j                  j                        d   S |j                  t         j                  j                  d      S )Nr[   r_   )rB   r;   rI   stripseprC   replace)rc   r;   s     r   _label_from_pathsz+extract_tarinfos.<locals>._label_from_paths   s^    ww||T"((5.7tzz"''++&r*[T\\"''++WZ=[[r   c                     d}| d   D ]i  } | d   t         j                  j                  |j                              }s|vr=	j                  ||| d   f       j                  |       |dz  }k |S )Nr   r=   r;   r*   r   )rB   r;   rQ   rN   )
infofnaddedslabelrh   build_class_maprV   labelsr=   s
        r   _add_samplesz&extract_tarinfos.<locals>._add_samples   s}    i 	A%d6lBGGOOAFF4KLE"u4E'ENNAr4:./MM% QJE	 r   z+Collecting samples and building tar states.r   )rk   r*   )r*   )keyz$Mapping targets and sorting samples.c                 8    t        | d   d   j                        S Nr   )r
   r;   )ks    r   <lambda>z"extract_tarinfos.<locals>.<lambda>   s    TUVWTXYZT[T`T`Ha r   zFinished processing z samples across z tar files.)+r   setrB   r;   rA   rD   rE   rC   re   rf   r   rI   rM   sumgetsizerK   rj   rH   CACHE_FILENAME_SUFFIXexistsrF   r   load
isinstancegetlist
ValueErrorr@   rR   r   relpathrJ   rL   rN   r   dumpr(   r-   sortedr
   zipnparray)(rootrV   rW   r5   rX   root_is_tartar_filenames	root_namenum_tarsf	tar_bytesrj   
cache_pathcache_filenamepfrP   rk   r;   r)   r4   num_samplesnum_childrentarfilesrq   tar_name	tar_stateparent_addedrU   child_addedsorted_labelsidxcrm   lsamples_and_targetstargetsrh   ro   rp   r=   s(    `                                  @@@@r   extract_tarinfosr   P   s\    9C#40JJK	ww~~dww%b)//1V;;;''---iGG$$Y/2	JJrww{{+11"''++>rB	RWW\\$8DI=!H?ARWW__Q'?@IG>tfBGG8LL9Yw.s33EFGDJ )J 6Ey+@@WW\\$7
	ww~~j!8AFG*d# 	0r$R(--/D	0$%Z8Ld-SFzlRSTUU4
#$0g2gg0}- 	1EAr$2"''*:*:277;K;KB;O*PQR*SDbt, v"D(AQU`blno.r;:V";z#:;c8*$>rd"\NR]^i]jjstv	v ##K0	1 LL::,aHIj$' &2D"%& GFO  H+/ \	 	 LL>@J' 3&4K,?J	%j1 	(J&zh?K9AZPTEU9V	"":f#56K'L		(
 	[X>>OOXy123 	VCK[AB2;M2JKQQVKLL79ADWfAUpAYZ^oYoA034pp$%8>ab/0GWhhwGhhwGLL'G~5Ec(m_T_`aG.88s @	0 	0v v& &b L qs=   &V3VA/V#V0
V=?W	WV #V-	0V:c                   8     e Zd ZdZd fd	Zd Zd ZddZ xZS )ReaderImageInTarzI Multi-tarfile dataset reader where there is one .tar file per class
    c                    t         	|           d }|rt        ||      }|| _        t	        | j                  ||      \  | _        | _        | _        }| j                  j                         D ci c]  \  }}||
 c}}| _	        t        |      dk(  r$|d   d   d| _        |d   d   | _        || _        y d| _        t        |      | _        || _        y c c}}w )N)rV   rW   r   r   TF)r   r.   r   r   r   r=   r   rV   itemsclass_idx_to_namerM   r   r   rH   cache_tarfiles)
r   r   	class_mapr   rW   rV   r   ru   vr   s
            r   r.   zReaderImageInTar.__init__   s      .y$ ?	GWII/'H
DdlD$:H
 483I3I3O3O3Q!R41a!Q$!Rx=A(1+a."8#D%a[^DN -  %D!(^DN, "Ss   0Cc                 ,    t        | j                        S r,   )rM   r=   r0   s    r   __len__zReaderImageInTar.__len__   s    4<<  r   c                    | j                   |   }| j                  |   }|\  }}}|r*t        j                  j	                  | j
                  |      n| j
                  }d }d }	| j                  r3| j                  r| j                  n| j                  |   }	|	j                  }|(t        j                  |      }| j                  r||	_	        || j                  r#|	j                  |j                     j                  nd }
|
Ot        j                  |j                  |            }
| j                  r|
|	j                  |j                     _	        |
}|j                  |      |fS )N)r9   )r=   r   rB   r;   rI   r   r   r   r   r)   r   rF   r-   r   rG   )r   indexsampletarget	sample_ti	parent_fnchild_ti
parent_absr)   cache_staterT   s              r   __getitem__zReaderImageInTar.__getitem__   s&   e$e$)/&	9h;DRWW\\$))Y7$))
,0,<,<$..$..QZB[KB:j)B""!#<@<O<O+&&x}}588UYC{ll2>>(+CD&&=@K((7:B~~i(&00r   c                     | j                   |   d   j                  }|rt        j                  j	                  |      }|S rt   )r=   r   rB   r;   rR   )r   r   rR   absolutefilenames        r   	_filenamezReaderImageInTar._filename   s7    <<&q)..ww''1Hr   )r^   TN)FF)	r    r!   r"   r#   r.   r   r   r   r%   r&   s   @r   r   r      s    -*!12r   r   )NNNT)$r#   loggingrB   r   r   r   typingr   r   r   r   r   r	   numpyr   timm.utils.miscr
   r   r   img_extensionsr   readerr   	getLoggerr    rK   rz   	Unpicklerr   r(   r2   r$   rJ   boolr   r   r3   r   r   <module>r      s   
  	    : :  ' % . 
'

H
%* k(( k"  t S . -1(,8<l9#D>l9  ~l9 U4#345	l9
 l9^9v 9r   