Ë
    éÓ^jPB  ã                   óž  — d Z ddlZddlZddlZddlZddlZddlZddlZddlm	Z	 ddl
mZ ddlmZ ddlmZmZmZmZmZmZ ddlZddlmZ ddlZddlmZ ddlmZmZmZ 	 ddl Z!dd	l"m#Z#m$Z$ dd
l%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+ ddl-m.Z. ddl/m0Z0 ddl1m2Z2  ejf                  e4«      Z5 e6ejn                  jq                  dd«      «      Z9 e6ejn                  jq                  dd«      «      Z:d#d„Z;e	 G d„ d«      «       Z<de=defd„Z>d„ Z?	 	 	 	 d$d„Z@d„ ZAe!� G d„ de!j„                  «      ZCndZC G d„ d e«      ZD G d!„ d"e0«      ZEy# e,$ r dZ!dZ&Y ŒÍw xY w)%zR Dataset reader for webdataset

Hacked together by / Copyright 2022 Ross Wightman
é    N)Ú	dataclass)Úpartial)Úislice)ÚAnyÚCallableÚDictÚListÚOptionalÚTuple)ÚImage)ÚDatasetÚIterableDatasetÚget_worker_info)Ú_shuffleÚgetfirst)Úexpand_urls)Úbase_plus_extÚ
url_openerÚtar_file_expanderÚvalid_sampleé   )Úload_class_map)ÚReader)ÚSharedCountÚWDS_SHUFFLE_SIZEi    ÚWDS_INITIAL_SIZEi   c                 óò  — t        |t        «      r|f}g }d}|D ]‘  }t        j                  j	                  | |«      }	 |j                  |«       t        j                  |«      5 }|j                  d«      rt        j                  |«      }nt        j                  |«      }d d d «       c S  t        j                  d|› d|› d�«       i S # 1 sw Y   Œ,xY w# t        $ r}t        |«      }Y d }~Œ×d }~ww xY w)NÚ z.jsonzDataset info file not found at z	. Error: z.. Falling back to provided split and size arg.)Ú
isinstanceÚstrÚosÚpathÚjoinÚappendÚwdsÚgopenÚendswithÚjsonÚloadÚyamlÚ	safe_loadÚ	ExceptionÚ_loggerÚwarning)	ÚrootÚnamesÚtriedÚerr_strÚnÚ	full_pathÚfÚ	info_dictÚes	            úg/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/data/readers/reader_wds.pyÚ
_load_infor9   *   sê   € Ü�%œÔØ�ˆØ€EØ€GØò ˆÜ—G‘G—L‘L  qÓ)ˆ	ð		Ø�L‰L˜Ô#Ü—‘˜9Ó%ð 2¨Ø—:‘:˜gÔ&Ü $§	¡	¨!£‘Iä $§¡¨qÓ 1�I÷	2ð
 Òðô ‡O�OØ
)¨%¨°	¸'¸ð C7ð 	7ô8ð €I÷2ð 2ûô ò 	Ü˜!“f�Gûð	ús/   ¾&CÁ$=CÂ!	CÃC	ÃCÃ	C6Ã!C1Ã1C6c                   óX   — e Zd ZU eed<   ee   ed<   dZee   ed<   dZeed<   dZ	eed<   y)	Ú	SplitInfoÚnum_samplesÚ	filenames© Úshard_lengthsr   Ú	alt_labelÚnameN)
Ú__name__Ú
__module__Ú__qualname__ÚintÚ__annotations__r   r    r?   r@   rA   r>   ó    r8   r;   r;   B   s3   … àÓØ�S‰zÓØ "€M�5˜‘:Ó"Ø€IˆsÓØ€Dˆ#„NrG   r;   ÚsplitÚinfoc           	      óô  ‡— d„ }d| v sd| v �r| j                  d«      } d}d}t        | «      dkD  rt        | d   «      }| d   } d| vr5| j                  d	d
«      }t        |«      dz
  }|rd|v r||   |d   v r||   }t        | «      }|r‹|d   |   }|swt	        |d   |d   «      D �	�
ci c]  \  }	}
|	|
“Œ
 c}
}	Št        ˆfd„|D «       «      }t        ‰j                  «       «      |d<   t        ‰j                  «       «      |d<   ||d<    ||«      }|S t        |||¬«      }|S d|vs| |d   vr0t        d| › d|j                  di «      j                  «       › d�«      ‚| } |d   |    } ||«      }|S c c}
}	w )Nc           	      ó~   — t        | d   t        | d   «      t        | d   «      | j                  dd«      | d   ¬«      S )Nr<   r=   r?   r@   r   rA   )r<   r=   r?   r@   rA   )r;   ÚtupleÚget)Ú	dict_infos    r8   Ú_info_convertz(_parse_split_info.<locals>._info_convertL   sG   € ÜØ! -Ñ0Ü˜I kÑ2Ó3Ü 	¨/Ñ :Ó;Ø—m‘m K°Ó4Ø˜6Ñ"ô
ð 	
rG   Útarz..ú|r   r   r   z::ú-é   Úsplitsr=   r?   c              3   ó(   •K  — | ]	  }‰|   –— Œ y ­w©Nr>   )Ú.0r5   Ú_fcs     €r8   ú	<genexpr>z$_parse_split_info.<locals>.<genexpr>i   s   øè ø€ Ò!B¨Q # a¥&Ñ!Bùs   ƒr<   )rA   r<   r=   zsplit z not found in info (ú))rH   ÚlenrE   r   ÚzipÚsumrL   ÚkeysÚvaluesr;   ÚRuntimeErrorrM   )rH   rI   rO   r<   Ú
split_nameÚsplit_partsÚ	split_idxÚsplit_filenamesÚ
split_infor5   ÚcrX   s              @r8   Ú_parse_split_inforg   K   sÉ  ø€ ò
ð ��~˜ šð —‘˜CÓ ˆØˆØˆ
Üˆu‹:˜Š>Ü˜e A™h›-ˆKØ�a‘ˆØ�uÑØŸ+™+ c¨1Ó-ˆKÜ˜KÓ(¨1Ñ,ˆIÙ˜X¨Ñ-°+¸iÑ2HÈDÐQYÉNÑ2ZØ(¨Ñ3�
ä% eÓ,ˆÙØ˜h™¨
Ñ3ˆJÙÜ(+¨J°{Ñ,CÀZÐP_ÑE`Ó(a×b¡  1�q˜!‘tÓb�Ü!Ó!B°/Ô!BÓB�Ü*/°·±³
Ó*;�
˜;Ñ'Ü.3°C·J±J³LÓ.A�
˜?Ñ+Ø,7�
˜=Ñ)Ù& zÓ2ˆJð Ðô #ØØ'Ø)ôˆJð Ðð ˜4Ñ 5°°X±Ñ#>Ü ¨ wÐ.BÀ4Ç8Á8ÈHÐVXÓCY×C^ÑC^ÓC`ÐBaÐabÐcÓdÐdØˆØ˜(‘^ EÑ*ˆ
Ù" :Ó.ˆ
àÐùó' cs   Â)E4c                 ól   — t         j                  dt        | «      › d�«       t        | t        «      r| ‚y)zQCall in an exception handler to ignore exceptions, issue a warning, and continue.zHandling webdataset error (z). Ignoring.T)r-   r.   Úreprr   Ú	TypeError)Úexns    r8   Úlog_and_continuerl   ~   s.   € ä‡O�OÐ1´$°s³)°¸LÐIÔJä�#”yÔ!Øˆ	ØrG   c                 ó–  — |r,t        j                  | d   «      }t        ||   «      }|dk  ryt        | |   «      }t        | |«      }t	        j
                  |«      5 }t        j                  |«      }|j                  «        ddd«       |r|j                  |«      }t        ||| j                  dd«      ¬«      }	|	S # 1 sw Y   Œ<xY w)z¡ Custom sample decode
    * decode and convert PIL Image
    * cls byte string label to int
    * pass through JSON byte string (if it exists) without parse
    r(   r   N)ÚjpgÚclsr(   )r(   ÚloadsrE   r   ÚioÚBytesIOr   Úopenr)   ÚconvertÚdictrM   )
ÚsampleÚ	image_keyÚ
image_modeÚ
target_keyr@   ÚmetaÚclass_labelÚimgÚbÚdecodeds
             r8   Ú_decoder   ‡   s¹   € ñ ä�z‰z˜& ™.Ó)ˆÜ˜$˜y™/Ó*ˆØ˜Š?àä˜& Ñ,Ó-ˆô �6˜9Ó
%€CÜ	�‰�C‹ð ˜AÜ�j‰j˜‹mˆØ�‰Œ
÷ñ Ø�k‰k˜*Ó%ˆô �s °&·*±*¸VÀTÓ2JÔK€GØ€N÷ð ús   Á&B?Â?Cc                  ón   — t        «       } | �| j                  S t        j                  j	                  «       S )z'get dataloader worker seed from pytorch)r   Úseedr%   ÚutilsÚpytorch_worker_seed)Úworker_infos    r8   rƒ   rƒ   «   s/   € ä!Ó#€KØÐà×ÑÐä�9‰9×(Ñ(Ó*Ð*rG   c                   ó"   — e Zd Z	 	 	 	 dd„Zd„ Zy)Údetshuffle2c                 ó<   — || _         || _        || _        || _        y rV   )ÚbufsizeÚinitialr�   Úepoch)Úselfrˆ   r‰   r�   rŠ   s        r8   Ú__init__zdetshuffle2.__init__¹   s    € ð #ˆDŒLØ"ˆDŒLØˆDŒIØˆD�JrG   c                 ól  — t        | j                  t        «      r| j                  j                  }n!| xj                  dz  c_        | j                  }| j                  dk  rt        «       |z   }n| j                  |z   }t        j                  |«      }t        || j                  | j                  |«      S )Nr   r   )r   rŠ   r   Úvaluer�   rƒ   ÚrandomÚRandomr   rˆ   r‰   )r‹   ÚsrcrŠ   r�   Úrngs        r8   Úrunzdetshuffle2.runÅ   s„   € Ü˜$Ÿ*™*¤kÔ2ØŸ
™
×(Ñ(‘ð —
’
˜a‘•
ØŸ
™
�à�y‰y˜1Š}Ü*Ó,¨uÑ4‘à—y‘y 5Ñ(�ä—-‘- Ó%ˆCÜ˜C §¡¨t¯|©|¸SÓAÐArG   N)iè  éd   r   éÿÿÿÿ)rB   rC   rD   rŒ   r“   r>   rG   r8   r†   r†   ¸   s   „ ð ØØØó
	ó	BrG   r†   c                   óF   ‡ — e Zd ZdZej
                  dddfˆ fd„	Zd„ Zˆ xZS )ÚResampledShards2z,An iterable dataset yielding a list of urls.NTr•   c                 ó&  •— t         ‰| �  «        t        j                  j	                  |«      }|| _        t        | j
                  d   t        «      sJ ‚|| _        t        j                  «       | _        |€t        n|| _        || _        || _        y)z‹Sample shards from the shard list with replacement.

        :param urls: a list of URLs as a Python list or brace notation string
        r   N)ÚsuperrŒ   r%   Ú
shardlistsr   Úurlsr   r    Únshardsr�   r�   r’   rƒ   Úworker_seedÚdeterministicrŠ   )r‹   r›   rœ   r�   rž   rŠ   Ú	__class__s         €r8   rŒ   zResampledShards2.__init__Ý   sv   ø€ ô 	‰ÑÔÜ�~‰~×)Ñ)¨$Ó/ˆØˆŒ	Ü˜$Ÿ)™) A™,¬Ô,Ð,Ð,ØˆŒÜ—=‘=“?ˆŒØ2=Ð2EÕ.È;ˆÔØ*ˆÔØˆ�
rG   c              #   óì  K  — t        | j                  t        «      r| j                  j                  }n!| xj                  dz  c_        | j                  }| j                  r+t        j                  | j                  «       |z   «      | _        t        | j                  «      D ]O  }| j                  j                  dt        | j                  «      dz
  «      }t        | j                  |   ¬«      –— ŒQ y­w)z#Return an iterator over the shards.r   r   )ÚurlN)r   rŠ   r   rŽ   rž   r�   r�   r�   r’   Úrangerœ   Úrandintr[   r›   ru   )r‹   rŠ   Ú_Úindexs       r8   Ú__iter__zResampledShards2.__iter__ó   s±   è ø€ ä�d—j‘j¤+Ô.Ø—J‘J×$Ñ$‰Eð �JŠJ˜!‰O�JØ—J‘JˆEà×Òä—}‘} T×%5Ñ%5Ó%7¸%Ñ%?Ó@ˆDŒHä�t—|‘|Ó$ò 	-ˆAØ—H‘H×$Ñ$ Q¬¨D¯I©I«¸Ñ(:Ó;ˆEÜ˜4Ÿ9™9 UÑ+Ô,Ó,ñ	-ùs   ‚C2C4)	rB   rC   rD   Ú__doc__ÚsysÚmaxsizerŒ   r¦   Ú__classcell__©rŸ   s   @r8   r—   r—   Ú   s"   ø„ Ù6ð
 —‘ØØØõö,-rG   r—   c            "       óè   ‡ — e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddedee   dededee   dededed	ee   d
edededededee   dee   f ˆ fd„Zd„ Z		 ddee   fd„Z
d„ Zd„ Zd„ Zd„ Zd„ Zdd„Zdd„Zˆ xZS )Ú	ReaderWdsr/   rA   rH   Úis_trainingr<   Ú
batch_sizeÚrepeatsr�   Ú	class_mapÚ	input_keyÚinput_img_modery   Útarget_img_modeÚfilename_keyÚsample_shuffle_sizeÚsample_initial_sizec                 óÜ  •— t         ‰| �  «        t        €t        d«      ‚|| _        || _        || _        || _        || _        d| _	        |xs t        | _        |xs t        | _        |
| _        || _        || _        || _        d| _        t'        | j                  «      | _        t+        || j(                  «      | _        |�|| _        n| j,                  j.                  | _        |r| j.                  st        d«      ‚d| _        |	rt3        |	«      | _        d| _        ni | _        d| _        d| _        t;        j<                  «       r]t;        j>                  «       rIt;        j@                  «       dkD  r2t;        jB                  «       | _        t;        j@                  «       | _        d | _"        d| _#        || _$        d| _%        d| _&        d| _'        d| _(        tS        «       | _*        d | _+        y )	NzcPlease install webdataset 0.2.x package `pip install git+https://github.com/webdataset/webdataset`.iô  z.JPEGzBInvalid split definition, num_samples not specified in train mode.FTr   r   ),r™   rŒ   r%   r`   r/   r®   r¯   r°   Úcommon_seedÚshard_shuffle_sizeÚSAMPLE_SHUFFLE_SIZEr¶   ÚSAMPLE_INITIAL_SIZEr·   r²   r³   ry   rµ   Úkey_extr9   rI   rg   re   r<   Úremap_classr   Úclass_to_idxÚ	dist_rankÚdist_num_replicasÚdistÚis_availableÚis_initializedÚget_world_sizeÚget_rankr„   Ú	worker_idr�   Únum_workersÚglobal_worker_idÚglobal_num_workersÚ
init_countr   Úepoch_countÚds)r‹   r/   rA   rH   r®   r<   r¯   r°   r�   r±   r²   r³   ry   r´   rµ   r¶   r·   rŸ   s                    €r8   rŒ   zReaderWds.__init__  s¸  ø€ ô& 	‰ÑÔÜˆ;ÜØuówð wàˆŒ	Ø&ˆÔØ$ˆŒØˆŒØˆÔØ"%ˆÔØ#6Ò#MÔ:MˆÔ Ø#6Ò#MÔ:MˆÔ à"ˆŒØ,ˆÔØ$ˆŒØ(ˆÔØˆŒä˜tŸy™yÓ)ˆŒ	Ü+¨E°4·9±9Ó=ˆŒØÐ"Ø*ˆDÕà#Ÿ™×:Ñ:ˆDÔÙ˜t×/Ò/ÜÐ!cÓeÐeØ ˆÔÙÜ .¨yÓ 9ˆDÔØ#ˆDÕà "ˆDÔð ˆŒØ!"ˆÔÜ×ÑÔ¤4×#6Ñ#6Ô#8¼T×=PÑ=PÓ=RÐUVÒ=VÜ!Ÿ]™]›_ˆDŒNÜ%)×%8Ñ%8Ó%:ˆDÔ"ð  ˆÔØˆŒØˆÔØˆÔØ !ˆÔØ"#ˆÔØˆŒÜ&›=ˆÔð ˆ�rG   c                 ó&   — || j                   _        y rV   )rÌ   rŽ   )r‹   Úcounts     r8   Ú	set_epochzReaderWds.set_epochQ  s   € Ø!&ˆ×ÑÕrG   rÈ   c                 ón   — | j                   �y |�&|| _        | j                  | j                  z  | _        y y rV   )rÍ   rÈ   rÁ   rÊ   )r‹   rÈ   s     r8   Úset_loader_cfgzReaderWds.set_loader_cfgT  s=   € ð �7‰7ÐØØÐ"Ø*ˆDÔØ&*×&<Ñ&<¸t×?OÑ?OÑ&OˆDÕ#ð #rG   c                 ó`  — | j                   €­t        j                  j                  j	                  «       }|�:|| _         |j
                  | _        |j                  | _        |j                  | _	        | j                  | j                  z  | _        | j                  | j                  z  | j                  z   | _        | j                  j                  D �cg c],  }t         j"                  j%                  | j&                  |«      ‘Œ. }}t)        j*                  |«      g}| j,                  r©|j/                  t1        | j2                  | j4                  | j6                  ¬«      | j8                  t)        j:                  t<        ¬«      t)        j>                  | j@                  | jB                  tE        jF                  | j                  «      ¬«      g«       n5|j/                  | j8                  t)        j:                  t<        ¬«      g«       |j/                  t)        jH                  tK        tL        | jN                  | jP                  | j                  jR                  ¬«      t<        ¬«      t)        jT                  | jN                  | jV                  ¬«      g«       t)        jX                  |Ž | _-        yc c}w )z8 Lazily initialize worker (in worker processes)
        N)r�   rŠ   )Úhandler)rˆ   r‰   r’   )rw   rx   r@   )ÚimageÚtarget).r„   Útorchr‚   Údatar   ÚidrÇ   r�   r�   rÈ   rÁ   rÊ   rÀ   rÉ   re   r=   r!   r"   r#   r/   r%   ÚSimpleShardListr®   Úextendr†   rº   r¹   rÌ   Ú_split_by_node_and_workerÚtarfile_to_samplesrl   Úshuffler¶   r·   r�   r�   Úmapr   r   r²   r³   r@   Úrenamery   ÚDataPipelinerÍ   )r‹   r„   r5   Úabs_shard_filenamesÚpipelines        r8   Ú
_lazy_initzReaderWds._lazy_init^  s  € ð ×ÑÐ#ÜŸ+™+×*Ñ*×:Ñ:Ó<ˆKØÐ&Ø#.�Ô Ø!,§¡�”Ø#.×#3Ñ#3�Ô Ø#.×#:Ñ#:�Ô Ø&*×&<Ñ&<¸t×?OÑ?OÑ&OˆDÔ#Ø$(§N¡N°T×5EÑ5EÑ$EÈÏÉÑ$VˆDÔ!ð DHÇ?Á?×C\ÑC\Ö]¸aœrŸw™wŸ|™|¨D¯I©I°qÕ9Ð]ÐÐ]Ü×'Ñ'Ð(;Ó<Ð=ˆà×ÒØ�O‰OÜØ×+Ñ+Ø×)Ñ)Ø×*Ñ*ôð
 ×.Ñ.ä×&Ñ&Ô/?Ô@Ü—‘Ø ×4Ñ4Ø ×4Ñ4ÜŸ™ d×&6Ñ&6Ó7ôðõ ð  �O‰OØ×.Ñ.ä×&Ñ&Ô/?Ô@ðô ð
 	�‰Ü�G‰GÜÜØ"Ÿn™nØ#×2Ñ2Ø"Ÿo™o×7Ñ7ô	ô )ôô �J‰J˜TŸ^™^°D·O±OÔDð
ô 	ô ×"Ñ" HÐ-ˆ�ùòK ^s   Ã1J+c              #   ó˜   K  — | j                   dkD  r,t        || j                  d | j                   «      D ]  }|–— Œ y |D ]  }|–— Œ y ­w©Nr   )rÊ   r   rÉ   )r‹   r‘   Úss      r8   rÜ   z#ReaderWds._split_by_node_and_worker“  sU   è ø€ Ø×"Ñ" QÒ&Ü˜C ×!6Ñ!6¸¸d×>UÑ>UÓVò �Ø“ñð ò �Ø“ñùs   ‚AA
c                 óH  — | j                   t        | j                  | j                  «      z  }| j                  s| j                  dkD  rt        j                  |«      }| j                  r/t        j                  || j                  z  «      | j                  z  }t        |«      S ræ   )	r<   ÚmaxrÊ   rÁ   r®   ÚmathÚceilr¯   rE   )r‹   Únum_worker_sampless     r8   Ú_num_samples_per_workerz!ReaderWds._num_samples_per_worker›  s†   € Ø!×-Ñ-´°D×4KÑ4KÈT×McÑMcÓ0dÑdÐØ×Ò˜t×5Ñ5¸Ò9Ü!%§¡Ð+=Ó!>ÐØ×ÒÜ!%§¡Ð+=ÀÇÁÑ+OÓ!PÐSW×SbÑSbÑ!bÐÜÐ%Ó&Ð&rG   c              #   óX  K  — | j                   €| j                  «        | j                  «       }| j                  s| j                  dkD  r| j                   j                  |«      }n| j                   }d}|D ]0  }|d   }| j                  r| j                  |   }|d   |f–— |dz  }Œ2 y ­w)Nr   r   rÖ   rÕ   )rÍ   rä   rí   r®   rÁ   Ú
with_epochr¾   r¿   )r‹   rì   rÍ   Úirv   rÖ   s         r8   r¦   zReaderWds.__iter__£  s­   è ø€ Ø�7‰7ˆ?Ø�O‰OÔà!×9Ñ9Ó;ÐØ×Ò˜t×5Ñ5¸Ò9ð
 —‘×#Ñ#Ð$6Ó7‰Bà—‘ˆBàˆàò 	ˆFØ˜HÑ%ˆFØ×ÒØ×*Ñ*¨6Ñ2�Ø˜‘/ 6Ð)Ò)Ø�‰F‰Añ	ùs   ‚B(B*c                 ó@   — | j                  «       | j                  z  }|S rV   )rí   rÈ   )r‹   r<   s     r8   Ú__len__zReaderWds.__len__»  s!   € Ø×2Ñ2Ó4°t×7GÑ7GÑGˆØÐrG   c                 ó   — J d«       ‚)NzNot supportedr>   )r‹   r¥   ÚbasenameÚabsolutes       r8   Ú	_filenamezReaderWds._filename¿  s   € Ø%�oÓ%ˆurG   c                 ó4  — | j                   €| j                  «        g }| j                   D ]j  }| j                  |v r|| j                     }nd|v r|d   | j                  z   }nJ d«       ‚|j	                  |«       t        |«      | j                  k\  sŒi |S  |S )z0 Return all filenames in dataset, overrides baseÚ__key__zNo supported name field present)rÍ   rä   rµ   r½   r$   r[   r<   )r‹   rô   rõ   r0   rv   rA   s         r8   r=   zReaderWds.filenamesÂ  sž   € à�7‰7ˆ?Ø�O‰OÔàˆØ—g‘gò 		ˆFØ× Ñ  FÑ*Ø˜d×/Ñ/Ñ0‘Ø˜fÑ$Ø˜iÑ(¨4¯<©<Ñ7‘à?Ð?Ó?�uØ�L‰L˜ÔÜ�5‹z˜T×-Ñ-Ó-ØØˆð		ð ˆrG   )NÚtrainFNr   r   é*   Nzjpg;png;webpÚRGBro   r   ÚfilenameNNrV   )FF)rB   rC   rD   r    r
   ÚboolrE   ru   rŒ   rÐ   rÒ   rä   rÜ   rí   r¦   rò   rö   r=   rª   r«   s   @r8   r­   r­     s<  ø„ ð #'Ø Ø %Ø)-ØØØØ(,Ø+Ø"'Ø#Ø#%Ø *Ø15Ø15ñ#HàðHð ˜3‘-ðHð ð	Hð
 ðHð " #™ðHð ðHð ðHð ðHð   ‘~ðHð ðHð  ðHð ðHð !ðHð ðHð  "*¨#¡ð!Hð" "*¨#¡õ#HòT'ð
 *.ñPà! #™óPò3.òjò'òò0ó&÷rG   r­   ))z
_info.jsonz	info.json)rn   rû   ro   r   )Fr§   rq   r(   Úloggingrê   r!   r�   r¨   Údataclassesr   Ú	functoolsr   Ú	itertoolsr   Útypingr   r   r   r	   r
   r   r×   Útorch.distributedÚdistributedrÂ   r*   ÚPILr   Útorch.utils.datar   r   r   Ú
webdatasetr%   Úwebdataset.filtersr   r   Úwebdataset.shardlistsr   Úwebdataset.tariteratorsr   r   r   r   ÚImportErrorr±   r   Úreaderr   Úshared_countr   Ú	getLoggerrB   r-   rE   ÚenvironrM   r»   r¼   r9   r;   r    rg   rl   r   rƒ   ÚPipelineStager†   r—   r­   r>   rG   r8   ú<module>r     sR  ðñó 
Û Û Û Û 	Û Û 
Ý !Ý Ý ß =× =ã Ý  Û Ý ß FÑ FðÛß5Ý1ßbÓbõ
 &Ý Ý %à
ˆ'×
Ñ
˜HÓ
%€á˜"Ÿ*™*Ÿ.™.Ð);¸TÓBÓCÐ Ù˜"Ÿ*™*Ÿ.™.Ð);¸TÓBÓCÐ óð0 ÷ð ó ðð0˜Sð 0¨ó 0òfð ØØØó!òH+ð €?ôB�c×'Ñ'õ Bð> €Kô)-�ô )-ôXL�õ LøðU ò Ø
€CØ‚Kðús   Á E  Å 	EÅE