Ë
    éÓ^jÙ   ã                   ó"  — d Z ddlZddlZddlmZmZ ddlmZ ddlZddl	m
Z ddlmZ 	 ddlZddlmZ ddlmZ d	d
lmZ d	dlmZ d	dlmZ  eej:                  j=                  dd«      «      Z G d„ de«      Z y# e$ rZ ed«       e‚dZ[ww xY w)z' Dataset reader for HF IterableDataset
é    N)ÚrepeatÚchain)ÚOptional)ÚImage)Úsplit_dataset_by_node)Ú	SplitInfozDPlease install Hugging Face datasets package `pip install datasets`.é   )Úload_class_map)ÚReader)ÚSharedCountÚHFIDS_SHUFFLE_SIZEi   c            !       óÜ   ‡ — e Zd Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddedee   dedededededed	ee   d
ededededee   dee   def ˆ fd„Zd„ Z		 ddee   fd„Z
d„ Zd„ Zd„ Zd„ Zdd„Zdd„Zˆ xZS )ÚReaderHfidsÚnameÚrootÚsplitÚis_trainingÚ
batch_sizeÚdownloadÚrepeatsÚseedÚ	class_mapÚ	input_keyÚinput_img_modeÚ
target_keyÚtarget_img_modeÚshuffle_sizeÚnum_samplesÚtrust_remote_codec                 ó¦  •— t         ‰| �  «        || _        || _        || _        || _        || _        || _        || _        |xs t        | _
        |
| _        || _        || _        || _        t        j                   |||¬«      | _        |r| j"                  j%                  «        d }| j"                  j&                  j(                  rv|| j"                  j&                  j(                  v rTt+        | j"                  j&                  j(                  |   t,        «      r#| j"                  j&                  j(                  |   }|r|| _        n+|r|j0                  r|j0                  | _        nt3        d«      ‚d| _        |	rt7        |	«      | _        d| _        ni | _        d| _        d| _        t?        j@                  «       r]t?        jB                  «       rIt?        jD                  «       dkD  r2t?        jF                  «       | _        t?        jD                  «       | _        d | _$        d| _%        d| _&        d| _'        d| _(        d | _)        tU        «       | _+        y )N)Ú	cache_dirr   z’Dataset length is unknown, please pass `num_samples` explicitly. The number of steps needs to be known in advance for the learning rate scheduler.FTr   r	   ),ÚsuperÚ__init__r   r   r   r   r   r   Úcommon_seedÚSHUFFLE_SIZEr   r   r   r   r   ÚdatasetsÚload_dataset_builderÚbuilderÚdownload_and_prepareÚinfoÚsplitsÚ
isinstancer   r   Únum_examplesÚ
ValueErrorÚremap_classr
   Úclass_to_idxÚ	dist_rankÚdist_num_replicasÚdistÚis_availableÚis_initializedÚget_world_sizeÚget_rankÚworker_infoÚ	worker_idÚnum_workersÚglobal_worker_idÚglobal_num_workersÚdsr   Úepoch)Úselfr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Ú
split_infoÚ	__class__s                     €úi/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/timm/data/readers/reader_hfids.pyr#   zReaderHfids.__init__   sù  ø€ ô& 	‰ÑÔØˆŒ	ØˆŒ
Ø&ˆÔØ$ˆŒØ ˆŒØˆŒØˆÔØ(Ò8¬LˆÔà"ˆŒØ,ˆÔØ$ˆŒØ.ˆÔä×4Ñ4ØØØ/ô
ˆŒñ
 Ø�L‰L×-Ñ-Ô/à*.ˆ
Ø�<‰<×Ñ×#Ò#¨°·±×1BÑ1B×1IÑ1IÑ(IÜ˜$Ÿ,™,×+Ñ+×2Ñ2°5Ñ9¼9ÔEØ26·,±,×2CÑ2C×2JÑ2JÈ5Ñ2Q�
áØ*ˆDÕÙ˜J×3Ò3Ø)×6Ñ6ˆDÕäðdóð ð
 !ˆÔÙÜ .¨yÓ 9ˆDÔØ#ˆDÕà "ˆDÔð ˆŒØ!"ˆÔÜ×ÑÔ¤4×#6Ñ#6Ô#8¼T×=PÑ=PÓ=RÐUVÒ=VÜ!Ÿ]™]›_ˆDŒNÜ%)×%8Ñ%8Ó%:ˆDÔ"ð  ˆÔØˆŒØˆÔØ !ˆÔØ"#ˆÔð 7;ˆŒÜ “]ˆ�
ó    c                 ó&   — || j                   _        y ©N)r>   Úvalue)r?   Úcounts     rB   Ú	set_epochzReaderHfids.set_epochp   s   € à ˆ�
‰
ÕrC   r:   c                 ón   — | j                   �y |�&|| _        | j                  | j                  z  | _        y y rE   )r=   r:   r2   r<   )r?   r:   s     rB   Úset_loader_cfgzReaderHfids.set_loader_cfgt   s=   € ð �7‰7ÐØØÐ"Ø*ˆDÔØ&*×&<Ñ&<¸t×?OÑ?OÑ&OˆDÕ#ð #rC   c                 óò  — | j                   €œt        j                  j                  j	                  «       }|�)|| _         |j
                  | _        |j                  | _        | j                  | j                  z  | _	        | j                  | j                  z  | j                  z   | _        | j                  rC| j                  j                  | j                  ¬«      }|j!                  | j                  ¬«      }n&| j                  j#                  | j                  ¬«      }| j$                  r'|j'                  | j(                  | j*                  ¬«      }t-        || j                  | j                  ¬«      | _        y)z8 Lazily initialize worker (in worker processes)
        N)r   )Ú
num_shards)r   Úbuffer_size)ÚrankÚ
world_size)r8   ÚtorchÚutilsÚdataÚget_worker_infoÚidr9   r:   r2   r<   r1   r;   r   r(   Ú
as_datasetr   Úto_iterable_datasetÚas_streaming_datasetr   Úshuffler$   r   r   r=   )r?   r8   Údatasetr=   s       rB   Ú
_lazy_initzReaderHfids._lazy_init~   s#  € ð ×ÑÐ#ÜŸ+™+×*Ñ*×:Ñ:Ó<ˆKØÐ&Ø#.�Ô Ø!,§¡�”Ø#.×#:Ñ#:�Ô Ø&*×&<Ñ&<¸t×?OÑ?OÑ&OˆDÔ#Ø$(§N¡N°T×5EÑ5EÑ$EÈÏÉÑ$VˆDÔ!à�=Š=Ø—l‘l×-Ñ-°D·J±JÐ-Ó?ˆGà×,Ñ,¸×8OÑ8OÐ,ÓP‰Bð —‘×2Ñ2¸¿¹Ð2ÓDˆBà×Òà—‘ ×!1Ñ!1¸t×?PÑ?P�ÓQˆBô (¨°·±ÈD×LbÑLbÔcˆ�rC   c                 óŽ  — t        d| j                  «      | j                  z  t        | j                  | j                  «      z  }| j
                  s| j                  dkD  rt        j                  |«      }| j
                  r;| j                  �/t        j                  || j                  z  «      | j                  z  }t        |«      S )Nr	   )
Úmaxr   r   r<   r2   r   ÚmathÚceilr   Úint)r?   Únum_worker_sampless     rB   Ú_num_samples_per_workerz#ReaderHfids._num_samples_per_workerŸ   s¢   € ä��4—<‘<Ó  4×#3Ñ#3Ñ3´c¸$×:QÑ:QÐSW×SiÑSiÓ6jÑjð 	à×Ò˜t×5Ñ5¸Ò9Ü!%§¡Ð+=Ó!>ÐØ×Ò §¡Ð ;Ü!%§¡Ð+=ÀÇÁÑ+OÓ!PÐSW×SbÑSbÑ!bÐÜÐ%Ó&Ð&rC   c              #   ó\  K  — | j                   €| j                  «        | j                   j                  | j                  j                  «       | j                  «       }d}| j                  r)t        j                  t        | j                   «      «      }nt        | j                   «      }|D ]ü  }|| j                     }| j                  r4|j                  | j                  k7  r|j                  | j                  «      }|| j                     }| j                   rVt#        |t$        j$                  «      sJ d«       ‚|j                  | j                   k7  r7|j                  | j                   «      }n| j&                  r| j(                  |   }||f–— |dz  }| j                  sŒö||k\  sŒü y  y ­w)Nr   z7target_img_mode is specified but target is not an imager	   )r=   rZ   rH   r>   rF   ra   r   r   Úfrom_iterabler   Úiterr   r   ÚmodeÚconvertr   r   r,   r   r/   r0   )r?   Útarget_sample_countÚsample_countÚds_iterÚsampleÚ
input_dataÚtarget_datas          rB   Ú__iter__zReaderHfids.__iter__¨   sa  è ø€ Ø�7‰7ˆ?Ø�O‰OÔØ�‰×Ñ˜$Ÿ*™*×*Ñ*Ô+à"×:Ñ:Ó<ÐØˆà×ÒÜ×)Ñ)¬&°·±«/Ó:‰Gä˜4Ÿ7™7“mˆGØò 	ˆFØ&,¨T¯^©^Ñ&<ˆJØ×"Ò" z§¡¸$×:MÑ:MÒ'MØ'×/Ñ/°×0CÑ0CÓD�
Ø  §¡Ñ1ˆKØ×#Ò#Ü! +¬u¯{©{Ô;ÐvÐ=vÓvÐ;Ø×#Ñ# t×';Ñ';Ò;Ø"-×"5Ñ"5°d×6JÑ6JÓ"K‘KØ×!Ò!Ø"×/Ñ/°Ñ<�Ø˜kÐ)Ò)Ø˜AÑˆLØ×Ó LÐ4GÓ$GÙñ	ùs   ‚FF,Æ"F,Æ(F,c                 ó@   — | j                  «       | j                  z  }|S rE   )ra   r:   )r?   r   s     rB   Ú__len__zReaderHfids.__len__Ä   s!   € Ø×2Ñ2Ó4°t×7GÑ7GÑGˆØÐrC   c                 ó   — J d«       ‚)NzNot supported© )r?   ÚindexÚbasenameÚabsolutes       rB   Ú	_filenamezReaderHfids._filenameÈ   s   € Ø%�oÓ%ˆurC   c                 óä   — | j                   €| j                  «        g }| j                   D ]B  }d|v r|d   }n%d|v r|d   }nd|v r|d   }nd|v r|d   }nJ d«       ‚|j                  |«       ŒD |S )z0 Return all filenames in dataset, overrides baseÚ	file_nameÚfilenamerT   Úimage_idzNo supported name field present)r=   rZ   Úappend)r?   rs   rt   Únamesrj   r   s         rB   Ú	filenameszReaderHfids.filenamesË   s‘   € à�7‰7ˆ?Ø�O‰OÔØˆØ—g‘gò 	ˆFØ˜fÑ$Ø˜kÑ*‘Ø˜vÑ%Ø˜jÑ)‘Ø˜‘Ø˜d‘|‘Ø˜vÑ%Ø˜jÑ)‘à?Ð?Ó?�uØ�L‰L˜Õð	ð ˆrC   )NÚtrainFr	   Fr   é*   NÚimageÚRGBÚlabelÚ NNFrE   )FF)Ú__name__Ú
__module__Ú__qualname__Ústrr   Úboolr_   Údictr#   rH   rJ   rZ   ra   rm   ro   ru   r|   Ú__classcell__)rA   s   @rB   r   r      s4  ø„ ð #'Ø Ø %ØØ"ØØØ(,Ø$Ø"'Ø%Ø#%Ø*.Ø)-Ø&+ñ#P#àðP#ð ˜3‘-ðP#ð ð	P#ð
 ðP#ð ðP#ð ðP#ð ðP#ð ðP#ð   ‘~ðP#ð ðP#ð  ðP#ð ðP#ð !ðP#ð # 3™-ðP#ð  " #™ð!P#ð"  $õ#P#òd!ð *.ñPà! #™óPòdòB'òò8ó&÷rC   r   )!Ú__doc__r]   ÚosÚ	itertoolsr   r   Útypingr   rP   Útorch.distributedÚdistributedr3   ÚPILr   r&   Údatasets.distributedr   Údatasets.splitsr   ÚImportErrorÚeÚprintr   r
   Úreaderr   Úshared_countr   r_   ÚenvironÚgetr%   r   rq   rC   rB   ú<module>rš      s‚   ðñã Û 	ß #Ý ã Ý  Ý ðÛÝ:Ý)õ &Ý Ý %ñ �2—:‘:—>‘>Ð"6¸Ó=Ó>€ô�&õ øð ò Ù	Ð
PÔQØ
€Gûðús   ªA: Á:BÁ?
B	Â	B