Ë
    ÛÓ^j2  ã                   óÐ   — d dl Z d dlZd dlZd dlmZ ddlmZmZ  ej                  e«      Z	e G d„ d«      «       Z
 ed¬«       G d	„ d
«      «       Z G d„ d«      Z G d„ de«      Zy)é    N)Ú	dataclassé   )Úis_torch_availableÚloggingc                   óT   — e Zd ZU dZeed<   eed<   dZedz  ed<   dZedz  ed<   d„ Zy)ÚInputExamplea5  
    A single training/test example for simple sequence classification.

    Args:
        guid: Unique id for the example.
        text_a: string. The untokenized text of the first sequence. For single
            sequence tasks, only this sequence must be specified.
        text_b: (Optional) string. The untokenized text of the second sequence.
            Only must be specified for sequence pair tasks.
        label: (Optional) string. The label of the example. This should be
            specified for train and dev examples, but not for test examples.
    ÚguidÚtext_aNÚtext_bÚlabelc                 ó\   — t        j                  t        j                  | «      d¬«      dz   S )ú*Serializes this instance to a JSON string.é   )Úindentú
©ÚjsonÚdumpsÚdataclassesÚasdict©Úselfs    úm/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/data/processors/utils.pyÚto_json_stringzInputExample.to_json_string/   s#   € ä�z‰zœ+×,Ñ,¨TÓ2¸1Ô=ÀÑDÐDó    )	Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚstrÚ__annotations__r   r   r   © r   r   r   r      s5   … ñð ƒIØƒKØ€FˆC�$‰JÓØ€Eˆ3�‰:ÓóEr   r   T)Úfrozenc                   óv   — e Zd ZU dZee   ed<   dZee   dz  ed<   dZee   dz  ed<   dZ	ee
z  dz  ed<   d„ Zy)ÚInputFeaturesa¿  
    A single set of features of data. Property names are the same names as the corresponding inputs to a model.

    Args:
        input_ids: Indices of input sequence tokens in the vocabulary.
        attention_mask: Mask to avoid performing attention on padding token indices.
            Mask values selected in `[0, 1]`: Usually `1` for tokens that are NOT MASKED, `0` for MASKED (padded)
            tokens.
        token_type_ids: (Optional) Segment token indices to indicate first and second
            portions of the inputs. Only some models use them.
        label: (Optional) Label corresponding to the input. Int for classification problems,
            float for regression problems.
    Ú	input_idsNÚattention_maskÚtoken_type_idsr   c                 óX   — t        j                  t        j                  | «      «      dz   S )r   r   r   r   s    r   r   zInputFeatures.to_json_stringI   s!   € ä�z‰zœ+×,Ñ,¨TÓ2Ó3°dÑ:Ð:r   )r   r   r   r   ÚlistÚintr!   r'   r(   r   Úfloatr   r"   r   r   r%   r%   4   sQ   … ñð �C‰yÓØ'+€N�D˜‘I Ñ$Ó+Ø'+€N�D˜‘I Ñ$Ó+Ø $€Eˆ3�‰;˜ÑÓ$ó;r   r%   c                   óF   — e Zd ZdZd„ Zd„ Zd„ Zd„ Zd„ Zd„ Z	e
d
d	„«       Zy)ÚDataProcessorzEBase class for data converters for sequence classification data sets.c                 ó   — t        «       ‚)z·
        Gets an example from a dict.

        Args:
            tensor_dict: Keys and values should match the corresponding Glue
                tensorflow_dataset examples.
        ©ÚNotImplementedError)r   Útensor_dicts     r   Úget_example_from_tensor_dictz*DataProcessor.get_example_from_tensor_dictQ   s   € ô "Ó#Ð#r   c                 ó   — t        «       ‚)z8Gets a collection of [`InputExample`] for the train set.r0   ©r   Údata_dirs     r   Úget_train_examplesz DataProcessor.get_train_examples[   ó   € ä!Ó#Ð#r   c                 ó   — t        «       ‚)z6Gets a collection of [`InputExample`] for the dev set.r0   r5   s     r   Úget_dev_exampleszDataProcessor.get_dev_examples_   r8   r   c                 ó   — t        «       ‚)z7Gets a collection of [`InputExample`] for the test set.r0   r5   s     r   Úget_test_exampleszDataProcessor.get_test_examplesc   r8   r   c                 ó   — t        «       ‚)z*Gets the list of labels for this data set.r0   r   s    r   Ú
get_labelszDataProcessor.get_labelsg   r8   r   c                 ó”   — t        | j                  «       «      dkD  r+| j                  «       t        |j                  «         |_        |S )z¦
        Some tensorflow_datasets datasets are not formatted the same way the GLUE datasets are. This method converts
        examples to the correct format.
        é   )Úlenr>   r+   r   )r   Úexamples     r   Útfds_mapzDataProcessor.tfds_mapk   s9   € ô
 ˆt�‰Ó Ó! AÒ%Ø ŸO™OÓ-¬c°'·-±-Ó.@ÑAˆGŒMØˆr   Nc                 óŒ   — t        |dd¬«      5 }t        t        j                  |d|¬«      «      cddd«       S # 1 sw Y   yxY w)z!Reads a tab separated value file.Úrz	utf-8-sig)Úencodingú	)Ú	delimiterÚ	quotecharN)Úopenr*   ÚcsvÚreader)ÚclsÚ
input_filerI   Úfs       r   Ú	_read_tsvzDataProcessor._read_tsvt   s@   € ô �*˜c¨KÔ8ð 	L¸AÜœŸ
™
 1°À	ÔJÓK÷	L÷ 	Lò 	Lús	   �!:ºA©N)r   r   r   r   r3   r7   r:   r<   r>   rC   ÚclassmethodrP   r"   r   r   r.   r.   N   s9   „ ÙOò$ò$ò$ò$ò$òð òLó ñLr   r.   c                   ó|   — e Zd ZdZdd„Zd„ Zd„ Ze	 dd„«       Zedd„«       Z		 	 	 	 	 	 	 dd„Z
	 dd	„Z	 	 	 	 	 dd
„Zy)Ú%SingleSentenceClassificationProcessorz@Generic processor for a single sentence classification data set.Nc                 óL   — |€g n|| _         |€g n|| _        || _        || _        y rQ   )ÚlabelsÚexamplesÚmodeÚverbose)r   rV   rW   rX   rY   s        r   Ú__init__z.SingleSentenceClassificationProcessor.__init__~   s+   € Ø"˜N‘b°ˆŒØ&Ð.™°HˆŒØˆŒ	Øˆ�r   c                 ó,   — t        | j                  «      S rQ   )rA   rW   r   s    r   Ú__len__z-SingleSentenceClassificationProcessor.__len__„   s   € Ü�4—=‘=Ó!Ð!r   c                 óˆ   — t        |t        «      r$t        | j                  | j                  |   ¬«      S | j                  |   S )N)rV   rW   )Ú
isinstanceÚslicerT   rV   rW   )r   Úidxs     r   Ú__getitem__z1SingleSentenceClassificationProcessor.__getitem__‡   s9   € Ü�cœ5Ô!Ü8ÀÇÁÐVZ×VcÑVcÐdgÑVhÔiÐiØ�}‰}˜SÑ!Ð!r   c           
      óH   —  | di |¤Ž}|j                  ||||||dd¬«       |S )NT)Ú
split_nameÚcolumn_labelÚcolumn_textÚ	column_idÚskip_first_rowÚoverwrite_labelsÚoverwrite_examplesr"   )Úadd_examples_from_csv)	rM   Ú	file_namerc   rd   re   rf   rg   ÚkwargsÚ	processors	            r   Úcreate_from_csvz5SingleSentenceClassificationProcessor.create_from_csvŒ   sB   € ñ ‘M˜&‘Mˆ	Ø×'Ñ'ØØ!Ø%Ø#ØØ)Ø!Ø#ð 	(ô 		
ð Ðr   c                 ó<   —  | di |¤Ž}|j                  ||¬«       |S )N)rV   r"   )Úadd_examples)rM   Útexts_or_text_and_labelsrV   rl   rm   s        r   Úcreate_from_examplesz:SingleSentenceClassificationProcessor.create_from_examples�   s'   € á‘M˜&‘Mˆ	Ø×ÑÐ7ÀÐÔGØÐr   c	                 óX  — | j                  |«      }	|r|	dd  }	g }
g }g }t        |	«      D ]i  \  }}|
j                  ||   «       |j                  ||   «       |�|j                  ||   «       ŒE|r|› d|› �n
t        |«      }|j                  |«       Œk | j	                  |
||||¬«      S )Nr@   ú-)rh   ri   )rP   Ú	enumerateÚappendr    rp   )r   rk   rc   rd   re   rf   rg   rh   ri   ÚlinesÚtextsrV   ÚidsÚiÚliner	   s                   r   rj   z;SingleSentenceClassificationProcessor.add_examples_from_csv£   sÊ   € ð —‘˜yÓ)ˆÙØ˜!˜"�IˆEØˆØˆØˆÜ  Ó'ò 	!‰GˆAˆtØ�L‰L˜˜kÑ*Ô+Ø�M‰M˜$˜|Ñ,Ô-ØÐ$Ø—
‘
˜4 	™?Õ+á.8˜*˜ Q q cÑ*¼cÀ!»f�Ø—
‘
˜4Õ ð	!ð × Ñ Ø�6˜3Ð1AÐVhð !ó 
ð 	
r   c           	      ó  — |�:t        |«      t        |«      k7  r#t        dt        |«      › dt        |«      › �«      ‚|�:t        |«      t        |«      k7  r#t        dt        |«      › dt        |«      › �«      ‚|€d gt        |«      z  }|€d gt        |«      z  }g }t        «       }t        |||«      D ]U  \  }}	}
t	        |t
        t        f«      r|	€|\  }}	n|}|j                  |	«       |j                  t        |
|d |	¬«      «       ŒW |r|| _
        n| j                  j                  |«       |rt        |«      | _        | j                  S t        t        | j                  «      j                  |«      «      | _        | j                  S )Nz(Text and labels have mismatched lengths z and z%Text and ids have mismatched lengths )r	   r
   r   r   )rA   Ú
ValueErrorÚsetÚzipr^   Útupler*   Úaddrv   r   rW   ÚextendrV   Úunion)r   rq   rV   ry   rh   ri   rW   Úadded_labelsÚtext_or_text_and_labelr   r	   Útexts               r   rp   z2SingleSentenceClassificationProcessor.add_examplesÁ   s–  € ð Ð¤#Ð&>Ó"?Ä3ÀvÃ;Ò"NÜØ:¼3Ð?WÓ;XÐ:YÐY^Ô_bÐciÓ_jÐ^kÐlóð ð ˆ?œsÐ#;Ó<ÄÀCÃÒHÜÐDÄSÐIaÓEbÐDcÐchÔilÐmpÓiqÐhrÐsÓtÐtØˆ;Ø�&œ3Ð7Ó8Ñ8ˆCØˆ>Ø�VœcÐ":Ó;Ñ;ˆFØˆÜ“uˆÜ36Ð7OÐQWÐY\Ó3]ò 	\Ñ/Ð" E¨4ÜÐ0´5¼$°-Ô@ÀUÀ]Ø4‘�‘eà-�Ø×Ñ˜UÔ#Ø�O‰OœL¨d¸4ÈÐTYÔZÕ[ð	\ñ Ø$ˆD�Mà�M‰M× Ñ  Ô*ñ Ü˜|Ó,ˆDŒKð �}‰}Ðô œs 4§;¡;Ó/×5Ñ5°lÓCÓDˆDŒKà�}‰}Ðr   c                 ó	  — |€|j                   }t        | j                  «      D ��ci c]  \  }}||“Œ
 }	}}g }
t        | j                  «      D ]h  \  }}|dz  dk(  rt        j                  d|› �«       |j                  |j                  dt        ||j                   «      ¬«      }|
j                  |«       Œj t        d„ |
D «       «      }g }t        t        |
| j                  «      «      D �]?  \  }\  }}|dz  dk(  r.t        j                  d|› d	t        | j                  «      › �«       |rd
ndgt        |«      z  }|t        |«      z
  }|r|g|z  |z   }|rdnd
g|z  |z   }n||g|z  z   }||rdnd
g|z  z   }t        |«      |k7  rt        dt        |«      › d|› �«      ‚t        |«      |k7  rt        dt        |«      › d|› �«      ‚| j                  dk(  r|	|j                     }n:| j                  dk(  rt!        |j                  «      }nt        | j                  «      ‚|dk  rå| j"                  rÙt        j                  d«       t        j                  d|j$                  › �«       t        j                  ddj'                  |D �cg c]  }t)        |«      ‘Œ c}«      › �«       t        j                  ddj'                  |D �cg c]  }t)        |«      ‘Œ c}«      › �«       t        j                  d|j                  › d|› d�«       |j                  t+        |||¬«      «       �ŒB |€|S |dk(  �rt-        «       st/        d«      ‚ddl}ddlm} |j7                  |D �cg c]  }|j8                  ‘Œ c}|j:                  ¬«      }
|j7                  |D �cg c]  }|j<                  ‘Œ c}|j:                  ¬«      }| j                  dk(  r6|j7                  |D �cg c]  }|j                  ‘Œ c}|j:                  ¬«      }nD| j                  dk(  r5|j7                  |D �cg c]  }|j                  ‘Œ c}|j                   ¬«      } ||
|«      }|S t        d«      ‚c c}}w c c}w c c}w c c}w c c}w c c}w c c}w )aÕ  
        Convert examples in a list of `InputFeatures`

        Args:
            tokenizer: Instance of a tokenizer that will tokenize the examples
            max_length: Maximum example length
            pad_on_left: If set to `True`, the examples will be padded on the left rather than on the right (default)
            pad_token: Padding token
            mask_padding_with_zero: If set to `True`, the attention mask will be filled by `1` for actual values
                and by `0` for padded values. If set to `False`, inverts it (`1` for padded values, `0` for actual
                values)

        Returns:
            Will return a list of task-specific `InputFeatures` which can be fed to the model.

        Ni'  r   zTokenizing example T)Úadd_special_tokensÚ
max_lengthc              3   ó2   K  — | ]  }t        |«      –— Œ y ­wrQ   )rA   )Ú.0r&   s     r   ú	<genexpr>zESingleSentenceClassificationProcessor.get_features.<locals>.<genexpr>  s   è ø€ ÒI¨iœ3˜yŸ>ÑIùs   ‚zWriting example ú/r@   zError with input length z vs ÚclassificationÚ
regressioné   z*** Example ***zguid: zinput_ids: ú zattention_mask: zlabel: z (id = ú))r&   r'   r   Úptz8return_tensors set to 'pt' but PyTorch can't be imported)ÚTensorDataset)Údtypez)return_tensors should be `'pt'` or `None`)Úmax_lenru   rV   rW   ÚloggerÚinfoÚencoder
   Úminrv   Úmaxr   rA   r}   rX   r   r,   rY   r	   Újoinr    r%   r   ÚRuntimeErrorÚtorchÚtorch.utils.datar”   Útensorr&   Úlongr'   )r   Ú	tokenizerr‰   Úpad_on_leftÚ	pad_tokenÚmask_padding_with_zeroÚreturn_tensorsrz   r   Ú	label_mapÚall_input_idsÚex_indexrB   r&   Úbatch_lengthÚfeaturesr'   Úpadding_lengthÚxrž   r”   rO   Úall_attention_maskÚ
all_labelsÚdatasets                            r   Úget_featuresz2SingleSentenceClassificationProcessor.get_featuresæ   s,  € ð2 ÐØ"×*Ñ*ˆJä.7¸¿¹Ó.D×E¡( ! U�U˜A‘XÐEˆ	ÑEàˆÜ!*¨4¯=©=Ó!9ò 		,ÑˆH�gØ˜%Ñ 1Ò$Ü—‘Ð1°(°Ð<Ô=à!×(Ñ(Ø—‘Ø#'Ü˜z¨9×+<Ñ+<Ó=ð )ó ˆIð
 × Ñ  Õ+ð		,ô ÑI¸=ÔIÓIˆàˆÜ.7¼¸MÈ4Ï=É=Ó8YÓ.Zó #	lÑ*ˆHÑ*�y 'Ø˜%Ñ 1Ò$Ü—‘Ð.¨x¨j¸¼#¸d¿m¹mÓ:LÐ9MÐNÔOñ $:™a¸qÐAÄCÈ	ÃNÑRˆNð *¬C°	«NÑ:ˆNÙØ'˜[¨>Ñ9¸YÑF�	Ù(>¡1ÀAÐ"FÈÑ"WÐ[iÑ!i‘à%¨)¨°~Ñ)EÑF�	Ø!/Ñ9O±AÐUVÐ3WÐZhÑ3hÑ!i�ä�9‹~ Ò-Ü Ð#;¼CÀ	»NÐ;KÈ4ÐP\È~Ð!^Ó_Ð_Ü�>Ó" lÒ2Ü Ð#;¼CÀÓ<OÐ;PÐPTÐUaÐTbÐ!cÓdÐdà�y‰yÐ,Ò,Ø! '§-¡-Ñ0‘Ø—‘˜lÒ*Ü˜gŸm™mÓ,‘ä  §¡Ó+Ð+à˜!Š| §¢Ü—‘Ð-Ô.Ü—‘˜f W§\¡\ NÐ3Ô4Ü—‘˜k¨#¯(©(ÀIÖ3N¸q´C¸µFÒ3NÓ*OÐ)PÐQÔRÜ—‘Ð.¨s¯x©xÈÖ8XÀA¼¸Q½Ò8XÓ/YÐ.ZÐ[Ô\Ü—‘˜g g§m¡m _°G¸E¸7À!ÐDÔEà�O‰OœM°IÈnÐdiÔjÖkðG#	lðJ Ð!ØˆOØ˜tÓ#Ü%Ô'Ü"Ð#]Ó^Ð^ÛÝ6à!ŸL™L¸xÖ)H¸!¨!¯+«+Ò)HÐPU×PZÑPZ˜LÓ[ˆMØ!&§¡ÈÖ.RÀA¨q×/?Ó/?Ò.RÐZ_×ZdÑZd Ó!eÐØ�y‰yÐ,Ò,Ø"Ÿ\™\¸HÖ*E°q¨1¯7«7Ò*EÈUÏZÉZ˜\ÓX‘
Ø—‘˜lÒ*Ø"Ÿ\™\¸HÖ*E°q¨1¯7«7Ò*EÈUÏ[É[˜\ÓY�
á# MÐ3EÀzÓRˆGØˆNäÐHÓIÐIùóQ Fùò` 4OùÚ8Xùò *IùÚ.Rùâ*Eùâ*Es)   §Q"ÊQ(Ë
Q-Í'Q2ÎQ7Ï Q<Ð%R)NNrŽ   F)Ú r   r@   NFrQ   )r²   r   r@   NFFF)NNFF)NFr   TN)r   r   r   r   rZ   r\   ra   rR   rn   rr   rj   rp   r±   r"   r   r   rT   rT   {   s€   „ ÙJóò"ò"ð
 àejòó ðð  òó ðð ØØØØØØ ó
ð> kpó#ðP ØØØ#ØôdJr   rT   )rK   r   r   r   Úutilsr   r   Ú
get_loggerr   r—   r   r%   r.   rT   r"   r   r   ú<module>rµ      s�   ðó  Û Û Ý !ç 0ð 
ˆ×	Ñ	˜HÓ	%€ð ÷Eð Eó ðEñ0 �$Ô÷;ð ;ó ð;÷2*Lñ *LôZOJ¨Mõ OJr   