Ë
    ÜÓ^j!  ã                   óž   — d dl mZmZmZmZ d dlmZ ddlmZ ddl	m
Z
 ddlmZ  ej                  e«      Zddd	œZg d
¢Z G d„ de
«      ZdgZy)é    )Ú	TokenizerÚdecodersÚpre_tokenizersÚ
processors)ÚUnigramé   )Ú
AddedToken)ÚTokenizersBackend)Úloggingzsentencepiece.bpe.modelztokenizer.json)Ú
vocab_fileÚtokenizer_file)Úar_ARÚcs_CZÚde_DEÚen_XXÚes_XXÚet_EEÚfi_FIÚfr_XXÚgu_INÚhi_INÚit_ITÚja_XXÚkk_KZÚko_KRÚlt_LTÚlv_LVÚmy_MMÚne_NPÚnl_XXÚro_ROÚru_RUÚsi_LKÚtr_TRÚvi_VNÚzh_CNc                   ó  ‡ — e Zd ZU dZeZddgZeZg Z	e
e   ed<   g Ze
e   ed<   	 	 	 	 	 	 	 	 	 	 	 ddeez  e
z  dz  fˆ fd„Zed	efd
„«       Zej&                  ded	dfd„«       Zdededz  dedz  fd„Zd„ Zd„ Zdd„Zded	dfd„Zˆ xZS )ÚMBartTokenizeruC  
    Construct an MBART tokenizer (backed by HuggingFace's *tokenizers* library). Based on
    [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models).

    This tokenizer inherits from [`TokenizersBackend`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    The tokenization method is `<tokens> <eos> <language code>` for source language documents, and `<language code>
    <tokens> <eos>` for target language documents.

    Examples:

    ```python
    >>> from transformers import MBartTokenizer

    >>> tokenizer = MBartTokenizer.from_pretrained(
    ...     "facebook/mbart-large-en-ro", src_lang="en_XX", tgt_lang="ro_RO"
    ... )
    >>> example_english_phrase = " UN Chief Says There Is No Military Solution in Syria"
    >>> expected_translation_romanian = "Åžeful ONU declarÄƒ cÄƒ nu existÄƒ o soluÅ£ie militarÄƒ Ã®n Siria"
    >>> inputs = tokenizer(example_english_phrase, text_target=expected_translation_romanian, return_tensors="pt")
    ```Ú	input_idsÚattention_maskÚprefix_tokensÚsuffix_tokensNÚvocabc                 ól  •— t        |t        «      rt        |dd¬«      n|}t        j	                  «       }|�$|j                  |D �cg c]	  }||vsŒ|‘Œ c}«       |€rt        |«      dft        |«      dft        |«      dft        |«      dfg}|dgz  }t        D ]  }|j                  |df«       Œ |j                  t        |«      df«       || _        t        t        | j                  dd¬«      «      | _
        d | j                  _        t        j                  t        j                  «       t        j                  dd	d¬
«      g«      | j                  _        t#        j                  dd	d¬
«      | j                  _        t'        ‰| �P  d||||||||	|
|dœ
|¤Ž t        D �ci c]  }|| j+                  |«      “Œ c}| _        d| _        dddddœ| _        | j0                  j3                  | j,                  «       | j+                  t        |«      «      | j0                  d<   | j0                  j5                  «       D ��ci c]  \  }}||“Œ
 c}}| _        |	�|	nd| _        | j+                  | j8                  «      | _        |
| _        | j?                  | j8                  «       y c c}w c c}w c c}}w )NTF)ÚlstripÚrstripg        )õ   â–�g       Àr   )Úunk_idÚbyte_fallbackr1   Úalways)ÚreplacementÚprepend_schemeÚsplit)
Ú	bos_tokenÚ	eos_tokenÚ	sep_tokenÚ	cls_tokenÚ	unk_tokenÚ	pad_tokenÚ
mask_tokenÚsrc_langÚtgt_langÚadditional_special_tokensé   r   é   )ú<s>ú<pad>ú</s>ú<unk>ú<mask>r   © ) Ú
isinstanceÚstrr	   ÚFAIRSEQ_LANGUAGE_CODESÚcopyÚextendÚappendÚ_vocabr   r   Ú
_tokenizerÚ
normalizerr   ÚSequenceÚWhitespaceSplitÚ	MetaspaceÚpre_tokenizerr   ÚdecoderÚsuperÚ__init__Úconvert_tokens_to_idsÚlang_code_to_idÚfairseq_offsetÚfairseq_tokens_to_idsÚupdateÚitemsÚfairseq_ids_to_tokensÚ	_src_langÚcur_lang_coder@   Úset_src_lang_special_tokens)Úselfr-   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   ÚkwargsÚ_additional_special_tokensÚtÚ	lang_codeÚkÚvÚ	__class__s                     €úw/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/mbart/tokenization_mbart.pyrY   zMBartTokenizer.__init__@   s¢  ø€ ô KUÐU_ÔadÔJe”Z 
°4ÀÕFÐkuˆ
ä%;×%@Ñ%@Ó%BÐ"Ø$Ð0Ø&×-Ñ-Ø5Ö]�q¸ÐB\Ò9\’Ò]ôð ˆ=ä�Y“ Ð%Ü�Y“ Ð%Ü�Y“ Ð%Ü�Y“ Ð%ð	ˆEð �m�_Ñ$ˆEÜ3ò /�	Ø—‘˜i¨Ð-Õ.ð/à�L‰Lœ#˜j›/¨3Ð/Ô0àˆŒÜ#¤G¨D¯K©KÀÐQVÔ$WÓXˆŒà%)ˆ�‰Ô"ä(6×(?Ñ(?ä×.Ñ.Ó0Ü×(Ñ(°UÈ8Ð[_Ô`ðó)
ˆ�‰Ô%ô #+×"4Ñ"4ÀÐW_ÐgkÔ"lˆ�‰Ôä‰Ñð 	
ØØØØØØØ!ØØØ&@ñ	
ð ò	
ô Oeö 
ØAJˆI�t×1Ñ1°)Ó<Ñ<ò 
ˆÔð  ˆÔð ØØØñ	&
ˆÔ"ð 	×"Ñ"×)Ñ)¨$×*>Ñ*>Ô?Ø/3×/IÑ/IÌ#ÈjË/Ó/Zˆ×"Ñ" 8Ñ,Ø7;×7QÑ7Q×7WÑ7WÓ7Y×%Z©t¨q°! a¨¡dÓ%ZˆÔ"à%-Ð%9™¸wˆŒØ!×7Ñ7¸¿¹ÓGˆÔØ ˆŒØ×(Ñ(¨¯©Õ8ùò} ^ùòV 
ùó &[s   Á	J&ÁJ&Æ%J+ÉJ0Úreturnc                 ó   — | j                   S ©N)ra   ©rd   s    rl   r?   zMBartTokenizer.src_lang”   s   € à�~‰~Ðó    Únew_src_langc                 óH   — || _         | j                  | j                   «       y ro   )ra   rc   )rd   rr   s     rl   r?   zMBartTokenizer.src_lang˜   s   € à%ˆŒØ×(Ñ(¨¯©Õ8rq   Úreturn_tensorsr?   r@   c                 óv   — |�|€t        d«      ‚|| _         | |fd|dœ|¤Ž}| j                  |«      }||d<   |S )zIUsed by translation pipeline, to prepare inputs for the generate functionzATranslation requires a `src_lang` and a `tgt_lang` for this modelT)Úadd_special_tokensrt   Úforced_bos_token_id)Ú
ValueErrorr?   rZ   )rd   Ú
raw_inputsrt   r?   r@   Úextra_kwargsÚinputsÚtgt_lang_ids           rl   Ú_build_translation_inputsz(MBartTokenizer._build_translation_inputs�   sY   € ð Ð˜xÐ/ÜÐ`ÓaÐaØ ˆŒÙ�jÐi°TÈ.ÑiÐ\hÑiˆØ×0Ñ0°Ó:ˆØ(3ˆÐ$Ñ%Øˆrq   c                 ó8   — | j                  | j                  «      S ro   )rc   r?   rp   s    rl   Ú_switch_to_input_modez$MBartTokenizer._switch_to_input_mode©   s   € Ø×/Ñ/°·±Ó>Ð>rq   c                 ór   — | j                   €| j                  | _         | j                  | j                   «      S ro   )r@   ra   Úset_tgt_lang_special_tokensrp   s    rl   Ú_switch_to_target_modez%MBartTokenizer._switch_to_target_mode¬   s,   € Ø�=‰=Ð Ø ŸN™NˆDŒMØ×/Ñ/°·±Ó>Ð>rq   c                 óª  — | j                  |«      | _        g | _        | j                  | j                  g| _        | j                  | j                  «      }| j                  | j                  «      }t        j                  |dgz   |z   |ddgz   |z   t        t        ||z   | j                  | j                  z   «      «      ¬«      | j                  _        y)z_Reset the special tokens to the source lang setting. No prefix and suffix=[eos, src_lang_code].ú$Aú$B©ÚsingleÚpairÚspecial_tokensN©rZ   rb   r+   Úeos_token_idr,   Úconvert_ids_to_tokensr   ÚTemplateProcessingÚlistÚziprQ   Úpost_processor)rd   r?   Úprefix_tokens_strÚsuffix_tokens_strs       rl   rc   z*MBartTokenizer.set_src_lang_special_tokens±   sÅ   € à!×7Ñ7¸ÓAˆÔØˆÔØ"×/Ñ/°×1CÑ1CÐDˆÔà ×6Ñ6°t×7IÑ7IÓJÐØ ×6Ñ6°t×7IÑ7IÓJÐä)3×)FÑ)FØ$¨ vÑ-Ð0AÑAØ" d¨D \Ñ1Ð4EÑEÜ¤Ð$5Ð8IÑ$IÈ4×K]ÑK]Ð`d×`rÑ`rÑKrÓ sÓtô*
ˆ�‰Õ&rq   Úlangc                 óª  — | j                  |«      | _        g | _        | j                  | j                  g| _        | j                  | j                  «      }| j                  | j                  «      }t        j                  |dgz   |z   |ddgz   |z   t        t        ||z   | j                  | j                  z   «      «      ¬«      | j                  _        y)zcReset the special tokens to the target language setting. No prefix and suffix=[eos, tgt_lang_code].r„   r…   r†   NrŠ   )rd   r“   r‘   r’   s       rl   r�   z*MBartTokenizer.set_tgt_lang_special_tokensÀ   sÅ   € à!×7Ñ7¸Ó=ˆÔØˆÔØ"×/Ñ/°×1CÑ1CÐDˆÔà ×6Ñ6°t×7IÑ7IÓJÐØ ×6Ñ6°t×7IÑ7IÓJÐä)3×)FÑ)FØ$¨ vÑ-Ð0AÑAØ" d¨D \Ñ1Ð4EÑEÜ¤Ð$5Ð8IÑ$IÈ4×K]ÑK]Ð`d×`rÑ`rÑKrÓ sÓtô*
ˆ�‰Õ&rq   )NrD   rF   rF   rD   rG   rE   rH   NNN)rm   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr   Úmodelr+   rŽ   ÚintÚ__annotations__r,   rK   ÚdictrY   Úpropertyr?   Úsetterr}   r   r‚   rc   r�   Ú__classcell__)rk   s   @rl   r(   r(   !   s  ø… ñð. *ÐØ$Ð&6Ð7ÐØ€Eà!€M�4˜‘9Ó!Ø!€M�4˜‘9Ó!ð +/ØØØØØØØØØØ"&ñR9à�T‰z˜DÑ  4Ñ'õR9ðh ð˜#ò ó ðð ‡_�_ð9 Sð 9¨Tò 9ó ð9ð
Ø*-ð
Ø9<¸t¹ð
ØORÐUYÉzó
ò?ò?ó

ð
°ð 
¸÷ 
rq   r(   N)Ú
tokenizersr   r   r   r   Útokenizers.modelsr   Útokenization_pythonr	   Útokenization_utils_tokenizersr
   Úutilsr   Ú
get_loggerr•   Úloggerr™   rL   r(   Ú__all__rI   rq   rl   ú<module>r«      s_   ð÷  GÓ FÝ %å -Ý >Ý ð 
ˆ×	Ñ	˜HÓ	%€ð $=ÐP`ÑaÐ ò {Ð ôl
Ð&ô l
ð^ Ð
�rq   