Ë
    ÜÓ^j`9  ã                   óª   — d dl mZmZmZmZmZmZ d dlmZ ddl	m
Z
mZ ddlmZ ddlmZ  ej                   e«      Zddd	œZg d
¢Z G d„ de«      ZdgZy)é    )ÚRegexÚ	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEé   )Ú
AddedTokenÚBatchEncoding)ÚTokenizersBackend)Úloggingzsentencepiece.bpe.modelztokenizer.json)Ú
vocab_fileÚtokenizer_file)ÊÚace_ArabÚace_LatnÚacm_ArabÚacq_ArabÚaeb_ArabÚafr_LatnÚajp_ArabÚaka_LatnÚamh_EthiÚapc_ArabÚarb_ArabÚars_ArabÚary_ArabÚarz_ArabÚasm_BengÚast_LatnÚawa_DevaÚayr_LatnÚazb_ArabÚazj_LatnÚbak_CyrlÚbam_LatnÚban_LatnÚbel_CyrlÚbem_LatnÚben_BengÚbho_DevaÚbjn_ArabÚbjn_LatnÚbod_TibtÚbos_LatnÚbug_LatnÚbul_CyrlÚcat_LatnÚceb_LatnÚces_LatnÚcjk_LatnÚckb_ArabÚcrh_LatnÚcym_LatnÚdan_LatnÚdeu_LatnÚdik_LatnÚdyu_LatnÚdzo_TibtÚell_GrekÚeng_LatnÚepo_LatnÚest_LatnÚeus_LatnÚewe_LatnÚfao_LatnÚpes_ArabÚfij_LatnÚfin_LatnÚfon_LatnÚfra_LatnÚfur_LatnÚfuv_LatnÚgla_LatnÚgle_LatnÚglg_LatnÚgrn_LatnÚguj_GujrÚhat_LatnÚhau_LatnÚheb_HebrÚhin_DevaÚhne_DevaÚhrv_LatnÚhun_LatnÚhye_ArmnÚibo_LatnÚilo_LatnÚind_LatnÚisl_LatnÚita_LatnÚjav_LatnÚjpn_JpanÚkab_LatnÚkac_LatnÚkam_LatnÚkan_KndaÚkas_ArabÚkas_DevaÚkat_GeorÚknc_ArabÚknc_LatnÚkaz_CyrlÚkbp_LatnÚkea_LatnÚkhm_KhmrÚkik_LatnÚkin_LatnÚkir_CyrlÚkmb_LatnÚkon_LatnÚkor_HangÚkmr_LatnÚlao_LaooÚlvs_LatnÚlij_LatnÚlim_LatnÚlin_LatnÚlit_LatnÚlmo_LatnÚltg_LatnÚltz_LatnÚlua_LatnÚlug_LatnÚluo_LatnÚlus_LatnÚmag_DevaÚmai_DevaÚmal_MlymÚmar_DevaÚmin_LatnÚmkd_CyrlÚplt_LatnÚmlt_LatnÚmni_BengÚkhk_CyrlÚmos_LatnÚmri_LatnÚzsm_LatnÚmya_MymrÚnld_LatnÚnno_LatnÚnob_LatnÚnpi_DevaÚnso_LatnÚnus_LatnÚnya_LatnÚoci_LatnÚgaz_LatnÚory_OryaÚpag_LatnÚpan_GuruÚpap_LatnÚpol_LatnÚpor_LatnÚprs_ArabÚpbt_ArabÚquy_LatnÚron_LatnÚrun_LatnÚrus_CyrlÚsag_LatnÚsan_DevaÚsat_BengÚscn_LatnÚshn_MymrÚsin_SinhÚslk_LatnÚslv_LatnÚsmo_LatnÚsna_LatnÚsnd_ArabÚsom_LatnÚsot_LatnÚspa_LatnÚals_LatnÚsrd_LatnÚsrp_CyrlÚssw_LatnÚsun_LatnÚswe_LatnÚswh_LatnÚszl_LatnÚtam_TamlÚtat_CyrlÚtel_TeluÚtgk_CyrlÚtgl_LatnÚtha_ThaiÚtir_EthiÚtaq_LatnÚtaq_TfngÚtpi_LatnÚtsn_LatnÚtso_LatnÚtuk_LatnÚtum_LatnÚtur_LatnÚtwi_LatnÚtzm_TfngÚuig_ArabÚukr_CyrlÚumb_LatnÚurd_ArabÚuzn_LatnÚvec_LatnÚvie_LatnÚwar_LatnÚwol_LatnÚxho_LatnÚydd_HebrÚyor_LatnÚyue_HantÚzho_HansÚzho_HantÚzul_Latnc                   ó¦  ‡ — e Zd ZU dZeZddgZeZg Z	e
e   ed<   g Ze
e   ed<   	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddeeeef   z  dz  dee
e   z  dz  d	edz  fˆ fd
„Zedefd„«       Zej&                  deddfd„«       Zdededz  dedz  fd„Z	 	 	 	 	 	 	 	 d de
e   dede
e   dz  dededz  dedz  dededz  dedefd„Zd„ Zd„ Zd!d„Zdeddfd„Zˆ xZS )"ÚNllbTokenizera	  
    Construct an NLLB tokenizer (backed by HuggingFace's *tokenizers* library). Based on
    [Unigram](https://huggingface.co/docs/tokenizers/python/latest/components.html?highlight=unigram#models).

    This tokenizer inherits from [`TokenizersBackend`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    The tokenization method is `<tokens> <eos> <language code>` for source language documents, and `<language code>
    <tokens> <eos>` for target language documents.

    Examples:

    ```python
    >>> from transformers import NllbTokenizer

    >>> tokenizer = NllbTokenizer.from_pretrained(
    ...     "facebook/nllb-200-distilled-600M", src_lang="eng_Latn", tgt_lang="fra_Latn"
    ... )
    >>> example_english_phrase = " UN Chief Says There Is No Military Solution in Syria"
    >>> expected_translation_french = "Le chef de l'ONU affirme qu'il n'y a pas de solution militaire en Syrie."
    >>> inputs = tokenizer(example_english_phrase, text_target=expected_translation_french, return_tensors="pt")
    ```

    Args:
        vocab_file (`str`, *optional*):
            Path to the vocabulary file.
        bos_token (`str`, *optional*, defaults to `"<s>"`):
            The beginning of sequence token that was used during pretraining.
        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.
        sep_token (`str`, *optional*, defaults to `"</s>"`):
            The separator token.
        cls_token (`str`, *optional*, defaults to `"<s>"`):
            The classifier token.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding.
        mask_token (`str`, *optional*, defaults to `"<mask>"`):
            The token used for masking values.
        src_lang (`str`, *optional*):
            The language to use as source language for translation.
        tgt_lang (`str`, *optional*):
            The language to use as target language for translation.
        legacy_behaviour (`bool`, *optional*, defaults to `False`):
            Whether to use legacy behaviour (suffix pattern) or new behaviour (prefix pattern).
    Ú	input_idsÚattention_maskÚprefix_tokensÚsuffix_tokensNÚvocabÚmergesÚ_spm_precompiled_charsmapc                 ó   •— |�|}n|€t         }t        |	t        «      rt        |	ddd¬«      n|	}	|| _        |€.t        |«      dt        |«      dt        |«      dt        |«      di}|| _        |xs g | _        t        t        | j
                  | j                  d t        |«      dd¬«      «      | _	        |�Vt        j                  t        j                  |«      t        j                  t        d	«      d
«      g«      | j                  _        t!        j"                  ddd¬«      | j                  _        t'        j"                  ddd¬«      | j                  _        t+        ‰| �X  d|||||||
||	||dœ|¤Ž d| _        dddddœ| _        | j0                  j3                  «       D ��ci c]  \  }}||“Œ
 c}}| _        |
�|
nd| _        | j9                  | j6                  «      | _        || _        | j?                  | j6                  «       y c c}}w )NT)Ú
normalizedÚlstripÚspecialr   é   é   r
   F)rá   râ   ÚdropoutÚ	unk_tokenÚfuse_unkÚbyte_fallbackz {2,}ú u   â–�Úalways)ÚreplacementÚprepend_schemeÚsplit)Ú	bos_tokenÚ	eos_tokenÚ	sep_tokenÚ	cls_tokenrë   Ú	pad_tokenÚsrc_langÚtgt_langÚ
mask_tokenÚextra_special_tokensÚlegacy_behaviour)ú<s>ú<pad>ú</s>ú<unk>r?   © ) ÚFAIRSEQ_LANGUAGE_CODESÚ
isinstanceÚstrr   rü   Ú_vocabÚ_mergesr   r	   Ú
_tokenizerr   ÚSequenceÚPrecompiledÚReplacer   Ú
normalizerr   Ú	MetaspaceÚpre_tokenizerr   ÚdecoderÚsuperÚ__init__Úfairseq_offsetÚfairseq_tokens_to_idsÚitemsÚfairseq_ids_to_tokensÚ	_src_langÚconvert_tokens_to_idsÚcur_lang_coderù   Úset_src_lang_special_tokens)Úselfrá   râ   ró   rô   rõ   rö   rë   r÷   rú   rø   rù   rã   Úadditional_special_tokensrû   rü   ÚkwargsÚkÚvÚ	__class__s                      €úu/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/nllb/tokenization_nllb.pyr  zNllbTokenizer.__init__Y   s  ø€ ð*  Ð+Ø(<Ñ%Ø&Ð.Ü(>Ð%ô ˜*¤cÔ*ô �z¨d¸4ÈÕNàð 	ð
 !1ˆÔàˆ=ä�I“ Ü�I“ Ü�I“ Ü�I“ ð	ˆEð ˆŒØ’| ˆŒä#ÜØ—k‘kØ—|‘|ØÜ˜i›.ØØ#ôó	
ˆŒð %Ð0Ü)4×)=Ñ)=ä×+Ñ+Ð,EÓFÜ×'Ñ'¬¨h«¸Ó=ðó*ˆD�O‰OÔ&ô )7×(@Ñ(@ÈUÐckÐswÔ(xˆ�‰Ô%Ü"*×"4Ñ"4ÀÐW_ÐgkÔ"lˆ�‰Ôä‰Ñð 	
ØØØØØØØØØ!Ø!:Ø-ñ	
ð ò	
ð   ˆÔàØØØñ	&
ˆÔ"ð 8<×7QÑ7Q×7WÑ7WÓ7Y×%Z©t¨q°! a¨¡dÓ%ZˆÔ"à%-Ð%9™¸zˆŒØ!×7Ñ7¸¿¹ÓGˆÔØ ˆŒØ×(Ñ(¨¯©Õ8ùó &[s   Æ&H
Úreturnc                 ó   — | j                   S ©N)r  ©r  s    r  rø   zNllbTokenizer.src_lang¸   s   € à�~‰~Ðó    Únew_src_langc                 óH   — || _         | j                  | j                   «       y r"  )r  r  )r  r%  s     r  rø   zNllbTokenizer.src_lang¼   s   € à%ˆŒØ×(Ñ(¨¯©Õ8r$  Úreturn_tensorsrø   rù   c                 óv   — |�|€t        d«      ‚|| _         | |fd|dœ|¤Ž}| j                  |«      }||d<   |S )zIUsed by translation pipeline, to prepare inputs for the generate functionzATranslation requires a `src_lang` and a `tgt_lang` for this modelT)Úadd_special_tokensr'  Úforced_bos_token_id)Ú
ValueErrorrø   r  )r  Ú
raw_inputsr'  rø   rù   Úextra_kwargsÚinputsÚtgt_lang_ids           r  Ú_build_translation_inputsz'NllbTokenizer._build_translation_inputsÁ   sY   € ð Ð˜xÐ/ÜÐ`ÓaÐaØ ˆŒÙ�jÐi°TÈ.ÑiÐ\hÑiˆØ×0Ñ0°Ó:ˆØ(3ˆÐ$Ñ%Øˆr$  Ú	src_textsÚ	tgt_textsÚ
max_lengthÚmax_target_lengthÚpaddingÚ
truncationc
           	      óÚ   — || _         || _        |€| j                  } | |fd||||	dœ|
¤Ž}|€|S |€|}| j                  «         | |fd||||	dœ|
¤Ž}|d   |d<   | j	                  «        |S )NT)r)  r'  r3  r5  r6  )r)  r'  r5  r3  r6  rÝ   Úlabels)rø   rù   Úmodel_max_lengthÚ_switch_to_target_modeÚ_switch_to_input_mode)r  r1  rø   r2  rù   r3  r4  r5  r'  r6  r  Úmodel_inputsr8  s                r  Úprepare_seq2seq_batchz#NllbTokenizer.prepare_seq2seq_batchÍ   sÈ   € ð !ˆŒØ ˆŒàÐØ×.Ñ.ˆJáØð
à#Ø)Ø!ØØ!ñ
ð ñ
ˆð ÐØÐð Ð$Ø *Ðð 	×#Ñ#Ô%ÙØð
à#Ø)ØØ(Ø!ñ
ð ñ
ˆð "(¨Ñ!4ˆ�XÑð 	×"Ñ"Ô$àÐr$  c                 ó8   — | j                  | j                  «      S r"  )r  rø   r#  s    r  r;  z#NllbTokenizer._switch_to_input_mode  s   € Ø×/Ñ/°·±Ó>Ð>r$  c                 ór   — | j                   €| j                  | _         | j                  | j                   «      S r"  )rù   r  Úset_tgt_lang_special_tokensr#  s    r  r:  z$NllbTokenizer._switch_to_target_mode  s,   € Ø�=‰=Ð Ø ŸN™NˆDŒMØ×/Ñ/°·±Ó>Ð>r$  c                 óv  — | j                  |«      | _        |}| j                  r‹g | _        | j                  | j                  g| _        t        j                  d| j                  |gdd| j                  |g| j                  | j                  f|| j                  fg¬«      | j                  _
        y| j                  g| _        | j                  g| _        t        j                  |d| j                  g|dd| j                  g| j                  | j                  f|| j                  fg¬«      | j                  _
        y)zÄReset the special tokens to the source lang setting.
        - In legacy mode: No prefix and suffix=[eos, src_lang_code].
        - In default mode: Prefix=[src_lang_code], suffix = [eos]
        ú$Aú$B©ÚsingleÚpairÚspecial_tokensN©r  r  rü   rß   Úeos_token_idrà   r   ÚTemplateProcessingrô   r  Úpost_processor)r  rø   Úlang_code_tokens      r  r  z)NllbTokenizer.set_src_lang_special_tokens  s  € ð
 "×7Ñ7¸ÓAˆÔØ"ˆà× Ò Ø!#ˆDÔØ"&×"3Ñ"3°T×5GÑ5GÐ!HˆDÔÜ-7×-JÑ-JØ˜dŸn™n¨oÐ>Ø˜D $§.¡.°/ÐBØ!%§¡°×1BÑ1BÐ CÀoÐW[×WiÑWiÐEjÐkô.ˆD�O‰OÕ*ð #'×"4Ñ"4Ð!5ˆDÔØ"&×"3Ñ"3Ð!4ˆDÔÜ-7×-JÑ-JØ'¨¨t¯~©~Ð>Ø% t¨T°4·>±>ÐBØ!%§¡°×1BÑ1BÐ CÀoÐW[×WiÑWiÐEjÐkô.ˆD�O‰OÕ*r$  Úlangc                 óv  — | j                  |«      | _        |}| j                  r‹g | _        | j                  | j                  g| _        t        j                  d| j                  |gdd| j                  |g| j                  | j                  f|| j                  fg¬«      | j                  _
        y| j                  g| _        | j                  g| _        t        j                  |d| j                  g|dd| j                  g| j                  | j                  f|| j                  fg¬«      | j                  _
        y)zÄReset the special tokens to the target lang setting.
        - In legacy mode: No prefix and suffix=[eos, tgt_lang_code].
        - In default mode: Prefix=[tgt_lang_code], suffix = [eos]
        rB  rC  rD  NrH  )r  rM  rL  s      r  r@  z)NllbTokenizer.set_tgt_lang_special_tokens$  s  € ð
 "×7Ñ7¸Ó=ˆÔØˆà× Ò Ø!#ˆDÔØ"&×"3Ñ"3°T×5GÑ5GÐ!HˆDÔÜ-7×-JÑ-JØ˜dŸn™n¨oÐ>Ø˜D $§.¡.°/ÐBØ!%§¡°×1BÑ1BÐ CÀoÐW[×WiÑWiÐEjÐkô.ˆD�O‰OÕ*ð #'×"4Ñ"4Ð!5ˆDÔØ"&×"3Ñ"3Ð!4ˆDÔÜ-7×-JÑ-JØ'¨¨t¯~©~Ð>Ø% t¨T°4·>±>ÐBØ!%§¡°×1BÑ1BÐ CÀoÐW[×WiÑWiÐEjÐkô.ˆD�O‰OÕ*r$  )NNrý   rÿ   rÿ   rý   r   rþ   z<mask>NNNNNF)r?   NrI   NNÚlongestNT)r   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr	   Úmodelrß   ÚlistÚintÚ__annotations__rà   r  Údictr  Úpropertyrø   Úsetterr0  Úboolr   r=  r;  r:  r  r@  Ú__classcell__)r  s   @r  rÜ   rÜ   !   sÚ  ø… ñ.ð` *ÐØ$Ð&6Ð7ÐØ€Eà!€M�4˜‘9Ó!Ø!€M�4˜‘9Ó!ð .2Ø)-ØØØØØØØØØØ04Ø"&Ø!Øñ!]9à�T˜#˜s˜(‘^Ñ# dÑ*ð]9ð �d˜3‘i‘ $Ñ&ð]9ð $'¨¡:õ]9ð~ ð˜#ò ó ðð ‡_�_ð9 Sð 9¨Tò 9ó ð9ð
Ø*-ð
Ø9<¸t¹ð
ØORÐUYÉzó
ð #Ø&*Ø"Ø!%Ø(,Ø Ø%)Øñ4à˜‘9ð4ð ð4ð ˜‘9˜tÑ#ð	4ð
 ð4ð ˜$‘Jð4ð  ™:ð4ð ð4ð ˜d™
ð4ð ð4ð 
ó4òl?ò?ó
ð2°ð ¸÷ r$  rÜ   N)Ú
tokenizersr   r   r   r   r   r   Útokenizers.modelsr	   Útokenization_pythonr   r   Útokenization_utils_tokenizersr   Úutilsr   Ú
get_loggerrP  ÚloggerrT  r  rÜ   Ú__all__r  r$  r  ú<module>rh     s_   ð÷  [× ZÝ !ç <Ý >Ý ð 
ˆ×	Ñ	˜HÓ	%€ð $=ÐP`ÑaÐ ò R&Ð ôZÐ%ô Zðz Ð
�r$  