Ë
    ÜÓ^jR=  ã                   óØ   — d dl mZ ddlmZ ddlmZ ddlmZ ddlm	Z	 ddl
mZ  e	j                  e«      ZdZd	d
dœZg d¢g d¢dœZddddddddœZ ed¬«       G d„ de«      «       ZdgZy)é    )ÚAnyé   )ÚBatchEncoding)Ú
AddedToken)ÚSentencePieceBackend)Úlogging)Úrequiresu   â–�zsentencepiece.bpe.modelztokenizer.json)Ú
vocab_fileÚtokenizer_file)Ú__java__Ú
__python__Ú	__en_XX__)r   r   r   Ú__javascript__Ú__php__Ú__ruby__Ú__go__)ÚbaseÚmultir   r   r   r   r   r   r   )ÚjavaÚpythonÚen_XXÚ
javascriptÚphpÚrubyÚgo)Úsentencepiece)Úbackendsc                   óŠ  ‡ — e Zd ZU dZeZddgZg Zee	   e
d<   g Zee	   e
d<   	 	 	 	 	 	 	 	 	 	 	 	 	 ddeeef   dz  fˆ fd„Zed	„ «       Zd
„ Zedefd„«       Zej(                  deddfd„«       Zdededz  dedz  fd„Zd„ Zd„ Z	 	 	 d dee   dedee   dz  dedef
ˆ fd„Zd„ Zd„ Zd!d„Zdeddfd„Zdedefd„Zd"ˆ fd„	Zˆ xZ S )#ÚPLBartTokenizeraÕ  
    Construct an PLBART tokenizer.

    Adapted from [`RobertaTokenizer`] and [`XLNetTokenizer`]. Based on
    [SentencePiece](https://github.com/google/sentencepiece).

    The tokenization method is `<tokens> <eos> <language code>` for source language documents, and `<language code>
    <tokens> <eos>` for target language documents.

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        src_lang (`str`, *optional*):
            A string representing the source language.
        tgt_lang (`str`, *optional*):
            A string representing the target language.
        bos_token (`str`, *optional*, defaults to `"<s>"`):
            The start of sequence token.
        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.
        sep_token (`str`, *optional*, defaults to `"</s>"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        cls_token (`str`, *optional*, defaults to `"<s>"`):
            The cls token, which is a special token used as the first token for all tasks.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        mask_token(`str`, *optional*, defaults to `"<mask>"`):
            The token used for masking values. This is the token used when training this model with masking tasks. This
            is only used in the `"base"` tokenizer type. For `"multi"` tokenizer, masking is never done for the
            downstream tasks.
        language_codes (`str`, *optional*, defaults to `"base"`):
            What language codes to use. Should be one of `"base"` or `"multi"`.
        sp_model_kwargs (`dict`, *optional*):
            Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
            SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
            to set:
            - `enable_sampling`: Enable subword regularization.
            - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.
              - `nbest_size = {0,1}`: No sampling is performed.
              - `nbest_size > 1`: samples from the nbest_size results.
              - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
                using forward-filtering-and-backward-sampling algorithm.
            - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
              BPE-dropout.

    Examples:

    ```python
    >>> from transformers import PLBartTokenizer

    >>> tokenizer = PLBartTokenizer.from_pretrained("uclanlp/plbart-python-en_XX", src_lang="python", tgt_lang="en_XX")
    >>> example_python_phrase = "def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])"
    >>> expected_translation_english = "Returns the maximum value of a b c."
    >>> inputs = tokenizer(example_python_phrase, text_target=expected_translation_english, return_tensors="pt")
    ```Ú	input_idsÚattention_maskÚprefix_tokensÚsuffix_tokensNÚsp_model_kwargsc                 ór  •— t        |t        «      rt        |dd¬«      n|}|€i n|| _        | j	                  |
«      }
| j	                  |«      }|	| _        t        | j
                     }|| _        i | _        i | _	        dddddœ| _
        | j                  j                  «       D ��ci c]  \  }}||“Œ
 c}}| _        d| _        t        |«      }|�$|j                  |D �cg c]	  }||vsŒ|‘Œ c}«       t!        ‰| �D  d i d	|“d
|“d|“d|“d|“d|“d|“d|“d|
“d|“d|“d| j                  “d|“d|	“dd“dd“|¤Ž t%        | j&                  «      | _        t+        |«      D ��ci c]"  \  }}|| j(                  |z   | j                  z   “Œ$ c}}| _        | j                  j                  «       D ��ci c]  \  }}||“Œ
 c}}| _	        dddddœ| _
        | j
                  dk(  rEt%        | j&                  «      t%        | j                  «      z   | j                  z   | j                  d<   | j                  j-                  | j                  «       | j                  j                  «       D ��ci c]  \  }}||“Œ
 c}}| _        h d£}|j-                  t        | j
                     «       d}|D ]?  }| j.                  j1                  |d «      }|€Œ"| j2                  j1                  |d «       d}ŒA |r | j5                  «        | j7                  «        d}| j.                  j                  «       D ]3  \  }}|| j2                  v rŒt        |dddd¬«      | j2                  |<   d}Œ5 |r | j5                  «        | j7                  «        | j
                  dk(  r>|
| _        | j8                  �| j                  | j8                     n| j8                  | _        n)|
�|
nd| _        | j                  | j8                     | _        || _        | j?                  | j8                  «       y c c}}w c c}w c c}}w c c}}w c c}}w )!NTF)ÚlstripÚrstripr   é   é   r   )ú<s>ú<pad>ú</s>ú<unk>r
   Ú	bos_tokenÚ	eos_tokenÚ	unk_tokenÚ	sep_tokenÚ	cls_tokenÚ	pad_tokenÚ
mask_tokenÚsrc_langÚtgt_langÚadditional_special_tokensr$   Úclean_up_tokenization_spacesÚlanguage_codesÚspecial_tokens_patternÚprefix_suffixÚtoken_type_ids_patternÚ	all_zerosr   ú<mask>>   r*   r,   r+   r-   r>   )ÚspecialÚ
normalizedr&   r'   r   © ) Ú
isinstanceÚstrr   r$   Ú!_convert_lang_code_special_formatr9   ÚFAIRSEQ_LANGUAGE_CODESr
   Úlang_code_to_idÚid_to_lang_codeÚfairseq_tokens_to_idsÚitemsÚfairseq_ids_to_tokensÚfairseq_offsetÚlistÚextendÚsuperÚ__init__ÚlenÚsp_modelÚsp_model_sizeÚ	enumerateÚupdateÚ_added_tokens_encoderÚpopÚ_added_tokens_decoderÚ_update_trieÚ_update_total_vocab_sizeÚ	_src_langÚcur_lang_code_idr6   Úset_src_lang_special_tokens)Úselfr
   r.   r/   r1   r2   r0   r3   r4   r9   r5   r6   r$   r7   r8   ÚkwargsÚfairseq_language_codesÚkÚvÚ_additional_special_tokensÚtÚiÚcodeÚreserved_tokensÚremovedÚtokenÚidxÚsyncedÚ	__class__s                               €úy/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/plbart/tokenization_plbart.pyrO   zPLBartTokenizer.__init__t   sl  ø€ ô& KUÐU_ÔadÔJe”Z 
°4ÀÕFÐkuˆ
à%4Ð%<™rÀ/ˆÔØ×9Ñ9¸(ÓCˆØ×9Ñ9¸(ÓCˆØ,ˆÔÜ!7¸×8KÑ8KÑ!LÐð %ˆŒØ!ˆÔØ!ˆÔØ-.¸ÀAÐPQÑ%RˆÔ"Ø7;×7QÑ7Q×7WÑ7WÓ7Y×%Z©t¨q°! a¨¡dÓ%ZˆÔ"ØˆÔÜ%)Ð*@Ó%AÐ"à$Ð0Ø&×-Ñ-Ø5Ö]�q¸ÐB\Ò9\’Ò]ôô 	‰Ñò 	
Ù!ð	
áð	
ñ  ð	
ñ  ð		
ñ
  ð	
ñ  ð	
ñ  ð	
ñ "ð	
ñ ð	
ñ ð	
ñ 'Að	
ð !×0Ò0ð	
ñ *Fð	
ñ *ð	
ñ $3ð	
ñ  $/Øò#	
ô* ! §¡Ó/ˆÔäNWÐXnÓNo÷ 
ÙCJÀ1ÀdˆD�$×$Ñ$ qÑ(¨4×+>Ñ+>Ñ>Ñ>ó 
ˆÔð 26×1EÑ1E×1KÑ1KÓ1M×N©¨¨A  1¡ÓNˆÔØ-.¸ÀAÐPQÑ%RˆÔ"à×Ñ &Ò(Ü36°t·}±}Ó3EÌÈD×L`ÑL`ÓHaÑ3aÐdh×dwÑdwÑ3wˆD×&Ñ& xÑ0à×"Ñ"×)Ñ)¨$×*>Ñ*>Ô?Ø7;×7QÑ7Q×7WÑ7WÓ7Y×%Z©t¨q°! a¨¡dÓ%ZˆÔ"ÚEˆØ×ÑÔ5°d×6IÑ6IÑJÔKàˆØ$ò 	ˆEØ×,Ñ,×0Ñ0°¸Ó=ˆCØ‰Ø×*Ñ*×.Ñ.¨s°DÔ9Ø‘ð		ñ
 Ø×ÑÔØ×)Ñ)Ô+àˆØ×4Ñ4×:Ñ:Ó<ò 	‰JˆE�3Ø�d×0Ñ0Ñ0ØÜ.8Ø˜t°¸eÈEô/ˆD×&Ñ& sÑ+ð ‰Fð	ñ Ø×ÑÔØ×)Ñ)Ô+à×Ñ &Ò(Ø%ˆDŒNà8<¿¹Ð8R�×$Ñ$ T§^¡^Ò4ÐX\×XfÑXfð Õ!ð *2Ð)=™XÀ;ˆDŒNØ$(×$8Ñ$8¸¿¹Ñ$HˆDÔ!à ˆŒØ×(Ñ(¨¯©Õ8ùó[ &[ùò ^ùó2 
ùó  Oùó &[s$   Â'PÃ 	P"Ã*P"Å&'P'Æ3P-É+P3c                 óÔ   — t        t        | di «      «      }t        | dd«      }t        | d«      rt        | j                  «      nd}t        | dd«      dk(  r||z   |z   dz   S ||z   |z   S )NrF   rK   r(   rQ   r   r9   r   )rP   ÚgetattrÚhasattrrQ   )r]   Úlang_code_countrK   Ú
base_vocabs       rl   Ú
vocab_sizezPLBartTokenizer.vocab_sizeé   su   € äœg dÐ,=¸rÓBÓCˆÜ  Ð'7¸Ó;ˆÜ+2°4¸Ô+D”S˜Ÿ™Ô'È!ˆ
Ü�4Ð)¨6Ó2°fÒ<Ø Ñ/°.Ñ@À1ÑDÐDØ˜OÑ+¨nÑ<Ð<ó    c                 ó¤  — | j                   j                  «       }t        | j                  j	                  «       «      D ]G  }| j                  j                  |«      }|dk(  r| j                  n|| j                  z   }||vsŒC|||<   ŒI |j                  | j                  j                  «       D ��ci c]  \  }}||vsŒ||“Œ c}}«       |S c c}}w )z,Override to use fairseq vocabulary structurer   )rH   ÚcopyÚrangerQ   Úget_piece_sizeÚ	IdToPieceÚunk_token_idrK   rT   rU   rI   )r]   Úvocabrd   Úsp_tokenÚvocab_idrh   ri   s          rl   Ú	get_vocabzPLBartTokenizer.get_vocabò   s¾   € à×*Ñ*×/Ñ/Ó1ˆÜ�t—}‘}×3Ñ3Ó5Ó6ò 	+ˆAØ—}‘}×.Ñ.¨qÓ1ˆHà,-°ªF�t×(Ò(¸¸T×=PÑ=PÑ9PˆHØ˜uÒ$Ø"*��h’ð	+ð 	�‰°4×3MÑ3M×3SÑ3SÓ3U×l¡Z U¨CÐY^ÐfkÒYk�e˜S‘jÓlÔmØˆùó ms   Â0C
Â=C
Úreturnc                 ó   — | j                   S ©N)rZ   ©r]   s    rl   r5   zPLBartTokenizer.src_langÿ   s   € à�~‰~Ðrs   Únew_src_langc                 ój   — | j                  |«      }|| _        | j                  | j                  «       y r€   )rD   rZ   r\   )r]   r‚   s     rl   r5   zPLBartTokenizer.src_lang  s+   € à×=Ñ=¸lÓKˆØ%ˆŒØ×(Ñ(¨¯©Õ8rs   Úreturn_tensorsr5   r6   c                 óÔ   — |�|€t        d«      ‚| j                  |«      | _        | j                  |«      | _         | |fd|dœ|¤Ž}| j	                  | j                  «      }||d<   |S )zIUsed by translation pipeline, to prepare inputs for the generate functionzATranslation requires a `src_lang` and a `tgt_lang` for this modelT)Úadd_special_tokensr„   Úforced_bos_token_id)Ú
ValueErrorrD   r5   r6   Úconvert_tokens_to_ids)r]   Ú
raw_inputsr„   r5   r6   Úextra_kwargsÚinputsÚtgt_lang_ids           rl   Ú_build_translation_inputsz)PLBartTokenizer._build_translation_inputs	  sz   € ð Ð˜xÐ/ÜÐ`ÓaÐaØ×>Ñ>¸xÓHˆŒØ×>Ñ>¸xÓHˆŒÙ�jÐi°TÈ.ÑiÐ\hÑiˆØ×0Ñ0°·±Ó?ˆØ(3ˆÐ$Ñ%Øˆrs   c                 ó¬   — || j                   v r| j                   |   S | j                  j                  |«      }|r|| j                  z   S | j                  S )z0Converts a token (str) in an id using the vocab.)rH   rQ   Ú	PieceToIdrK   ry   )r]   rh   Úspm_ids      rl   Ú_convert_token_to_idz$PLBartTokenizer._convert_token_to_id  sU   € à�D×.Ñ.Ñ.Ø×-Ñ-¨eÑ4Ð4Ø—‘×(Ñ(¨Ó/ˆñ 06ˆv˜×+Ñ+Ñ+ÐL¸4×;LÑ;LÐLrs   c                 óŒ   — || j                   v r| j                   |   S | j                  j                  || j                  z
  «      S )z=Converts an index (integer) in a token (str) using the vocab.)rJ   rQ   rx   rK   )r]   Úindexs     rl   Ú_convert_id_to_tokenz$PLBartTokenizer._convert_id_to_token  sA   € à�D×.Ñ.Ñ.Ø×-Ñ-¨eÑ4Ð4Ø�}‰}×&Ñ& u¨t×/BÑ/BÑ'BÓCÐCrs   Ú	src_textsÚ	tgt_textsc                 ó~   •— | j                  |«      | _        | j                  |«      | _        t        ‰| �  ||fi |¤ŽS r€   )rD   r5   r6   rN   Úprepare_seq2seq_batch)r]   r–   r5   r—   r6   r^   rk   s         €rl   r™   z%PLBartTokenizer.prepare_seq2seq_batch%  s@   ø€ ð ×>Ñ>¸xÓHˆŒØ×>Ñ>¸xÓHˆŒÜ‰wÑ,¨Y¸	ÑLÀVÑLÐLrs   c                 ó8   — | j                  | j                  «      S r€   )r\   r5   r�   s    rl   Ú_switch_to_input_modez%PLBartTokenizer._switch_to_input_mode1  ó   € Ø×/Ñ/°·±Ó>Ð>rs   c                 ó8   — | j                  | j                  «      S r€   )Úset_tgt_lang_special_tokensr6   r�   s    rl   Ú_switch_to_target_modez&PLBartTokenizer._switch_to_target_mode4  rœ   rs   c                 óÜ   — | j                  |«      }|�| j                  |   nd| _        g | _        | j                  �| j                  | j                  g| _        y| j                  g| _        y)z_Reset the special tokens to the source lang setting. No prefix and suffix=[eos, src_lang_code].N©rD   rF   Úcur_lang_coder"   Úeos_token_idr#   )r]   r5   s     rl   r\   z+PLBartTokenizer.set_src_lang_special_tokens7  sk   € à×9Ñ9¸(ÓCˆØ?GÐ?S˜T×1Ñ1°(Ò;ÐY]ˆÔØˆÔØ×ÑÐ)Ø"&×"3Ñ"3°T×5GÑ5GÐ!HˆDÕà"&×"3Ñ"3Ð!4ˆDÕrs   Úlangc                 óÜ   — | j                  |«      }|�| j                  |   nd| _        g | _        | j                  �| j                  | j                  g| _        y| j                  g| _        y)zcReset the special tokens to the target language setting. No prefix and suffix=[eos, tgt_lang_code].Nr¡   ©r]   r¤   s     rl   rž   z+PLBartTokenizer.set_tgt_lang_special_tokensA  sk   € à×5Ñ5°dÓ;ˆà;?Ð;K˜T×1Ñ1°$Ò7ÐQUˆÔØˆÔØ×ÑÐ)Ø"&×"3Ñ"3°T×5GÑ5GÐ!HˆDÕà"&×"3Ñ"3Ð!4ˆDÕrs   c                 ó2   — t         j                  ||«      }|S )z;Convert Language Codes to format tokenizer uses if required)ÚFAIRSEQ_LANGUAGE_CODES_MAPÚgetr¦   s     rl   rD   z1PLBartTokenizer._convert_lang_code_special_formatL  s   € ä)×-Ñ-¨d°DÓ9ˆØˆrs   c                 ó>   •— t        ‰| �  d||| j                  dœ|¤ŽS )zOOverride to use self.clean_up_tokenization_spaces as default for batched input.)Ú	token_idsÚskip_special_tokensr8   rA   )rN   Údecoder8   )r]   r«   r¬   r8   r^   rk   s        €rl   r­   zPLBartTokenizer.decodeQ  s2   ø€ ä‰w‰~ð 
ØØ 3Ø)-×)JÑ)Jñ
ð ñ	
ð 	
rs   )r*   r,   r,   r*   r-   r+   r>   r   NNNNT)r   Nr   )r~   N)FN)!Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr"   rL   ÚintÚ__annotations__r#   ÚdictrC   r   rO   Úpropertyrr   r}   r5   ÚsetterrŽ   r’   r•   r   r™   r›   rŸ   r\   rž   rD   r­   Ú__classcell__)rk   s   @rl   r   r   /   sŸ  ø… ñ;ðz *ÐØ$Ð&6Ð7Ðà!€M�4˜‘9Ó!Ø!€M�4˜‘9Ó!ð
 ØØØØØØØØØØ15Ø"&Ø%)ñs9ð ˜c 3˜h™¨$Ñ.õs9ðj ñ=ó ð=òð ð˜#ò ó ðð ‡_�_ð9 Sð 9¨Tò 9ó ð9ð
Ø*-ðØ9<¸t¹ðØORÐUYÉzóòMòDð  Ø&*Ø ñ
Mà˜‘9ð
Mð ð
Mð ˜‘9˜tÑ#ð	
Mð
 ð
Mð 
õ
Mò?ò?ó5ð	5°ð 	5¸ó 	5ð°cð ¸có ÷

ñ 
rs   r   N)Útypingr   Útokenization_pythonr   Útokenization_utils_baser   Ú tokenization_utils_sentencepiecer   Úutilsr   Úutils.import_utilsr	   Ú
get_loggerr®   ÚloggerÚSPIECE_UNDERLINEr²   rE   r¨   r   Ú__all__rA   rs   rl   ú<module>rÅ      sž   ðõ å 0Ý 1Ý DÝ Ý *ð 
ˆ×	Ñ	˜HÓ	%€àÐ à#<ÐP`ÑaÐ ò 4ÚgñÐ ð ØØØ"ØØØ
ñÐ ñ 
Ð%Ô&ôh
Ð*ó h
ó 'ðh
ðV	 Ð
�rs   