Ë
    ÛÓ^j  ã                   óˆ   — d dl mZmZmZmZ d dlmZ ddlmZ ddl	m
Z
  e
j                  e«      ZddiZ G d„ d	e«      Zd	gZy
)é    )Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizers)ÚBPEé   )ÚTokenizersBackend)ÚloggingÚtokenizer_fileztokenizer.jsonc                   óŒ   ‡ — e Zd ZdZeZdZddgZeZ		 	 	 	 	 	 	 dde
ee
ef   z  dz  de
ee
   z  dz  de
d	e
d
e
de
de
fˆ fd„Zˆ xZS )ÚGemmaTokenizeruá  
    Construct a fast Gemma tokenizer (backed by HuggingFace's tokenizers library).

    This tokenizer uses a BPE model with byte fallback, no prefix space, and a normalizer that replaces
    spaces with "â–�".

    Args:
        tokenizer_file (`str`, optional):
            A tokenizers JSON file containing the serialization of a tokenizer.
        unk_token (`str`, optional, defaults to "<unk>"):
            The unknown token.
        bos_token (`str`, optional, defaults to "<bos>"):
            The beginning of sequence token.
        eos_token (`str`, optional, defaults to "<eos>"):
            The end of sequence token.
        pad_token (`str`, optional, defaults to "<pad>"):
            The padding token.
        mask_token (`str`, optional, defaults to "<mask>"):
            The mask token.
        add_bos_token (`bool`, optional, defaults to True):
            Whether or not to add a `bos_token` at the start of sequences.
        add_eos_token (`bool`, optional, defaults to False):
            Whether or not to add an `eos_token` at the end of sequences.
        vocab (`str` or `dict[str, int]`, optional):
            Custom vocabulary dict. If not provided, a minimal vocabulary is created using the special tokens.
    ÚleftÚ	input_idsÚattention_maskNÚvocabÚmergesÚ	unk_tokenÚ	bos_tokenÚ	eos_tokenÚ	pad_tokenÚ
mask_tokenc                 ó˜  •— |€9t        |«      dt        |«      dt        |«      dt        |«      dt        |«      di}|| _        |xs g | _        t        t	        | j                  | j                  dt        |«      d d¬«      «      | _        t        j                  dd	d
¬«      | j
                  _        t        j                  t        j                  dd«      t        j                  «       t        j                  «       g«      | j
                  _        t        j                  dd«      | j
                  _        t#        ‰	| �H  d|||||dœ|¤Ž y )Nr   é   é   r   é   T)r   r   Úfuse_unkr   ÚdropoutÚbyte_fallbackú Úmerged_with_previousF)ÚpatternÚbehaviorÚinvertu   â–�)r   r   r   r   r   © )ÚstrÚ_vocabÚ_mergesr   r   Ú
_tokenizerr   ÚSplitÚpre_tokenizerr   ÚSequenceÚReplaceÚByteFallbackÚFuseÚdecoderr   Ú
normalizerÚsuperÚ__init__)
Úselfr   r   r   r   r   r   r   ÚkwargsÚ	__class__s
            €úw/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/gemma/tokenization_gemma.pyr2   zGemmaTokenizer.__init__;   s*  ø€ ð ˆ=ä�I“ Ü�I“ Ü�I“ Ü�I“ Ü�J“ ðˆEð ˆŒØ’| ˆŒä#ÜØ—k‘kØ—|‘|ØÜ˜i›.ØØ"ôó	
ˆŒô )7×(<Ñ(<ØÐ"8Àô)
ˆ�‰Ô%ô #+×"3Ñ"3Ü×Ñ˜e SÓ)¬8×+@Ñ+@Ó+BÄHÇMÁMÃOÐTó#
ˆ�‰Ôô &1×%8Ñ%8¸¸eÓ%Dˆ�‰Ô"Ü‰Ñð 	
ØØØØØ!ñ	
ð ó	
ó    )NNz<unk>z<bos>z<eos>z<pad>z<mask>)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚpadding_sideÚmodel_input_namesr   Úmodelr%   ÚdictÚintÚlistr2   Ú__classcell__)r5   s   @r6   r   r      sª   ø„ ñð6 *ÐØ€LØ$Ð&6Ð7ÐØ€Eð .2Ø)-Ø Ø Ø Ø Ø"ñ/
à�T˜#˜s˜(‘^Ñ# dÑ*ð/
ð �d˜3‘i‘ $Ñ&ð/
ð ð	/
ð
 ð/
ð ð/
ð ð/
ð ÷/
ñ /
r7   r   N)Ú
tokenizersr   r   r   r   Útokenizers.modelsr   Útokenization_utils_tokenizersr	   Úutilsr
   Ú
get_loggerr8   Úloggerr<   r   Ú__all__r$   r7   r6   ú<module>rL      sQ   ð÷ HÓ GÝ !å >Ý ð 
ˆ×	Ñ	˜HÓ	%€Ø%Ð'7Ð8Ð ôP
Ð&ô P
ðf Ð
�r7   