Ë
    ÜÓ^jû  ã                   ó¤   — d Z ddlmZmZmZmZmZmZ ddlm	Z	 ddl
mZ ddlmZ  ej                  e«      Zddd	d
œZddiZdZ G d„ de«      ZdgZy)zTokenization classes for Qwen2.é    )Ú
AddedTokenÚRegexÚ	TokenizerÚdecodersÚnormalizersÚpre_tokenizers)ÚBPEé   )ÚTokenizersBackend)Úloggingz
vocab.jsonz
merges.txtztokenizer.json)Ú
vocab_fileÚmerges_fileÚtokenizer_filezqwen/qwen-tokenizeri €  zn(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+c                   ó|   ‡ — e Zd ZeZddgZeZ	 	 	 	 	 	 	 d
dee	ee
f   z  dz  deee   z  dz  dededef
ˆ fd	„Zˆ xZS )ÚQwen2TokenizerÚ	input_idsÚattention_maskNÚvocabÚmergesÚ	unk_tokenÚ	eos_tokenÚ	pad_tokenc                 ó®  •— |�|nd| _         |�|nddi| _        |xs g | _        t        t	        | j                  | j                  d d dddd¬«      «      | _        t        j                  «       | j
                  _        t        j                  «       | j
                  _        t        j                  t        j                  t        t         «      dd¬«      t        j                  | j                   d¬«      g«      | j
                  _        t%        ‰
| �L  d|||||d	œ|¤Ž | j)                  | j*                  D �	cg c]  }	t-        |	d
¬«      ‘Œ c}	«       y c c}	w )NFú<|endoftext|>r   Ú )r   r   Údropoutr   Úcontinuing_subword_prefixÚend_of_word_suffixÚfuse_unkÚbyte_fallbackÚisolated)ÚbehaviorÚinvert)Úadd_prefix_spaceÚ	use_regex)r   Ú	bos_tokenr   r   r$   T)Úspecial© )r$   Ú_vocabÚ_mergesr   r	   Ú
_tokenizerr   Ú	ByteLevelÚdecoderr   ÚNFCÚ
normalizerr   ÚSequenceÚSplitr   ÚPRETOKENIZE_REGEXÚpre_tokenizerÚsuperÚ__init__Ú
add_tokensÚall_special_tokensr   )Úselfr   r   r   r&   r   r   r$   ÚkwargsÚtokenÚ	__class__s             €úw/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/qwen2/tokenization_qwen2.pyr5   zQwen2Tokenizer.__init__)   sM  ø€ ð 5EÐ4PÑ 0ÐV[ˆÔð Ð ñ ð   ðð 	Œð ’| ˆŒÜ#ÜØ—k‘kØ—|‘|ØØØ*,Ø#%ØØ#ô	ó
ˆŒô #+×"4Ñ"4Ó"6ˆ�‰ÔÜ%0§_¡_Ó%6ˆ�‰Ô"Ü(6×(?Ñ(?ä×$Ñ$ÜÔ+Ó,Ø'Ø ôô
 ×(Ñ(Ø%)×%:Ñ%:Ø#ôð
ó)
ˆ�‰Ô%ô 	‰Ñð 	
ØØØØØ-ñ	
ð ò	
ð 	�‰Àd×F]ÑF]Ö^¸Uœ E°4Ö8Ò^Õ_ùÒ^s   Ä6E)NNr   Nr   r   N)Ú__name__Ú
__module__Ú__qualname__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr	   ÚmodelÚstrÚdictÚintÚlistr5   Ú__classcell__)r;   s   @r<   r   r   $   s‘   ø„ Ø)ÐØ$Ð&6Ð7ÐØ€Eð .2Ø)-Ø(ØØ(Ø(Øñ9`à�T˜#˜s˜(‘^Ñ# dÑ*ð9`ð �d˜3‘i‘ $Ñ&ð9`ð ð	9`ð ð9`ð ÷9`ñ 9`ó    r   N)Ú__doc__Ú
tokenizersr   r   r   r   r   r   Útokenizers.modelsr	   Útokenization_utils_tokenizersr   Úutilsr   Ú
get_loggerr=   Úloggerr@   ÚMAX_MODEL_INPUT_SIZESr2   r   Ú__all__r(   rI   r<   ú<module>rS      sp   ðñ &ç Z× ZÝ !å >Ý ð 
ˆ×	Ñ	˜HÓ	%€ð ØØ&ñÐ ð /°Ð6Ð ð JÐ ô>`Ð&ô >`ðB Ð
�rI   