Ë
    ÛÓ^jR  ã                   ó˜   — d Z ddlmZmZmZmZmZmZ ddlm	Z	 ddl
mZ ddlmZ  ej                  e«      Zddd	d
œZ G d„ de«      ZdgZy)zTokenization classes for CLIP.é    )ÚRegexÚ	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEé   )ÚTokenizersBackend)Úloggingz
vocab.jsonz
merges.txtztokenizer.json)Ú
vocab_fileÚmerges_fileÚtokenizer_filec                   óˆ   ‡ — e Zd ZdZeZddgZeZ	 	 	 	 	 	 dde	e
e	ef   z  dz  de	ee	   z  dz  de	de	d	e	d
e	fˆ fd„Zd„ Zˆ xZS )ÚCLIPTokenizerav  
    Construct a CLIP tokenizer (backed by HuggingFace's *tokenizers* library). Based on byte-level
    Byte-Pair-Encoding.

    This tokenizer inherits from [`TokenizersBackend`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    Args:
        vocab (`str`, `dict` or `list`, *optional*):
            Vocabulary dict to use for the tokenizer.
        merges (`str` or `list`, *optional*):
            Merges list to use for the BPE tokenizer.
        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        bos_token (`str`, *optional*, defaults to `"<|startoftext|>"`):
            The beginning of sequence token.
        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The end of sequence token.
        pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
            The token used for padding, for example when batching sequences of different lengths.
    Ú	input_idsÚattention_maskNÚvocabÚmergesÚ	unk_tokenÚ	bos_tokenÚ	eos_tokenÚ	pad_tokenc                 ó„  •— |�|n"t        |«      dt        |«      dt        |«      di}|xs g | _        t        t        || j                  d dddt        |«      ¬«      «      | _        t        j                  t        j                  «       t        j                  t        d«      d	«      t        j                  «       g«      | j                  _        t        j                  t        j                  t        d
«      dd¬«      t        j                  d¬«      g«      | j                  _        t!        j                  «       | j                  _        t%        ‰	| �L  d||||dœ|¤Ž t)        j*                  t        |«      | j,                  ft        |«      | j.                  fdd¬«      | j                  _        | j3                  «        y )Nr   é   é   Ú z</w>F)r   r   ÚdropoutÚcontinuing_subword_prefixÚend_of_word_suffixÚfuse_unkr   z\s+ú z[<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+ÚremovedT)ÚbehaviorÚinvert)Úadd_prefix_space)r   r   r   r   )ÚsepÚclsr&   Útrim_offsets© )ÚstrÚ_mergesr   r	   Ú
_tokenizerr   ÚSequenceÚNFCÚReplacer   Ú	LowercaseÚ
normalizerr   ÚSplitÚ	ByteLevelÚpre_tokenizerr   ÚdecoderÚsuperÚ__init__r   ÚRobertaProcessingÚeos_token_idÚbos_token_idÚpost_processorÚ%_wrap_decode_method_backend_tokenizer)
Úselfr   r   r   r   r   r   ÚkwargsÚ_vocabÚ	__class__s
            €úu/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/clip/tokenization_clip.pyr8   zCLIPTokenizer.__init__8   s˜  ø€ ð Ð ñ ô �I“ Ü�I“ Ü�I“ ðð 	ð ’| ˆŒä#ÜØØ—|‘|ØØ*,Ø#)ØÜ˜i›.ôó

ˆŒô &1×%9Ñ%9Ü�_‰_Ó¤× 3Ñ 3´E¸&³MÀ3Ó GÌ×I^ÑI^ÓI`Ðaó&
ˆ�‰Ô"ô )7×(?Ñ(?ä×$Ñ$ÜØzóð 'Øôô ×(Ñ(¸%Ô@ð	ó)
ˆ�‰Ô%ô #+×"4Ñ"4Ó"6ˆ�‰Ôä‰Ñð 	
ØØØØñ		
ð
 ò	
ô *4×)EÑ)EÜ�Y“ ×!2Ñ!2Ð3Ü�Y“ ×!2Ñ!2Ð3Ø"Øô	*
ˆ�‰Ô&ð 	×2Ñ2Õ4ó    c                 ó¢   ‡‡— | j                   j                  Š| j                   j                  j                  Šˆˆfd„}|| j                   _        y )Nc                  óX   •—  ‰| i |¤Ž}|j                  ‰d«      j                  «       }|S )Nr"   )ÚreplaceÚstrip)Úargsr?   Útextr    Úorig_decode_methods      €€rB   Únew_decode_methodzNCLIPTokenizer._wrap_decode_method_backend_tokenizer.<locals>.new_decode_method†   s1   ø€ Ù% tÐ6¨vÑ6ˆDØ—<‘<Ð 2°CÓ8×>Ñ>Ó@ˆDØˆKrC   )Úbackend_tokenizerÚdecodeÚmodelr    )r>   rK   r    rJ   s     @@rB   r=   z3CLIPTokenizer._wrap_decode_method_backend_tokenizer   sD   ù€ Ø!×3Ñ3×:Ñ:Ðð "×3Ñ3×9Ñ9×LÑLÐõ	ð
 ):ˆ×ÑÕ%rC   )NNú<|endoftext|>z<|startoftext|>rO   rO   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr	   rN   r+   ÚdictÚintÚlistr8   r=   Ú__classcell__)rA   s   @rB   r   r      s    ø„ ñð. *ÐØ$Ð&6Ð7ÐØ€Eð .2Ø)-Ø(Ø*Ø(Ø(ñE5à�T˜#˜s˜(‘^Ñ# dÑ*ðE5ð �d˜3‘i‘ $Ñ&ðE5ð ð	E5ð
 ðE5ð ðE5ð õE5öN:rC   r   N)rS   Ú
tokenizersr   r   r   r   r   r   Útokenizers.modelsr	   Útokenization_utils_tokenizersr   Úutilsr   Ú
get_loggerrP   ÚloggerrT   r   Ú__all__r*   rC   rB   ú<module>rb      sU   ðñ %ç Z× ZÝ !å >Ý ð 
ˆ×	Ñ	˜HÓ	%€à#/ÀÐ`pÑqÐ ôo:Ð%ô o:ðd Ð
�rC   