Ë
    ÜÓ^jF  ã                   óX   — d dl mZmZmZmZ d dlmZ ddlmZ ddiZ	 G d„ de«      Z
dgZy	)
é    )Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizers)ÚBPEé   )ÚTokenizersBackendÚtokenizer_fileztokenizer.jsonc                   óŒ   ‡ — e Zd ZdZeZdZddgZeZ		 	 	 	 	 	 	 dde
ee
ef   z  dz  de
ee
   z  dz  de
d	e
d
e
de
de
fˆ fd„Zˆ xZS )ÚSiglip2TokenizerzN
    Gemma tokenizer + SigLIP2 training default: lowercase normalization.
    ÚleftÚ	input_idsÚattention_maskNÚvocabÚmergesÚ	unk_tokenÚ	bos_tokenÚ	eos_tokenÚ	pad_tokenÚ
mask_tokenc                 óî  •— |€9t        |«      dt        |«      dt        |«      dt        |«      dt        |«      di}|| _        |xs g | _        t        t	        | j                  | j                  dt        |«      d d¬«      «      | _        t        j                  dd	d
¬«      | j
                  _        t        j                  t        j                  dd«      t        j                  «       t        j                  «       g«      | j
                  _        t        j                  dd«      | j
                  _        t#        ‰
| �H  d|||||dœ|¤Ž t'        | d«      rJt)        | j*                  t,        «      r0| j*                  j/                  d| j0                  j2                  «       t5        | dd «      }	|	�F|	j                   �9t        j                  t        j6                  «       |	j                   g«      |	_        y y y )Nr   é   é   r   é   T)r   r   Úfuse_unkr   ÚdropoutÚbyte_fallbackú Úmerged_with_previousF)ÚpatternÚbehaviorÚinvertu   â–�)r   r   r   r   r   Úinit_kwargsÚtokenizer_classÚ
_tokenizer© )ÚstrÚ_vocabÚ_mergesr   r   r%   r   ÚSplitÚpre_tokenizerr   ÚSequenceÚReplaceÚByteFallbackÚFuseÚdecoderr   Ú
normalizerÚsuperÚ__init__ÚhasattrÚ
isinstancer#   ÚdictÚ
setdefaultÚ	__class__Ú__name__ÚgetattrÚ	Lowercase)Úselfr   r   r   r   r   r   r   ÚkwargsÚbackendr8   s             €ú{/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/siglip2/tokenization_siglip2.pyr3   zSiglip2Tokenizer.__init__)   s¶  ø€ ð ˆ=ä�I“ Ü�I“ Ü�I“ Ü�I“ Ü�J“ ðˆEð ˆŒØ’| ˆŒä#ÜØ—k‘kØ—|‘|ØÜ˜i›.ØØ"ôó	
ˆŒô )7×(<Ñ(<ØÐ"8Àô)
ˆ�‰Ô%ô #+×"3Ñ"3Ü×Ñ˜e SÓ)¬8×+@Ñ+@Ó+BÄHÇMÁMÃOÐTó#
ˆ�‰Ôô &1×%8Ñ%8¸¸eÓ%Dˆ�‰Ô"Ü‰Ñð 	
ØØØØØ!ñ	
ð ò	
ô �4˜Ô'¬J°t×7GÑ7GÌÔ,NØ×Ñ×'Ñ'Ð(9¸4¿>¹>×;RÑ;RÔSä˜$ ¨dÓ3ˆØÐ 7×#5Ñ#5Ð#AÜ!,×!5Ñ!5´{×7LÑ7LÓ7NÐPW×PbÑPbÐ6cÓ!dˆGÕð $BÐó    )NNz<unk>z<bos>z<eos>z<pad>z<mask>)r9   Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚpadding_sideÚmodel_input_namesr   Úmodelr'   r6   ÚintÚlistr3   Ú__classcell__)r8   s   @r?   r   r      s³   ø„ ñð *ÐØ€LØ$Ð&6Ð7ÐØ€Eð .2Ø)-Ø Ø Ø Ø Ø"ñ7eà�T˜#˜s˜(‘^Ñ# dÑ*ð7eð �d˜3‘i‘ $Ñ&ð7eð ð	7eð
 ð7eð ð7eð ð7eð ÷7eñ 7er@   r   N)Ú
tokenizersr   r   r   r   Útokenizers.modelsr   Útokenization_utils_tokenizersr	   rD   r   Ú__all__r&   r@   r?   ú<module>rP      s>   ð÷, HÓ GÝ !å >ð &Ð'7Ð8Ð ôAeÐ(ô AeðH Ð
�r@   