Ë
    ÜÓ^j  ã                   ób   — d dl Z d dlmZmZmZmZ d dlmZ ddlm	Z	 dddœZ
 G d	„ d
e	«      Zd
gZy)é    N)Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizers)ÚUnigramé   )ÚTokenizersBackendzspiece.modelztokenizer.json)Ú
vocab_fileÚtokenizer_filec                   óp   ‡ — e Zd ZdZeZddgZeZ	 	 	 	 	 	 	 d	de	e
ee	ef      z  dz  fˆ fd„Zd„ Zd„ Zˆ xZS )
ÚVideoPrismTokenizeraV  
    Constructs a VideoPrism tokenizer, which is essentially a T5 tokenizer without its postprocessor
    (appending an EOS token at the end of the sequence).

    This tokenizer inherits from [`T5Tokenizer`] which contains most of the main methods. Users should refer to this
    superclass for more information regarding those methods.
    Ú	input_idsÚattention_maskNÚvocabc           	      óØ  •— || _         |�q|D �	cg c]  }	dt        |	«      v sŒ|	‘Œ }
}	t        |
«      dk  r!|t        |«      D �cg c]  }d|› d�‘Œ
 c}z  }nC|dkD  r>|t        |
«      k7  r0t	        d|› d|› d�«      ‚t        |«      D �cg c]  }d|› d�‘Œ
 }
}|
}|�|| _        nbt        |«      dft        |«      dft        |«      dfd	g| _        t        |dz
  d
d
«      D ]#  }| j
                  j                  d|› d�df«       Œ% t        t        | j
                  dd¬«      «      | _	        |�$t        j                  |«      | j                  _        t        j                  t        j                  «       t        j                   ddd¬«      g«      | j                  _        t%        j                   ddd¬«      | j                  _        t)        ‰| �T  d|||||dœ|¤Ž y c c}	w c c}w c c}w )Nz
<extra_id_é   ú>r   zBoth extra_ids (z!) and additional_special_tokens (zs) are provided to VideoPrismTokenizer. In this case the additional_special_tokens must include the extra_ids tokensg        )õ   â–�g       Àéÿÿÿÿé   F)Úunk_idÚbyte_fallbackr   ÚalwaysT)ÚreplacementÚprepend_schemeÚsplit)Ú	eos_tokenÚ	unk_tokenÚ	pad_tokenÚ	extra_idsÚadditional_special_tokens© )Ú
_extra_idsÚstrÚlenÚrangeÚ
ValueErrorÚ_vocab_scoresÚappendr   r   Ú
_tokenizerr   ÚPrecompiledÚ
normalizerr   ÚSequenceÚWhitespaceSplitÚ	MetaspaceÚpre_tokenizerr   ÚdecoderÚsuperÚ__init__)Úselfr   r   r   r   Ú_spm_precompiled_charsmapr    r!   ÚkwargsÚxÚextra_tokensÚiÚ	__class__s               €ú�/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/videoprism/tokenization_videoprism.pyr3   zVideoPrismTokenizer.__init__.   s)  ø€ ð $ˆŒð %Ð0Ø'@Ö[ !ÀLÔTWÐXYÓTZÒDZšAÐ[ˆLÐ[Ü�<Ó  1Ò$Ø)ÌÈyÓIYÖ-ZÀA°
¸1¸#¸QÒ.?Ò-ZÑZÑ)Ø˜Q’ 9´°LÓ0AÒ#AÜ Ø& y kÐ1RÐSlÐRmð nð óð ô 8=¸YÓ7GÖH°!˜j¨¨¨1Ò-ÐHˆLÐHØ(4Ð%ð ÐØ!&ˆDÕô �Y“ Ð%Ü�Y“ Ð%Ü�Y“ Ð%Øð	"ˆDÔô ˜9 q™=¨"¨bÓ1ò D�Ø×"Ñ"×)Ñ)¨Z¸°s¸!Ð+<¸cÐ*BÕCðDô $ÜØ×"Ñ"ØØ#ôó
ˆŒð %Ð0Ü)4×)@Ñ)@ÐAZÓ)[ˆD�O‰OÔ&ä(6×(?Ñ(?ä×.Ñ.Ó0Ü×(Ñ(°UÈ8Ð[_Ô`ðó)
ˆ�‰Ô%ô #+×"4Ñ"4ÀÐW_ÐgkÔ"lˆ�‰Ôä‰Ñð 	
ØØØØØ&?ñ	
ð ó	
ùò[ \ùâ-Zùò Is   �G¢GÁG"Â	G'c                 óT   — t        t        t        d„ | j                  «      «      «      S )zQGet the list of sentinel tokens (extra_id tokens) from additional_special_tokens.c                 óD   — t        t        j                  d| «      «      d uS )Nz<extra_id_\d+>)ÚboolÚreÚsearch)r7   s    r;   ú<lambda>z9VideoPrismTokenizer.get_sentinel_tokens.<locals>.<lambda>v   s   € ¤¤b§i¡iÐ0AÀ1Ó&EÓ!FÈdÐ!R€ ó    )ÚlistÚsetÚfilterr!   )r4   s    r;   Úget_sentinel_tokensz'VideoPrismTokenizer.get_sentinel_tokenss   s&   € äÜ”ÑRÐTX×TrÑTrÓsÓtó
ð 	
rB   c                 óf   — | j                  «       D �cg c]  }| j                  |«      ‘Œ c}S c c}w )z&Get the token IDs for sentinel tokens.)rF   Úconvert_tokens_to_ids)r4   Útokens     r;   Úget_sentinel_token_idsz*VideoPrismTokenizer.get_sentinel_token_idsy   s*   € à?C×?WÑ?WÓ?YÖZ°e�×*Ñ*¨5Õ1ÒZÐZùÒZs   “.)Nz</s>z<unk>z<pad>Néd   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr   Úmodelr$   rC   ÚtupleÚfloatr3   rF   rJ   Ú__classcell__)r:   s   @r;   r   r   !   sk   ø„ ñð *ÐØ$Ð&6Ð7ÐØ€Eð 7;ØØØØ"&ØØ"&ñC
à�T˜%  U 
Ñ+Ñ,Ñ,¨tÑ3õC
òJ
ö[rB   r   )r?   Ú
tokenizersr   r   r   r   Útokenizers.modelsr   Útokenization_utils_tokenizersr	   rP   r   Ú__all__r"   rB   r;   ú<module>r[      s@   ðó, 
ç GÓ GÝ %å >ð $2ÐEUÑVÐ ôZ[Ð+ô Z[ðz !Ð
!�rB   