
    ^j                     b    d dl Z d dlmZmZmZmZ d dlmZ ddlm	Z	 dddZ
 G d	 d
e	      Zd
gZy)    N)	Tokenizerdecodersnormalizerspre_tokenizers)Unigram   )TokenizersBackendzspiece.modelztokenizer.json)
vocab_filetokenizer_filec                   p     e Zd ZdZeZddgZeZ	 	 	 	 	 	 	 d	de	e
ee	ef      z  dz  f fdZd Zd Z xZS )
VideoPrismTokenizeraV  
    Constructs a VideoPrism tokenizer, which is essentially a T5 tokenizer without its postprocessor
    (appending an EOS token at the end of the sequence).

    This tokenizer inherits from [`T5Tokenizer`] which contains most of the main methods. Users should refer to this
    superclass for more information regarding those methods.
    	input_idsattention_maskNvocabc           	         || _         |q|D 	cg c]  }	dt        |	      v s|	 }
}	t        |
      dk  r!|t        |      D cg c]  }d| d
 c}z  }nC|dkD  r>|t        |
      k7  r0t	        d| d| d      t        |      D cg c]  }d| d
 }
}|
}||| _        nbt        |      dft        |      dft        |      dfd	g| _        t        |dz
  d
d
      D ]#  }| j
                  j                  d| ddf       % t        t        | j
                  dd            | _	        |$t        j                  |      | j                  _        t        j                  t        j                         t        j                   ddd      g      | j                  _        t%        j                   ddd      | j                  _        t)        | T  d|||||d| y c c}	w c c}w c c}w )Nz
<extra_id_   >r   zBoth extra_ids (z!) and additional_special_tokens (zs) are provided to VideoPrismTokenizer. In this case the additional_special_tokens must include the extra_ids tokensg        )   ▁g          F)unk_idbyte_fallbackr   alwaysT)replacementprepend_schemesplit)	eos_token	unk_token	pad_token	extra_idsadditional_special_tokens )
_extra_idsstrlenrange
ValueError_vocab_scoresappendr   r   
_tokenizerr   Precompiled
normalizerr   SequenceWhitespaceSplit	Metaspacepre_tokenizerr   decodersuper__init__)selfr   r   r   r   _spm_precompiled_charsmapr    r!   kwargsxextra_tokensi	__class__s               /var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/videoprism/tokenization_videoprism.pyr3   zVideoPrismTokenizer.__init__.   s)    $ %0'@[!LTWXYTZDZA[L[< 1$)yIY-ZA
1#Q.?-ZZ)Q9L0A#A &yk1RSlRm n   8=Y7GH!j1-HLH(4% !&D Y%Y%Y%	"D 9q="b1 D""))Zs!+<c*BCD $""#
 %0)4)@)@AZ)[DOO&(6(?(?..0((U8[_`)
% #+"4"4W_gk"l 	
&?	
 	
[ \-Z Is   GGG"	G'c                 T    t        t        t        d | j                                    S )zQGet the list of sentinel tokens (extra_id tokens) from additional_special_tokens.c                 D    t        t        j                  d|             d uS )Nz<extra_id_\d+>)boolresearch)r7   s    r;   <lambda>z9VideoPrismTokenizer.get_sentinel_tokens.<locals>.<lambda>v   s    bii0A1&E!Fd!R     )listsetfilterr!   )r4   s    r;   get_sentinel_tokensz'VideoPrismTokenizer.get_sentinel_tokenss   s&    RTXTrTrst
 	
rB   c                 f    | j                         D cg c]  }| j                  |       c}S c c}w )z&Get the token IDs for sentinel tokens.)rF   convert_tokens_to_ids)r4   tokens     r;   get_sentinel_token_idsz*VideoPrismTokenizer.get_sentinel_token_idsy   s*    ?C?W?W?YZe**51ZZZs   .)Nz</s>z<unk>z<pad>Nd   N)__name__
__module____qualname____doc__VOCAB_FILES_NAMESvocab_files_namesmodel_input_namesr   modelr$   rC   tuplefloatr3   rF   rJ   __classcell__)r:   s   @r;   r   r   !   sk     *$&67E 7;"&"&C
T%U
+,,t3C
J
[rB   r   )r?   
tokenizersr   r   r   r   tokenizers.modelsr   tokenization_utils_tokenizersr	   rP   r   __all__r"   rB   r;   <module>r[      s@   , 
 G G % > $2EUV Z[+ Z[z !
!rB   