Ë
    ÜÓ^j$  ã                   óž   — d Z ddlZddlmZmZmZmZmZ ddlm	Z	 ddl
mZ ddlmZ  ej                  e«      Zdd	iZd
„ Z G d„ de«      ZdgZy)z"Tokenization classes for Splinter.é    N)Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)Ú	WordPieceé   )ÚTokenizersBackend)ÚloggingÚ
vocab_filez	vocab.txtc                 óæ   — t        j                  «       }t        | dd¬«      5 }|j                  «       }d d d «       t	        «      D ]  \  }}|j                  d«      }|||<   Œ |S # 1 sw Y   Œ4xY w)NÚrzutf-8)Úencodingú
)ÚcollectionsÚOrderedDictÚopenÚ	readlinesÚ	enumerateÚrstrip)r   ÚvocabÚreaderÚtokensÚindexÚtokens         ú}/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/splinter/tokenization_splinter.pyÚ
load_vocabr      sw   € Ü×#Ñ#Ó%€EÜ	ˆj˜#¨Ô	0ð $°FØ×!Ñ!Ó#ˆ÷$ä! &Ó)ò ‰ˆˆuØ—‘˜TÓ"ˆØˆˆeŠðð €L÷$ð $ús   £A'Á'A0c                   ó¤   ‡ — e Zd ZdZeZddgZeZ	 	 	 	 	 	 	 	 	 	 dde	e
e	ef   z  dz  dede	de	d	e	d
e	de	de	dededz  fˆ fd„Zed„ «       Zd„ Zˆ xZS )ÚSplinterTokenizeraú  
    Construct a Splinter tokenizer (backed by HuggingFace's tokenizers library). Based on WordPiece.

    This tokenizer inherits from [`TokenizersBackend`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`, *optional*):
            Path to a vocabulary file.
        tokenizer_file (`str`, *optional*):
            Path to a tokenizers JSON file containing the serialization of a tokenizer.
        do_lower_case (`bool`, *optional*, defaults to `True`):
            Whether or not to lowercase the input when tokenizing.
        unk_token (`str`, *optional*, defaults to `"[UNK]"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        sep_token (`str`, *optional*, defaults to `"[SEP]"`):
            The separator token, which is used when building a sequence from multiple sequences.
        pad_token (`str`, *optional*, defaults to `"[PAD]"`):
            The token used for padding, for example when batching sequences of different lengths.
        cls_token (`str`, *optional*, defaults to `"[CLS]"`):
            The classifier token which is used when doing sequence classification.
        mask_token (`str`, *optional*, defaults to `"[MASK]"`):
            The token used for masking values.
        question_token (`str`, *optional*, defaults to `"[QUESTION]"`):
            The token used for constructing question representations.
        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
            Whether or not to tokenize Chinese characters.
        strip_accents (`bool`, *optional*):
            Whether or not to strip all accents. If this option is not specified, then it will be determined by the
            value for `lowercase`.
        vocab (`str`, `dict` or `list`, *optional*):
            Custom vocabulary dictionary. If not provided, a minimal vocabulary is created.
    Ú	input_idsÚattention_maskNr   Údo_lower_caseÚ	unk_tokenÚ	sep_tokenÚ	pad_tokenÚ	cls_tokenÚ
mask_tokenÚquestion_tokenÚtokenize_chinese_charsÚstrip_accentsc                 óÖ  •— |�|nEt        |«      dt        |«      dt        |«      dt        |«      dt        |«      dt        |«      dddi| _        t        t        | j                  t        |«      ¬	«      «      | _        t        j                  d
|	|
|¬«      | j                  _        t        j                  «       | j                  _
        t        j                  d¬«      | j                  _        t        ‰| �8  d||||||||	|
dœ	|¤Ž || _        |	| _        |
| _        || _        | j$                  | j&                  vr| j)                  | j$                  gd
¬«       | j+                  «        y )Nr   é   é   r	   é   é   ú.é   )r#   T)Ú
clean_textÚhandle_chinese_charsr*   Ú	lowercasez##)Úprefix)	r#   r$   r%   r&   r'   r(   r"   r)   r*   )Úspecial_tokens© )ÚstrÚ_vocabr   r   Ú
_tokenizerr   ÚBertNormalizerÚ
normalizerr   ÚBertPreTokenizerÚpre_tokenizerr   ÚdecoderÚsuperÚ__init__r"   r)   r*   r(   Úall_special_tokensÚ
add_tokensÚupdate_post_processor)Úselfr   r"   r#   r$   r%   r&   r'   r(   r)   r*   ÚkwargsÚ	__class__s               €r   rA   zSplinterTokenizer.__init__Q   sZ  ø€ ð  Ð ñ ô �I“ Ü�I“ Ü�I“ Ü�I“ Ü�J“ Ü�NÓ# QØ�Qðð 	Œô $¤I¨d¯k©kÄSÈÃ^Ô$TÓUˆŒä%0×%?Ñ%?ØØ!7Ø'Ø#ô	&
ˆ�‰Ô"ô )7×(GÑ(GÓ(Iˆ�‰Ô%Ü"*×"4Ñ"4¸DÔ"Aˆ�‰Ôä‰Ñð 	
ØØØØØ!Ø)Ø'Ø#9Ø'ñ	
ð ò	
ð +ˆÔØ&<ˆÔ#Ø*ˆÔØ,ˆÔØ×Ñ d×&=Ñ&=Ñ=Ø�O‰O˜T×0Ñ0Ð1À$ˆOÔGØ×"Ñ"Õ$ó    c                 ó8   — | j                  | j                  «      S )N)Úconvert_tokens_to_idsr(   )rE   s    r   Úquestion_token_idz#SplinterTokenizer.question_token_id�   s   € à×)Ñ)¨$×*=Ñ*=Ó>Ð>rH   c           
      óš  — | j                   }| j                  }| j                  }d}| j                  }| j                  }| j
                  }| j                  d«      }|�|€y | j                  dk(  r|› d|› d|› d|› d|› d�
}	n|› d|› d|› d|› d|› d�
}	t        j                  |› d|› d�|	||f||f||f||fg¬«      | j                  _        y )	Nr0   Úrightz:0 $A:0 ú z:0 $B:1 z:1z:0)ÚsingleÚpairr6   )r&   r$   r(   Úcls_token_idÚsep_token_idrK   rJ   Úpadding_sider   ÚTemplateProcessingr:   Úpost_processor)
rE   ÚclsÚsepÚquestionÚdotrQ   rR   rK   Údot_token_idrP   s
             r   rD   z'SplinterTokenizer.update_post_processor‘   s  € Ø�n‰nˆØ�n‰nˆØ×&Ñ&ˆØˆØ×(Ñ(ˆØ×(Ñ(ˆØ ×2Ñ2ÐØ×1Ñ1°#Ó6ˆàˆ;˜#˜+Øà×Ñ Ò'Ø�U˜( 8 *¨A¨c¨U°!°C°5¸ÀÀÀRÐH‰Dà�U˜( 3 % x°¨z¸¸3¸%¸qÀÀÀRÐHˆDä)3×)FÑ)FØ�U˜( 3 % rÐ*Øà�lÐ#Ø�lÐ#ØÐ,Ð-Ø�lÐ#ð	ô	*
ˆ�‰Õ&rH   )
NTz[UNK]z[SEP]z[PAD]z[CLS]z[MASK]z
[QUESTION]TN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr   Úmodelr8   ÚdictÚintÚboolrA   ÚpropertyrK   rD   Ú__classcell__)rG   s   @r   r   r   )   sÙ   ø„ ñ!ðF *ÐØ$Ð&6Ð7ÐØ€Eð .2Ø"Ø Ø Ø Ø Ø"Ø*Ø'+Ø%)ñ:%à�T˜#˜s˜(‘^Ñ# dÑ*ð:%ð ð:%ð ð	:%ð
 ð:%ð ð:%ð ð:%ð ð:%ð ð:%ð !%ð:%ð ˜d‘{õ:%ðx ñ?ó ð?ö
rH   r   )r^   r   Ú
tokenizersr   r   r   r   r   Útokenizers.modelsr   Útokenization_utils_tokenizersr
   Úutilsr   Ú
get_loggerr[   Úloggerr_   r   r   Ú__all__r7   rH   r   ú<module>ro      sZ   ðñ )ã ç SÕ SÝ 'å >Ý ð 
ˆ×	Ñ	˜HÓ	%€à! ;Ð/Ð òôC
Ð)ô C
ðL Ð
�rH   