Ë
    ÛÓ^j9  ã                   ó¸   — d Z ddlZddlZddlmZmZ  e«       rddlZddlmZ ddl	m
Z
  e
j                  e«      ZddiZd	„ Z G d
„ d«      Z G d„ de«      ZdgZy)z Tokenization classes for CPMAnt.é    N)Úis_rjieba_availableÚrequires_backendsé   )ÚPreTrainedTokenizer)ÚloggingÚ
vocab_filez	vocab.txtc                 óæ   — t        j                  «       }t        | dd¬«      5 }|j                  «       }ddd«       t	        «      D ]  \  }}|j                  d«      }|||<   Œ |S # 1 sw Y   Œ4xY w)z*Loads a vocabulary file into a dictionary.Úrúutf-8©ÚencodingNú
)ÚcollectionsÚOrderedDictÚopenÚ	readlinesÚ	enumerateÚrstrip)r   ÚvocabÚreaderÚtokensÚindexÚtokens         úy/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/cpmant/tokenization_cpmant.pyÚ
load_vocabr   "   sw   € ä×#Ñ#Ó%€EÜ	ˆj˜#¨Ô	0ð $°FØ×!Ñ!Ó#ˆ÷$ä! &Ó)ò ‰ˆˆuØ—‘˜TÓ"ˆØˆˆeŠðð €L÷$ð $ús   £A'Á'A0c                   ó   — e Zd Zdd„Zd„ Zy)ÚWordpieceTokenizerc                 ó.   — || _         || _        || _        y ©N)r   Ú	unk_tokenÚmax_input_chars_per_word)Úselfr   r    r!   s       r   Ú__init__zWordpieceTokenizer.__init__.   s   € ØˆŒ
Ø"ˆŒØ(@ˆÕ%ó    c                 ó˜  — t        |«      }t        |«      | j                  kD  r| j                  gS d}g }|t        |«      k  r‡t        |«      }d }||k  r0dj	                  ||| «      }|| j
                  v r|}n|dz  }||k  rŒ0|€!|j                  | j                  «       |dz  }n|j                  |«       |}|t        |«      k  rŒ‡|S )Nr   Ú é   )ÚlistÚlenr!   r    Újoinr   Úappend)r"   r   ÚcharsÚstartÚ
sub_tokensÚendÚ
cur_substrÚsubstrs           r   ÚtokenizezWordpieceTokenizer.tokenize3   sÜ   € Ü�U“ˆÜˆu‹:˜×5Ñ5Ò5Ø—N‘NÐ#Ð#àˆØˆ
Ø”c˜%“jÒ Ü�e“*ˆCØˆJØ˜#’+ØŸ™  u¨SÐ!1Ó2�Ø˜TŸZ™ZÑ'Ø!'�JØØ�q‘�ð ˜#“+ð Ð!Ø×!Ñ! $§.¡.Ô1Ø˜‘
‘à×!Ñ! *Ô-Ø�ð ”c˜%“jÓ ð  Ðr$   N)ú<unk>éÈ   )Ú__name__Ú
__module__Ú__qualname__r#   r2   © r$   r   r   r   -   s   „ óAó
r$   r   c                   óì   ‡ — e Zd ZdZeZddgZdZ	 	 	 	 	 	 	 	 	 dˆ fd„	Ze	d„ «       Z
e	d„ «       Ze	d„ «       Ze	d	efd
„«       Zd„ Zd„ Zˆ fd„Zd„ Zdee   d	efd„Zd„ Zd„ Zddededz  d	ee   fd„Zˆ xZS )ÚCpmAntTokenizera²  
    Construct a CPMAnt tokenizer. Based on byte-level Byte-Pair-Encoding.

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        bod_token (`str`, *optional*, defaults to `"<d>"`):
            The beginning of document token.
        eod_token (`str`, *optional*, defaults to `"</d>"`):
            The end of document token.
        bos_token (`str`, *optional*, defaults to `"<s>"`):
            The beginning of sequence token.
        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token.
        line_token (`str`, *optional*, defaults to `"</n>"`):
            The line token.
        space_token (`str`, *optional*, defaults to `"</_>"`):
            The space token.
    Ú	input_idsÚattention_maskFc                 ó  •— t        | dg«       || _        || _        t        |«      | _        | j                  |	   | j                  d<   | j                  |   | j                  d<   | j                  |	= | j                  |= t        j                  t        | j                  j                  «       d„ ¬«      «      | _        | j                  j                  «       D ��ci c]  \  }}||“Œ
 c}}| _	        t        | j                  |¬«      | _        t        ‰| �4  d||||||||	|
ddd	d
œ|¤Ž |	|fD ]=  }| j                  j                  |d «      }|€Œ"| j                   j                  |d «       Œ? | j#                  «        y c c}}w )NÚrjiebaú r   c                 ó   — | d   S ©Nr'   r8   ©Úxs    r   ú<lambda>z*CpmAntTokenizer.__init__.<locals>.<lambda>‚   ó   € ÐZ[Ð\]ÑZ^€ r$   ©Úkey)r   r    Ú	all_zerosTÚbos)Ú	bod_tokenÚ	eod_tokenÚ	bos_tokenÚ	eos_tokenÚ	pad_tokenr    Ú
line_tokenÚspace_tokenÚpadding_sideÚtoken_type_ids_patternÚ%token_type_ids_include_special_tokensÚspecial_tokens_patternr8   )r   rJ   rK   r   Úencoderr   r   ÚsortedÚitemsÚdecoderr   Úwordpiece_tokenizerÚsuperr#   Úadded_tokens_encoderÚpopÚ_added_tokens_decoderÚ_update_total_vocab_size)r"   r   rJ   rK   rL   rM   rN   r    rO   rP   rQ   ÚkwargsÚkÚvÚspecial_tokenÚtoken_idÚ	__class__s                   €r   r#   zCpmAntTokenizer.__init__j   sp  ø€ ô 	˜$  
Ô+Ø"ˆŒØ"ˆŒÜ! *Ó-ˆŒØ ŸL™L¨Ñ5ˆ�‰�SÑØ!Ÿ\™\¨*Ñ5ˆ�‰�TÑà�L‰L˜Ð%Ø�L‰L˜Ð$ä"×.Ñ.¬v°d·l±l×6HÑ6HÓ6JÑP^Ô/_Ó`ˆŒØ)-¯©×);Ñ);Ó)=×>¡  A˜˜1™Ó>ˆŒä#5¸D¿L¹LÐT]Ô#^ˆÔ ä‰Ñð 	
ØØØØØØØ!Ø#Ø%Ø#.Ø26Ø#(ñ	
ð ò	
ð *¨:Ð6ò 	?ˆMØ×0Ñ0×4Ñ4°]ÀDÓIˆHØÑ#Ø×*Ñ*×.Ñ.¨x¸Õ>ð	?ð 	×%Ñ%Õ'ùó/ ?s   ÃE=c                 ó4   — | j                   | j                     S r   )rU   rJ   ©r"   s    r   Úbod_token_idzCpmAntTokenizer.bod_token_idœ   ó   € à�|‰|˜DŸN™NÑ+Ð+r$   c                 ó4   — | j                   | j                     S r   )rU   rK   rf   s    r   Úeod_token_idzCpmAntTokenizer.eod_token_id    rh   r$   c                 ó    — | j                   d   S )Nr   ©rU   rf   s    r   Ú
newline_idzCpmAntTokenizer.newline_id¤   s   € à�|‰|˜DÑ!Ð!r$   Úreturnc                 ó,   — t        | j                  «      S r   )r)   rU   rf   s    r   Ú
vocab_sizezCpmAntTokenizer.vocab_size¨   s   € ä�4—<‘<Ó Ð r$   c                 óB   — t        | j                  fi | j                  ¤ŽS r   )ÚdictrU   r[   rf   s    r   Ú	get_vocabzCpmAntTokenizer.get_vocab¬   s   € Ü�D—L‘LÑ> D×$=Ñ$=Ñ>Ð>r$   c                 ó”   — g }t        j                  |d«      D ],  }|j                  | j                  j	                  |«      «       Œ. |S )zTokenize a string.F)r>   ÚcutÚextendrY   r2   )r"   ÚtextÚoutput_tokensrC   s       r   Ú	_tokenizezCpmAntTokenizer._tokenize¯   sH   € àˆÜ—‘˜D %Ó(ò 	GˆAØ× Ñ  ×!9Ñ!9×!BÑ!BÀ1Ó!EÕFð	GàÐr$   c                 óà   •— |D �cg c]
  }|dk\  sŒ	|‘Œ }}|D �cg c]4  }|| j                   k7  sŒ|| j                  k7  sŒ#|| j                  k7  sŒ3|‘Œ6 }}t        ‰| �  |fi |¤ŽS c c}w c c}w )zDecode ids into a string.r   )Úpad_token_idÚeos_token_idÚbos_token_idrZ   Ú_decode)r"   Ú	token_idsr_   ÚirC   rd   s        €r   r~   zCpmAntTokenizer._decode¶   s   ø€ à )Ö4˜1¨Q°!«V’QÐ4ˆ	Ð4à ö
Ø A¨×):Ñ):Ó$:¸qÀD×DUÑDUÓ?UÐZ[Ð_c×_pÑ_pÓZpŠAð
ˆ	ð 
ô ‰w‰˜yÑ3¨FÑ3Ð3ùò	 5ùò
s    †
A&‘A&›A+°A+Á A+ÁA+c                 ó   — || j                   v S r   rl   ©r"   r   s     r   ÚcheckzCpmAntTokenizer.check¾   s   € Ø˜Ÿ™Ð$Ð$r$   r   c                 ó$   — dj                  |«      S )Nr&   )r*   )r"   r   s     r   Úconvert_tokens_to_stringz(CpmAntTokenizer.convert_tokens_to_stringÁ   s   € Ø�w‰w�v‹Ðr$   c                 ó€   — | j                   j                  || j                   j                  | j                  «      «      S )z0Converts a token (str) in an id using the vocab.)rU   Úgetr    r‚   s     r   Ú_convert_token_to_idz$CpmAntTokenizer._convert_token_to_idÄ   s,   € à�|‰|×Ñ  t§|¡|×'7Ñ'7¸¿¹Ó'GÓHÐHr$   c                 óN   — | j                   j                  || j                  «      S )z=Converts an index (integer) in a token (str) using the vocab.)rX   r‡   r    )r"   r   s     r   Ú_convert_id_to_tokenz$CpmAntTokenizer._convert_id_to_tokenÈ   s   € à�|‰|×Ñ  t§~¡~Ó6Ð6r$   NÚsave_directoryÚfilename_prefixc                 ó   — t         j                  j                  |«      r2t         j                  j                  ||r|dz   ndt        d   z   «      }n|r|dz   nd|z   }d}d| j
                  v r)| j
                  d   | j
                  d<   | j
                  d= d| j
                  v r)| j
                  d   | j
                  d<   | j
                  d= t        j                  t        | j
                  j                  «       d	„ ¬
«      «      | _        t        |dd¬«      5 }| j
                  j                  «       D ]>  \  }}||k7  rt        j                  d|› d�«       |}|j                  |dz   «       |dz  }Œ@ 	 d d d «       |fS # 1 sw Y   |fS xY w)Nú-r&   r   r   r?   ú</_>r   ú</n>c                 ó   — | d   S rA   r8   rB   s    r   rD   z1CpmAntTokenizer.save_vocabulary.<locals>.<lambda>Ú   rE   r$   rF   Úwr   r   zSaving vocabulary to z\: vocabulary indices are not consecutive. Please check that the vocabulary is not corrupted!r'   )ÚosÚpathÚisdirr*   ÚVOCAB_FILES_NAMESrU   r   r   rV   rW   r   ÚloggerÚwarningÚwrite)r"   r‹   rŒ   r   r   Úwriterr   Útoken_indexs           r   Úsave_vocabularyzCpmAntTokenizer.save_vocabularyÌ   sx  € Ü�7‰7�=‰=˜Ô(ÜŸ™Ÿ™Ø¹/ °3Ò!6ÈrÔUfÐgsÑUtÑ tó‰Jñ 4C˜/¨CÒ/ÈÈnÑ\ˆJØˆØ�$—,‘,ÑØ#'§<¡<°Ñ#4ˆD�L‰L˜Ñ Ø—‘˜SÐ!Ø�4—<‘<ÑØ#'§<¡<°Ñ#5ˆD�L‰L˜Ñ Ø—‘˜TÐ"Ü"×.Ñ.¬v°d·l±l×6HÑ6HÓ6JÑP^Ô/_Ó`ˆŒÜ�*˜c¨GÔ4ð 		¸Ø&*§l¡l×&8Ñ&8Ó&:ò Ñ"��{Ø˜KÒ'Ü—N‘NØ/°
¨|ð <Nð Nôð (�EØ—‘˜U T™\Ô*Ø˜‘
‘ñ÷		ð ˆ}Ð÷		ð ˆ}Ðús   ÄAFÆF)	z<d>z</d>z<s>z</s>z<pad>r3   r�   r�   Úleftr   )r5   r6   r7   Ú__doc__r–   Úvocab_files_namesÚmodel_input_namesÚadd_prefix_spacer#   Úpropertyrg   rj   rm   Úintrp   rs   ry   r~   rƒ   r(   Ústrr…   rˆ   rŠ   Útuplerœ   Ú__classcell__)rd   s   @r   r:   r:   M   s÷   ø„ ñð0 *ÐØ$Ð&6Ð7ÐØÐð
 ØØØØØØØØõ0(ðd ñ,ó ð,ð ñ,ó ð,ð ñ"ó ð"ð ð!˜Cò !ó ð!ò?òô4ò%ð¨t°C©yð ¸Só òIò7ñ¨cð ÀCÈ$ÁJð ÐZ_Ð`cÑZd÷ r$   r:   )rž   r   r“   Útransformers.utilsr   r   r>   Útokenization_pythonr   Úutilsr   Ú
get_loggerr5   r—   r–   r   r   r:   Ú__all__r8   r$   r   ú<module>r¬      sm   ðñ 'ã Û 	ç Eñ ÔÛå 6Ý ð 
ˆ×	Ñ	˜HÓ	%€à! ;Ð/Ð ò÷ñ ô@XÐ)ô Xðv Ð
�r$   