Ë
    ÛÓ^jÇ  ã                   óÌ  — d Z ddlZddlZddlmZ ddlmZ  ej                  e	«      Z
dddœZi d	d
“dd“dd“dd“dd“dd“dd“dd“dd“dd“dd“dd “d!d"“d#d$“d%d&“d'd(“d)d*“i d+d,“d-d.“d/d0“d1d2“d3d4“d5d6“d7d8“d9d:“d;d<“d=d>“d?d@“dAdB“dCdD“dEdF“dGdH“dIdJ“dKdL“¥i dMdN“dOdP“dQdR“dSdT“dUdV“dWdX“dYdZ“d[d\“d]d^“d_d`“dadb“dcdd“dedf“dgdh“didj“dkdl“dmdn“¥dodpdqdrdsœ¥Zdt„ Z G du„ dve«      ZdvgZy)wz)Tokenization classes for Salesforce CTRL.é    Né   )ÚPreTrainedTokenizer)Úloggingz
vocab.jsonz
merges.txt)Ú
vocab_fileÚmerges_fileÚ	Pregnancyiµ’ ÚChristianityiû  ÚExplaini·Ÿ ÚFitnessiÐ÷  ÚSavingi»ö  ÚAski#j  ÚAssiñv ÚJokeiµ~ Ú	Questionsi6²  ÚThoughtsiÅÁ  ÚRetailivÌ  ÚFeminismiò� ÚWritingiØ.  ÚAtheismiï ÚNetflixiè½  Ú	Computingi×š  ÚOpinioniÍ¨  ÚAlonei§¯  ÚFunnyi%æ  ÚGamingi¦�  ÚHumaniø  ÚIndiai3  ÚJokeriR- ÚDietin�  ÚLegaliS.  ÚNormaniK  ÚTipiñ ÚWeightiwÌ  ÚMoviesiÁ´  ÚRunningi�[  ÚSciencei*  ÚHorrori¡“  Ú
Confessioniœì  ÚFinanceiÚ/  ÚPoliticsiè?  ÚScaryiñí ÚSupportin1  ÚTechnologiesi  ÚTeenageip ÚEventi€  ÚLearnedi„ ÚNotioniòÉ Ú	WikipediaiÏ’  ÚBooksi	  ÚExtracti) ÚConfessionsi-‘ Ú
Conspiracyiœ( ÚLinksiºø  Ú	Narcissusi™K ÚRelationshipiîÕ  ÚRelationshipsiŒ iÇ¢  i   iÄh  i–õ )ÚReviewsÚNewsÚTranslationÚmultilingualc                 óx   — t        «       }| d   }| dd D ]  }|j                  ||f«       |}Œ t        |«      }|S )z…
    Return set of symbol pairs in a word.

    Word is represented as tuple of symbols (symbols being variable-length strings).
    r   é   N)ÚsetÚadd)ÚwordÚpairsÚ	prev_charÚchars       úu/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/ctrl/tokenization_ctrl.pyÚ	get_pairsrH   [   sO   € ô ‹E€EØ�Q‘€IØ�Q�R�ò ˆØ�	‰	�9˜dÐ#Ô$Ø‰	ðô �‹J€EØ€Ló    c                   ó`   ‡ — e Zd ZdZeZeZd
ˆ fd„	Ze	d„ «       Z
d„ Zd„ Zd„ Zd„ Zd„ Zd	„ Zˆ xZS )ÚCTRLTokenizera`  
    Construct a CTRL tokenizer. Based on Byte-Pair-Encoding.

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
    this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        merges_file (`str`):
            Path to the merges file.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
    c           
      ój  •— t        |d¬«      5 }t        j                  |«      | _        d d d «       | j                  j	                  «       D ��ci c]  \  }}||“Œ
 c}}| _        t        |d¬«      5 }|j                  «       j                  d«      dd }	d d d «       	D �
cg c]  }
t        |
j                  «       «      ‘Œ }	}
t        t        |	t        t        |	«      «      «      «      | _        i | _        d| _        t!        ‰| �D  d
|dddd	œ|¤Ž y # 1 sw Y   ŒéxY wc c}}w # 1 sw Y   Œ�xY wc c}
w )Nzutf-8)Úencodingú
r@   éÿÿÿÿTÚ	all_zerosÚnone)Ú	unk_tokenÚtoken_type_ids_patternÚ%token_type_ids_include_special_tokensÚspecial_tokens_pattern© )ÚopenÚjsonÚloadÚencoderÚitemsÚdecoderÚreadÚsplitÚtupleÚdictÚzipÚrangeÚlenÚ	bpe_ranksÚcacheÚadd_bpe_version_headerÚsuperÚ__init__)Úselfr   r   rR   ÚkwargsÚvocab_handleÚkÚvÚmerges_handleÚmergesÚmergeÚ	__class__s              €rG   rh   zCTRLTokenizer.__init__   s  ø€ Ü�* wÔ/ð 	3°<ÜŸ9™9 \Ó2ˆDŒL÷	3à)-¯©×);Ñ);Ó)=×>¡  A˜˜1™Ó>ˆŒÜ�+¨Ô0ð 	<°MØ"×'Ñ'Ó)×/Ñ/°Ó5°a¸Ð;ˆF÷	<à4:Ö;¨5”%˜Ÿ™›Õ&Ð;ˆÐ;Üœc &¬%´°F³Ó*<Ó=Ó>ˆŒØˆŒ
Ø&*ˆÔ#Ü‰Ñð 	
ØØ#.Ø26Ø#)ñ		
ð
 ó	
÷	3ð 	3üã>÷	<ð 	<üâ;s#   �DÁDÁ2#D$Â! D0ÄDÄ$D-c                 ó,   — t        | j                  «      S ©N)rc   rZ   ©ri   s    rG   Ú
vocab_sizezCTRLTokenizer.vocab_size‘   s   € ä�4—<‘<Ó Ð rI   c                 óB   — t        | j                  fi | j                  ¤ŽS rs   )r`   rZ   Úadded_tokens_encoderrt   s    rG   Ú	get_vocabzCTRLTokenizer.get_vocab•   s   € Ü�D—L‘LÑ> D×$=Ñ$=Ñ>Ð>rI   c                 ó$  ‡ — |‰ j                   v r‰ j                   |   S t        |«      }t        t        |d d «      |d   dz   gz   «      }t        |«      }|s|S 	 t	        |ˆ fd„¬«      }|‰ j
                  vrnÎ|\  }}g }d}|t        |«      k  r�	 |j                  ||«      }	|j                  |||	 «       |	}||   |k(  r6|t        |«      dz
  k  r%||dz      |k(  r|j                  ||z   «       |dz  }n|j                  ||   «       |dz  }|t        |«      k  rŒ�t        |«      }|}t        |«      dk(  rnt        |«      }Œídj                  |«      }|d d	 }|‰ j                   |<   |S # t        $ r |j                  ||d  «       Y Œnw xY w)
NrO   z</w>c                 óN   •— ‰j                   j                  | t        d«      «      S )NÚinf)rd   ÚgetÚfloat)Úpairri   s    €rG   ú<lambda>z#CTRLTokenizer.bpe.<locals>.<lambda>£   s   ø€ °·±×1CÑ1CÀDÌ%ÐPUË,Ó1W€ rI   )Úkeyr   r@   é   ú@@ éüÿÿÿ)re   r_   ÚlistrH   Úminrd   rc   ÚindexÚextendÚ
ValueErrorÚappendÚjoin)
ri   ÚtokenrC   rD   ÚbigramÚfirstÚsecondÚnew_wordÚiÚjs
   `         rG   ÚbpezCTRLTokenizer.bpe˜   s±  ø€ Ø�D—J‘JÑØ—:‘:˜eÑ$Ð$Ü�U‹|ˆÜ”T˜$˜s ˜)“_¨¨R©°6Ñ(9Ð':Ñ:Ó;ˆÜ˜$“ˆáØˆLàÜ˜Ó$WÔXˆFØ˜TŸ^™^Ñ+ØØ"‰MˆE�6ØˆHØˆAØ”c˜$“i’-ðØŸ
™
 5¨!Ó,�Að
 —O‘O D¨¨1 IÔ.Ø�Aà˜‘7˜eÒ#¨¬C°«I¸©MÒ(9¸dÀ1ÀqÁ5¹kÈVÒ>SØ—O‘O E¨F¡NÔ3Ø˜‘F‘Aà—O‘O D¨¡GÔ,Ø˜‘F�Að ”c˜$“i“-ô  ˜X“ˆHØˆDÜ�4‹y˜AŠ~Øä! $›�ð9 ð: �z‰z˜$ÓˆØ�C�RˆyˆØ ˆ�
‰
�5ÑØˆøô- "ò Ø—O‘O D¨¨ HÔ-Ùðús   ÂE/ Å/FÆFc                 ó´   — g }t        j                  d|«      }|D ]:  }|j                  t        | j	                  |«      j                  d«      «      «       Œ< |S )zTokenize a string.z\S+\n?ú )ÚreÚfindallr‡   r„   r’   r^   )ri   ÚtextÚsplit_tokensÚwordsr‹   s        rG   Ú	_tokenizezCTRLTokenizer._tokenizeÄ   sT   € àˆä—
‘
˜9 dÓ+ˆàò 	BˆEØ×Ñ¤ T§X¡X¨e£_×%:Ñ%:¸3Ó%?Ó @ÕAð	BàÐrI   c                 ó€   — | j                   j                  || j                   j                  | j                  «      «      S )z0Converts a token (str) in an id using the vocab.)rZ   r|   rR   )ri   r‹   s     rG   Ú_convert_token_to_idz"CTRLTokenizer._convert_token_to_idÎ   s,   € à�|‰|×Ñ  t§|¡|×'7Ñ'7¸¿¹Ó'GÓHÐHrI   c                 óN   — | j                   j                  || j                  «      S )z=Converts an index (integer) in a token (str) using the vocab.)r\   r|   rR   )ri   r†   s     rG   Ú_convert_id_to_tokenz"CTRLTokenizer._convert_id_to_tokenÒ   s   € à�|‰|×Ñ  t§~¡~Ó6Ð6rI   c                 ód   — dj                  |«      j                  dd«      j                  «       }|S )z:Converts a sequence of tokens (string) in a single string.r”   r‚   Ú )rŠ   ÚreplaceÚstrip)ri   ÚtokensÚ
out_strings      rG   Úconvert_tokens_to_stringz&CTRLTokenizer.convert_tokens_to_stringÖ   s,   € à—X‘X˜fÓ%×-Ñ-¨e°RÓ8×>Ñ>Ó@ˆ
ØÐrI   )z<unk>)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚCONTROL_CODESÚcontrol_codesrh   Úpropertyru   rx   r’   rš   rœ   rž   r¥   Ú__classcell__)rq   s   @rG   rK   rK   k   sN   ø„ ñð  *ÐØ!€Mõ
ð$ ñ!ó ð!ò?ò*òXòIò7örI   rK   )r©   rX   Úregexr•   Útokenization_pythonr   Úutilsr   Ú
get_loggerr¦   Úloggerrª   r¬   rH   rK   Ú__all__rV   rI   rG   ú<module>r¶      sþ  ðñ 0ã ã å 6Ý ð 
ˆ×	Ñ	˜HÓ	%€ð ØñÐ ð8Ø�ð8à�Dð8ð ˆvð8ð ˆuð	8ð
 ˆeð8ð 
ˆ5ð8ð 
ˆ5ð8ð ˆFð8ð �ð8ð �ð8ð ˆeð8ð �ð8ð ˆuð8ð ˆvð8ð ˆuð8ð  �ð!8ð" ˆuñ#8ð$ ˆUð%8ð& ˆUð'8ð( ˆeð)8ð* ˆTð+8ð, ˆTð-8ð. ˆUð/8ð0 ˆEð18ð2 ˆUð38ð4 ˆdð58ð6 
ˆ5ð78ð8 ˆeð98ð: ˆeð;8ð< ˆuð=8ð> ˆtð?8ð@ ˆeðA8ðB �%ðC8ðD ˆuòE8ðF �ðG8ðH ˆVðI8ðJ ˆuðK8ðL �EðM8ðN ˆuðO8ðP ˆUðQ8ðR ˆuðS8ðT ˆfðU8ðV �ðW8ðX ˆTðY8ðZ ˆuð[8ð\ �6ð]8ð^ �%ð_8ð` ˆUða8ðb �ðc8ðd �Eðe8ðf �Vñg8ðh ØØØòo8€òvô nÐ'ô nðn Ð
�rI   