Ë
    ÛÓ^jŒ  ã                   ód   — d Z ddlmZmZ ddlmZ  ej                  e«      Z G d„ de«      Z	dgZ
y)zTokenization class for Dia.é   )Ú
AddedTokenÚPreTrainedTokenizer)Úloggingc            	       ó¦   ‡ — e Zd ZdZddgZ	 	 	 	 ddedz  dedz  dedz  defˆ fd	„Zed
„ «       Z	d„ Z
dedee   fd„Zd„ Zd„ Zdee   defd„Zˆ xZS )ÚDiaTokenizera¸  
    Construct a Dia tokenizer. Dia simply uses raw bytes utf-8 encoding except for special tokens `[S1]` and `[S2]`.

    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    Args:
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        unk_token (`str`, *optional*, defaults to `"<pad>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        max_length (`int`, *optional*, defaults to 1024):
            The maximum length of the sequences when encoding. Sequences longer than this will be truncated.
        offset (`int`, *optional*, defaults to 0):
            The offset of the tokenizer.
    Ú	input_idsÚattention_maskÚ	pad_tokenNÚ	unk_tokenÚ
max_lengthÚoffsetc                 óþ   •— t        |t        «      rt        |«      n|}t        |t        «      rt        |«      n|}d| _        |t        d«      t        d«      dœ| _        || _        t        ‰| �  d	||||ddddœ|¤Ž y )
Né   z[S1]z[S2])é    é   é   Ú	all_zerosTÚnone)r   r
   r   r   Útoken_type_ids_patternÚ%token_type_ids_include_special_tokensÚspecial_tokens_pattern© )Ú
isinstanceÚstrr   Ú_utf_vocab_sizeÚ_added_tokens_decoderr   ÚsuperÚ__init__)Úselfr
   r   r   r   ÚkwargsÚ	__class__s         €ús/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/dia/tokenization_dia.pyr   zDiaTokenizer.__init__,   s‰   ø€ ô .8¸	Ä3Ô-G”J˜yÔ)ÈYˆ	Ü-7¸	Ä3Ô-G”J˜yÔ)ÈYˆ	à#ˆÔØ)2´zÀ&Ó7IÌjÐY_ÓN`Ñ%aˆÔ"ØˆŒÜ‰Ñð 		
ØØØ!ØØ#.Ø26Ø#)ñ		
ð ó		
ó    c                 ó   — | j                   S ©N)r   )r   s    r"   Ú
vocab_sizezDiaTokenizer.vocab_sizeF   s   € à×#Ñ#Ð#r#   c                 óÄ   — t        | j                  | j                  z   «      D �ci c]  }| j                  |«      |“Œ }}|j	                  | j
                  «       |S c c}w r%   )Úranger&   r   Úconvert_ids_to_tokensÚupdateÚadded_tokens_encoder)r   ÚiÚvocabs      r"   Ú	get_vocabzDiaTokenizer.get_vocabJ   sW   € Ü;@ÀÇÁÐSW×S^ÑS^ÑA^Ó;_Ö`°a�×+Ñ+¨AÓ.°Ñ1Ð`ˆÐ`Ø�‰�T×.Ñ.Ô/Øˆùò as   ¥AÚtextÚreturnc                 ó^   — |j                  d«      D �cg c]  }t        |«      ‘Œ }}|S c c}w )zPTake as input a string and return a list of strings (tokens) for words/sub-wordsúutf-8)ÚencodeÚchr)r   r/   r,   Útokenss       r"   Ú	_tokenizezDiaTokenizer._tokenizeO   s,   € à"&§+¡+¨gÓ"6Ö7˜Q”#�a•&Ð7ˆÐ7Øˆùò 8s   ”*c                 óZ   — t        |«      dk7  rd}|S t        |«      | j                  z   }|S )z0Converts a token (str) in an id using the vocab.r   N)ÚlenÚordr   )r   ÚtokenÚtoken_ids      r"   Ú_convert_token_to_idz!DiaTokenizer._convert_token_to_idT   s4   € ô ˆu‹:˜Š?ØˆHð ˆô ˜5“z D§K¡KÑ/ˆHàˆr#   c                 ó6   — t        || j                  z
  «      }|S )z=Converts an index (integer) in a token (str) using the vocab.)r4   r   )r   Úindexr:   s      r"   Ú_convert_id_to_tokenz!DiaTokenizer._convert_id_to_token^   s   € ä�E˜DŸK™KÑ'Ó(ˆØˆr#   r5   c                 ó  — d}|D ]p  }|| j                   v r*| j                   |   }t        |«      j                  d«      }n1|| j                  v r|j                  d«      }n|j                  d«      }||z  }Œr |j	                  dd¬«      }|S )z:Converts a sequence of tokens (string) in a single string.r#   r2   Úignore)Úerrors)r   r   r3   Ú_added_tokens_encoderÚdecode)r   r5   Úbstringr:   Úadded_token_objÚ
tok_stringÚstrings          r"   Úconvert_tokens_to_stringz%DiaTokenizer.convert_tokens_to_stringc   s–   € àˆØò 	"ˆEØ˜×2Ñ2Ñ2Ø"&×"<Ñ"<¸UÑ"C�Ü  Ó1×8Ñ8¸ÓA‘
Ø˜$×4Ñ4Ñ4Ø"Ÿ\™\¨'Ó2‘
à"Ÿ\™\¨'Ó2�
Ø�zÑ!‰Gð	"ð —‘ °�Ó9ˆØˆr#   )ú<pad>rJ   i   r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úmodel_input_namesr   Úintr   Úpropertyr&   r.   Úlistr6   r<   r?   rI   Ú__classcell__)r!   s   @r"   r   r      s¬   ø„ ñð$ %Ð&6Ð7Ðð !(Ø 'Ø!%Øñ
à˜‘:ð
ð ˜‘:ð
ð ˜$‘Jð	
ð
 õ
ð4 ñ$ó ð$òð
˜cð  d¨3¡ió ò
òð
¨t°C©yð ¸S÷ r#   r   N)rN   Útokenization_pythonr   r   Úutilsr   Ú
get_loggerrK   Úloggerr   Ú__all__r   r#   r"   ú<module>rY      s>   ðñ "ç BÝ ð 
ˆ×	Ñ	˜HÓ	%€ôYÐ&ô Yðx Ð
�r#   