
    ^j1                        U d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZ ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZ ddlmZmZ ddlmZ ddl m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'  e       rddl(m)Z) ndZ) e       rddl*m+Z+ ndZ+ ejX                  e-      Z.i Z/e0e1e2e	   f   e3d<   i Z4e0e1e2e	   f   e3d<    ee1e1dz  f   g d e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rd ndfd! e       rd"ndfd# e       rdndfd$ e       rd%ndfd& e       rd'ndfd(d) e       rdndfd* e       rd+ndfd,d-d. e       rd/ndfd0 e       rdndfd1d2 e       rd3ndfd4d5 e       rdndfd6 e       rd7ndfd8d9 e       rdndfd:d; e       rd<ndfd=d> e       rdndfd?d@ e       rdndfdA e       rdndfdBdC e       rdDndfdE e       rd7ndfdF e       rd"ndfdG e       rd"ndfdH e       rd ndfdI e       rdndfdJ e       rd ndfdK e       rdLndfdMdNdOdPdQ e       rd7ndfdR e       rdSndfdT e       rdUndfdVdW e       rdXndfdY e       rdndfdZ e       rd[ndfd\ e       rdndfd] e       rd7ndfd^ e       rdndfd_d` e       rdandfdb e       rdcndfddde e       rdndfdf e       rdndfdg e       rdhndfdi e       rdjndfdkdl e       rdmndfdn e       rdXndfdo e       rdXndfdp e       rdXndfdq e       rdXndfdr e       rdXndfds e       rdXndfdt e       rdndfdu e       rdndfdv e       rdndfdw e       rdndfdx e       rdndfdy e       rdndfdz e       rdndfd{ e       rdndfd| e       rdndfd} e       rdndfd~ e       rdndfd e       rd7ndfd e       rdndfd e       rd7ndfd e       rdandfdd e       rd7ndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfdd e       rd ndfdd e       rdndfd e       rdndfd e       rd7ndfd e       rd7ndfd e       rd ndfd e       rd7ndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rd%ndfd e       rd%ndfdd e       rdndfd e       rdndfd e       rdandfd e       rdandfd e       rdndfd e       rdndfd e       rdndfd e       rdndfdd e       rdndfd e       rdndfdd e       rdndfd e       rdndfd e       rdn
 e       rdndfd e       rdn
 e       rdndfd e       rdn
 e       rdndfd e       rdn
 e       rdndfd e       rdn
 e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdandfdϑd e       rdndfd e       rdndfd e       rdndfd e       rdndfd֑d e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdandfd e       rdndfd e       rdndfd e       rdndfd e       rdandfd e       rdndfd e       rdndfd e       rdndfd e       rd7ndfd e       rd ndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfd e       rdndfdd e       rdndfd e       rd7ndfdd e       rdndfd e       rdn
 e       rdndfd e       rdndfd e       rdndfdd e       rdndfd  e       rd ndfd e       rd ndfd e       rd ndfd e       rd ndfd e       rd ndfd e       rd ndfd e       rd ndfd e       rd ndfd e       rd	ndfd
 e       rd	ndfd e       rd ndfd e       rd ndfd e       rd ndfd e       rd ndfd e       rd ndfd e       rd ndfdd e       rdndfd e       rdXndfd e       rdndfd e       rdndfd e       rdndfdddd e       rdndfd e       rdandfd e       rdndfd  e       rdndfd! e       rd"ndfd# e       rd"ndfd$ e       rdXndfd% e       rd&ndfd' e       rd(ndfd) e       rd*ndfd+ e       rd,ndfd-d. e       rdndfd/ e       rdndfd0 e       rd7ndfd1 e       rdndfd2 e       rdndfd3 e       rdXndfd4d5 e       rd6ndfd7 e       rdndfd8 e       rdndfd9 e       rd:ndfd; e       rdndfd<d=d> e       rd?ndfd@ e       rdndfdA e       rdndfdBdC e       rdn
 e       rdndfdD e       rdn
 e       rdndfdEdFdGdHdI e       rdJndfdK e       rdndfdL e       rdMndfdNdO e       rdndfdP e       rdndfdQ e       rdRndfdS e       rdandfdT e       rdndfdU e       rdndfdV e       rdXndf      Z5h dWZ6e7e1   e3dX<   e6D ]  Z8e8e5vs e       rdnde5e8<     e!e#e5      Z9 e#jt                         D  ci c]  \  } }|| 
 c}} Z;g dYZ<dZ Z=d[ Z>d\e1ej~                  e1   z  d]e@fd^ZAd_e1d]e2e	   dz  fd`ZB	 	 	 	 	 	 	 dld\e1ej~                  e1   z  dae1ej~                  e1   z  dz  dbe@dce0e1e1f   dz  dde@e1z  dz  dee1dz  dfe@dge1d]e0e1e	f   fdhZC G di dj      ZDdkdjgZEyc c}} w (m  zAuto Tokenizer class.    N)OrderedDict)Any)is_mistral_common_available   )PreTrainedConfig)get_class_from_dynamic_moduleresolve_trust_remote_code)load_gguf_checkpoint)TOKENIZER_CONFIG_FILE)extract_commit_hashis_g2p_en_availableis_sentencepiece_availableis_tokenizers_availablelogging)cached_filehas_file   )EncoderDecoderConfig   )_LazyAutoMapping)CONFIG_MAPPING_NAMES
AutoConfigconfig_class_to_model_typemodel_type_to_module_name!replace_list_option_in_docstrings)TokenizersBackend)SentencePieceBackendREGISTERED_TOKENIZER_CLASSESREGISTERED_FAST_ALIASESEvollaModelr   aimv2CLIPTokenizeralbertAlbertTokenizeralignBertTokenizerariaaudioflamingo3Qwen2Tokenizer
aya_visionCohereTokenizerbarkbartRobertaTokenizerbarthezBarthezTokenizer)bartphoBartphoTokenizerbertzbert-generationBertGenerationTokenizer)zbert-japaneseBertJapaneseTokenizer)bertweetBertweetTokenizerbig_birdBigBirdTokenizerbigbird_pegasus)biogptBioGptTokenizer
blenderbotBlenderbotTokenizer)zblenderbot-smallBlenderbotSmallTokenizerblipzblip-2GPT2Tokenizer)bridgetowerr.   bros)byt5ByT5Tokenizer	camembertCamembertTokenizer)canineCanineTokenizerchinese_clip)clapr.   clipclipseg)clvpClvpTokenizer
code_llamaCodeLlamaTokenizercodegencoherecohere2colqwen2convbertcosmos3_omnicpmCpmTokenizer)cpmantCpmAntTokenizer)ctrlCTRLTokenizer)zdata2vec-audioWav2Vec2CTCTokenizer)zdata2vec-textr.   dbrxdebertaDebertaTokenizerz
deberta-v2DebertaV2Tokenizer)diaDiaTokenizerdiffusion_gemmaGemmaTokenizer
distilbertdprDPRQuestionEncoderTokenizerelectraemu3ernie)esmEsmTokenizerfalcon_mambaGPTNeoXTokenizerfastspeech2_conformerFastSpeech2ConformerTokenizer)flaubertFlaubertTokenizerflava	flex_olmo	florence2BartTokenizerfnetFNetTokenizer)fsmtFSMTTokenizerfunnelFunnelTokenizergemmagemma2gemma3gemma3_textgemma3ngemma3n_textgitglmglm4glm4_moeglm4_moe_liteglm4v	glm4v_moe	glm_imageglmasrgot_ocr2zgpt-sw3GPTSw3Tokenizergpt2gpt_bigcodegpt_neogpt_neox)gpt_neox_japaneseGPTNeoXJapaneseTokenizergptjgranite
granitemoegranitemoehybridgranitemoesharedzgrounding-dinogroupvitherbertHerbertTokenizer)hubertr^   
hunyuan_vl)ibertr.   ideficsLlamaTokenizeridefics2instructblipinstructblipvideointernvljais2jina_embeddings_v3XLMRobertaTokenizerkimi_k25zkosmos-2lasr_ctcLasrTokenizerlasr_encoderlayoutlm
layoutlmv2LayoutLMv2Tokenizer
layoutlmv3LayoutLMv3Tokenizer	layoutxlmLayoutXLMTokenizerledLEDTokenizerlighton_ocrQwen2TokenizerFastlilt
longformer)lukeLukeTokenizerlxmertLxmertTokenizerm2m_100M2M100Tokenizermambamamba2marianMarianTokenizermarkuplmMarkupLMTokenizermbartMBartTokenizermbart50MBart50Tokenizer)megar.   zmegatron-bert
metaclip_2)zmgp-strMgpstrTokenizermimo_v2_flashminicpmv4_6	ministralMistralCommonBackend
ministral3mistralmistral3mixtralmlukeMLukeTokenizerzmm-grounding-dino
mobilebertMobileBertTokenizermpnetMPNetTokenizermpt)mrar.   mt5T5Tokenizermusicgenmusicgen_melodymvpMvpTokenizer)myt5MyT5Tokenizernemotron3_5_asrParakeetTokenizernemotron_asr_streamingnezhanllbNllbTokenizerznllb-moe
nomic_bertnougatNougatTokenizernystromformerolmoolmo2olmo3olmo_hybridolmoezomdet-turbo	oneformerz
openai-gptOpenAIGPTTokenizeroptovis2owlv2owlvitparakeet_ctcparakeet_rnntparakeet_tdtpegasusPegasusTokenizer	pegasus_x)	perceiverPerceiverTokenizer	persimmonphi)phobertPhobertTokenizer
pix2structpixtralplbartPLBartTokenizerpp_formulanet)
prophetnetProphetNetTokenizerqdqbertqianfan_ocrqwen2qwen2_5_omni
qwen2_5_vlqwen2_audio	qwen2_moeqwen2_vlqwen3qwen3_5Qwen3_5Tokenizerqwen3_5_moe	qwen3_asr	qwen3_moe
qwen3_nextqwen3_omni_moeqwen3_vlqwen3_vl_moe)ragRagTokenizerrealmrecurrent_gemmareformerReformerTokenizerrembertRemBertTokenizer	retribert)robertar.   )zroberta-prelayernormr.   )roc_bertRoCBertTokenizerroformerRoFormerTokenizerrwkvsam3
sam3_videoseamless_m4tSeamlessM4TTokenizerseamless_m4t_v2shieldgemma2siglipSiglipTokenizersiglip2Siglip2Tokenizerspeech_to_textSpeech2TextTokenizerspeecht5SpeechT5Tokenizer)splinterSplinterTokenizersqueezebertstablelm
starcoder2switch_transformerst5t5gemma)tapasTapasTokenizertipsv2Tipsv2TokenizertrocrtvpudopUdopTokenizerumt5)	unispeechr^   )zunispeech-satr^   
videoprismVideoPrismTokenizerviltvisual_bert)vitsVitsTokenizervoxtralvoxtral_realtime)wav2vec2r^   )zwav2vec2-bertr^   )zwav2vec2-conformerr^   )wav2vec2_phonemeWav2Vec2PhonemeCTCTokenizerwhisperWhisperTokenizerxclipxglmXGLMTokenizer)xlmXLMTokenizerzxlm-robertazxlm-roberta-xlxlnetXLNetTokenizerxlstmxmodyosozaya>-   camembertv2-basevision-encoder-decoderfuyuphi3jambajanusllavamolmonvfp4r  arcticchatlmmolmo2phi3_vphimoeopencuaopenvlasmolvlmstep3p5ernie4_5minicpm3minicpmvnemotronstep3_vlvipllava	chameleon	internlm2
cohere_asr
h2ovl_chat
llava_next
minimax_m2
modernbertdeepseek_v2deepseek_v3deepseek_v4deepseek_vlhyperclovaxdeepseek_ocrdeepseek_v32ernie4_5_moepaddleocr_vldeepseek_ocr2deepseek_vl_v2hyperclovax_vlmdeepseek_vl_hybrid)MODELS_WITH_INCORRECT_HUB_TOKENIZER_CLASS)z!deepseek-ai/deepseek-r1-distill-*zdeepseek-ai/deepseek-coder-*zallenai/dolma2-tokenizerzgoogle/umt5-smallzsalesforce/blip2-opt-*zsalesforce/blip2-flan-t5-*z!salesforce/instructblip-flan-t5-*c                 t    t        | dd      5 }t        j                  |      cddd       S # 1 sw Y   yxY w)z*Loads a vocabulary file into a dictionary.rutf-8encodingN)openjsonload)
vocab_filereaders     u/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/auto/tokenization_auto.py
load_vocabr    s1    	j#	0 !Fyy ! ! !s   .7c                     g }t        | dd      5 }|D ]O  }|j                         }|s|j                  d      r(|j                  t	        |j                                      Q 	 ddd       |S # 1 sw Y   |S xY w)z Loads a merges file into a list.r  r  r  #N)r  strip
startswithappendtuplesplit)merges_filemergesr  lines       r  load_mergesr    sr    F	k3	1 3V 	3D::<DDOOC0eDJJL12	33
 M3
 Ms   A1A1*A11A;pretrained_model_name_or_pathreturnc                 *   |j                  dd      }|r t        j                  j                  |d      nd}	 t	        | ||j                  d      |j                  d      |j                  d      |j                  dd      	      S # t
        $ r Y yw xY w)
N	subfolder ztekken.jsonrevisiontoken	cache_dirlocal_files_onlyF)r  r  r  r  )getospathjoinr   OSError)r  kwargsr  tekken_filenames       r  _has_tekken_tokenizer_filer    s     

;+I@Ibggll9m<}O
)ZZ
+**W%jj-#ZZ(:EB
 	
  s   AB 	BB
class_namec                 f   | dv rt         S | t        v r	t        |    S | t        v r	t        |    S | dk(  rt         S t        j	                         D ]  \  }}|| k(  st        |      }|dv r| dk(  rt        j                  dd      }nt        j                  d| d      }	 t        ||       }t        |d	d       x}rb|t        j                  v rPt        j                  |   }t        ||j                  d
z   |       t        j                  j                  |dz   |       |c S  t        j                   j#                         D ]  }t        |dd       | k(  s|c S  t        j                  d      }t%        ||       rt        ||       S | j'                  d
      rt)        | d d       S y # t        $ r Y cw xY w)N>   BloomTokenizerBloomTokenizerFastr   )r   r   r   r   r   r  rU  r   z.tokenization_mistral_commontransformers.ztransformers.models
__module__Fast_fast__name__)r   r   r   TOKENIZER_MAPPING_NAMESitemsr   	importlibimport_modulegetattrsysmodulessetattrr  
setdefaultAttributeErrorTOKENIZER_MAPPING_extra_contentvalueshasattrendswithtokenizer_class_from_name)	r  module_nametokenizer_classmoduleresultsubmodbase_mod	tokenizermain_modules	            r  r  r    s   ==  ,,&z2211+J77((   )@(E(E(G $_j(3K@Krr"88"001OQ_`"001[M1BDYZ	 4%flDAAFAvQTQ\Q\G\"{{62HHfoo&>GKK**6G+;XF#* '55<<> 	9j$/:= )).9K{J'{J// 6"(CR99% " s   A>F##	F0/F0r  force_downloadproxiesr  r  r  r  c                 "   |j                  d      }	t        | t        |||||||ddd|	      }
|
t        j	                  d       i S t        |
|	      }	t        |
d      5 }t        j                  |      }ddd       |	d<   |S # 1 sw Y   xY w)aY  
    Loads the tokenizer configuration from a pretrained model tokenizer configuration.

    Args:
        pretrained_model_name_or_path (`str` or `os.PathLike`):
            This can be either:

            - a string, the *model id* of a pretrained model configuration hosted inside a model repo on
              huggingface.co.
            - a path to a *directory* containing a configuration file saved using the
              [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.

        cache_dir (`str` or `os.PathLike`, *optional*):
            Path to a directory in which a downloaded pretrained model configuration should be cached if the standard
            cache should not be used.
        force_download (`bool`, *optional*, defaults to `False`):
            Whether or not to force to (re-)download the configuration files and override the cached versions if they
            exist.
        proxies (`dict[str, str]`, *optional*):
            A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
            'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
        token (`str` or *bool*, *optional*):
            The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
            when running `hf auth login` (stored in `~/.huggingface`).
        revision (`str`, *optional*, defaults to `"main"`):
            The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
            git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
            identifier allowed by git.
        local_files_only (`bool`, *optional*, defaults to `False`):
            If `True`, will only try to load the tokenizer configuration from local files.
        subfolder (`str`, *optional*, defaults to `""`):
            In case the tokenizer config is located inside a subfolder of the model repo on huggingface.co, you can
            specify the folder name here.

    <Tip>

    Passing `token=True` is required when you want to use a private model.

    </Tip>

    Returns:
        `dict`: The configuration of the tokenizer.

    Examples:

    ```python
    # Download configuration from huggingface.co and cache.
    tokenizer_config = get_tokenizer_config("google-bert/bert-base-uncased")
    # This model does not have a tokenizer config so the result will be an empty dict.
    tokenizer_config = get_tokenizer_config("FacebookAI/xlm-roberta-base")

    # Save a pretrained tokenizer locally and you can reload its config
    from transformers import AutoTokenizer

    tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-cased")
    tokenizer.save_pretrained("tokenizer-test")
    tokenizer_config = get_tokenizer_config("tokenizer-test")
    ```_commit_hashF)r  r  r  r  r  r  r   _raise_exceptions_for_gated_repo%_raise_exceptions_for_missing_entries'_raise_exceptions_for_connection_errorsr  Nz\Could not locate the tokenizer configuration file, will try to use the model config instead.r  r  )	r  r   r   loggerinfor   r  r  r  )r  r  r  r  r  r  r  r  r  commit_hashresolved_config_filer  r  s                r  get_tokenizer_configr    s    J **^,K&%%))..305  #rs	%&:KHK	"W	5 #6"#(F>M# #s    BBc                   \    e Zd ZdZd Ze ee      dee	z  fd              Z
e	 dd       Zy)AutoTokenizera  
    This is a generic tokenizer class that will be instantiated as one of the tokenizer classes of the library when
    created with the [`AutoTokenizer.from_pretrained`] class method.

    This class cannot be instantiated directly using `__init__()` (throws an error).
    c                     t        d      )Nz}AutoTokenizer is designed to be instantiated using the `AutoTokenizer.from_pretrained(pretrained_model_name_or_path)` method.)r  )selfs    r  __init__zAutoTokenizer.__init__s  s    _
 	
    r  c           	         |j                  dd      }d|d<   |j                  dd      }|j                  dd      }|j                  dd      }|j                  d      }|vt        j                  |d      }	|	,t        d	| d
dj	                  d t        D               d      t        |	      }
|
t        d|	 d       |
j                  |g|i |S |r3t        ||fi |}t        |d      d   }t        j                  d2i |}n|	 t        j                  |fd|i|}|j                  }t        |d      r|j                  nd}t!        |fi |}|j                  dd      }d}d|v r4t#        |d   t$        t&        f      r|d   }n|d   j                  dd      }t#        t)        |dd      x}t*              r|j-                         nd|7t.        1t1        fdt2        D              rt/        j                  |g|i |S |xs t)        |dd      }|5|2|/|dk7  r)t        j                  |      t        j                  |      j5                  d      |j5                  d      k7  rt        j                  |      j5                  d      }|dvrE|t6        v s|t6        v r|n|}t        |      }
|
$|
j8                  dvr |
j                  |g|i |S |dk(  r=t;               r3d|vr/t=        |fi |r#t        d      }
|
 |
j                  |g|i |S t.        t/        j                  |g|i |S t        d| d      d |v r|d    |d <   |r|j?                  d      r|dd! }|du}tA        |      tB        v xs% |duxr t        |      duxs t        |dz         du}|xrN tA        |      tB        vxr; |duxr5 t        |      xs t        |dz         jD                  jG                  d"       }|r|t6        v r|durd}d}|r<|s:|d#   |d#   }n|d$   }d%|v r|jI                  d%      d$   }nd}tK        |||||      }|rg|re|sc|rt        |j5                  d             tM        |fi |}
|j                  d&d      }|
jO                           |
j                  |g|d|i|S |c|}t        |      }
|
|j?                  d      st        |dz         }
|
|
j8                  d'k(  rt.        }
|
t.        }
 |
j                  |g|i |S t)        |dd      rG|jP                  }d(|vr|j?                  d      r|dd! }t        |      }
 |
j                  |g|i |S t#        |tR              rztA        |jT                        tA        |jV                        urDtX        j[                  d)|jV                  j\                   d*|jT                  j\                   d+       |jV                  }t_        tA        |      j8                        xs t)        |d,d      }|atB        j                  tA        |      t.              }
|
<t)        |
d-d      dk(  rd|v st=        |fi |st.        }
 |
j                  |g|i |S |j                  dd      }|||d.k7  r|j?                  d      r|dd! }t        |      }
|
|j?                  d      st        |dz         }
|
|
j8                  d'k(  rt.        }
|
t.        }
 |
j                  |g|i |S t        d/|j\                   d0dj	                  d1 tB        D               d      # t        t        f$ r t        j                  |fi |}Y [w xY w)3a  
        Instantiate one of the tokenizer classes of the library from a pretrained model vocabulary.

        The tokenizer class to instantiate is selected based on the `model_type` property of the config object (either
        passed as an argument or loaded from `pretrained_model_name_or_path` if possible), or when it's missing, by
        falling back to using pattern matching on `pretrained_model_name_or_path`:

        List options

        Params:
            pretrained_model_name_or_path (`str` or `os.PathLike`):
                Can be either:

                    - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co.
                    - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved
                      using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.
                    - a path to a single saved vocabulary file if and only if the tokenizer only requires a
                      single vocabulary file (like Bert or XLNet), e.g.: `./my_model_directory/vocab.txt`. (Not
                      applicable to all derived classes)
            inputs (additional positional arguments, *optional*):
                Will be passed along to the Tokenizer `__init__()` method.
            config ([`PreTrainedConfig`], *optional*)
                The configuration object used to determine the tokenizer class to instantiate.
            cache_dir (`str` or `os.PathLike`, *optional*):
                Path to a directory in which a downloaded pretrained model configuration should be cached if the
                standard cache should not be used.
            force_download (`bool`, *optional*, defaults to `False`):
                Whether or not to force the (re-)download the model weights and configuration files and override the
                cached versions if they exist.
            proxies (`dict[str, str]`, *optional*):
                A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
                'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.
            revision (`str`, *optional*, defaults to `"main"`):
                The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
                git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
                identifier allowed by git.
            subfolder (`str`, *optional*):
                In case the relevant files are located inside a subfolder of the model repo on huggingface.co (e.g. for
                facebook/rag-token-base), specify it here.
            tokenizer_type (`str`, *optional*):
                Tokenizer type to be loaded.
            backend (`str`, *optional*, defaults to `"tokenizers"`):
                Backend to use for tokenization. Valid options are:
                - `"tokenizers"`: Use the HuggingFace tokenizers library backend (default)
                - `"sentencepiece"`: Use the SentencePiece backend
            trust_remote_code (`bool`, *optional*, defaults to `False`):
                Whether or not to allow for custom models defined on the Hub in their own modeling files. This option
                should only be set to `True` for repositories you trust and in which you have read the code, as it will
                execute code present on the Hub on your local machine.
            kwargs (additional keyword arguments, *optional*):
                Will be passed to the Tokenizer `__init__()` method. Can be used to set special tokens like
                `bos_token`, `eos_token`, `unk_token`, `sep_token`, `pad_token`, `cls_token`, `mask_token`,
                `additional_special_tokens`. See parameters in the `__init__()` for more details.

        Examples:

        ```python
        >>> from transformers import AutoTokenizer

        >>> # Download vocabulary from huggingface.co and cache.
        >>> tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased")

        >>> # Download vocabulary from huggingface.co (user-uploaded) and cache.
        >>> tokenizer = AutoTokenizer.from_pretrained("dbmdz/bert-base-german-cased")

        >>> # If vocabulary files are in a directory (e.g. tokenizer was saved using *save_pretrained('./test/saved_model/')*)
        >>> # tokenizer = AutoTokenizer.from_pretrained("./test/bert_saved_model/")

        >>> # Download vocabulary from huggingface.co and define model-specific arguments
        >>> tokenizer = AutoTokenizer.from_pretrained("FacebookAI/roberta-base", add_prefix_space=True)

        >>> # Explicitly use the tokenizers backend
        >>> tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer", backend="tokenizers")

        >>> # Explicitly use the sentencepiece backend
        >>> tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer", backend="sentencepiece")
        ```configNT
_from_autouse_fasttokenizer_typetrust_remote_code	gguf_filezPassed `tokenizer_type` z3 does not exist. `tokenizer_type` should be one of z, c              3       K   | ]  }|  y wN .0cs     r  	<genexpr>z0AutoTokenizer.from_pretrained.<locals>.<genexpr>  s      Dq Ds   r  zTokenizer class z is not currently imported.F)return_tensors
model_namer  auto_mapr  _name_or_pathr  c              3   J   K   | ]  }t        j                   |        y wr  )fnmatch)r  p_config_name_or_paths     r  r  z0AutoTokenizer.from_pretrained.<locals>.<genexpr>  s     fGOO$8!<fs    #r  )r   PythonBackendPreTrainedTokenizerFastr   )r   r  r  r   fix_mistral_regexzTokenizer class 'zf' specified in the tokenizer config was not found. The tokenizer may need to be converted or re-saved.r  r  ztransformers.r   r   z--code_revisionr  r  z The encoder model config class: z3 is different from the decoder model config class: z. It is not recommended to use the `AutoTokenizer.from_pretrained()` method in this case. Please use the encoder and decoder specific tokenizer classes.
model_typer  r   z!Unrecognized configuration class z8 to build an AutoTokenizer.
Model type should be one of c              3   4   K   | ]  }|j                     y wr  )r  r  s     r  r  z0AutoTokenizer.from_pretrained.<locals>.<genexpr>  s     4[AQZZ4[s   r  )0popr  r  
ValueErrorr  r  from_pretrainedr   r
   r   	for_modelr  r   r  r  r   r  
isinstancer  listr  strlowerr   anyMODEL_IDS_TO_TOKENIZERS_BACKENDremovesuffixr  r  r   r  r  typer  r  r  r  r	   r   register_for_auto_classr  r   decoderencoderr  warning	__class__r   )clsr  inputsr  r  _r  r  r  tokenizer_class_namer  	gguf_pathconfig_dictconfig_model_typeconfig_model_nametokenizer_configtokenizer_config_classtokenizer_auto_mapname
_hub_classregistered_class_namer  has_remote_codehas_local_codeexplicit_local_code	class_refupstream_repotokenizer_class_candidate_classr  r  s                                 @r  r  zAutoTokenizer.from_pretrainedy  s
	   d Hd+#| JJz4($4d;"JJ':DAJJ{+	 %#:#>#>~t#T #+ .~.>>qyy D,C DDEQH 
 88LMO& #34H3IId!eff2?223PdSYd]cdd#$A9WPVWI.yOPXYK))8K8F^c#331EVZ` #--181NF--TX 00MXQWX!1!5!56G!N "))*:6F%5j%A"%5j%A%E%EoW[%\" 'QU0V(VY\]DJJLce 	 &!-fFeff$445RfU[f_eff
 ,Wwv?PRV/W
&&!-!R''++,=>J(,,->?LLVT''/1 %<$?$?@Q$R$_$_`f$g!$ -  *-VV,0YY *
 $  #<J"G".?3K3K T 4
 ;?::;Xl[alekll &)??/1'v5./LWPVW";<R"S".:?::;Xl[alekll ,(889VjY_jcijj#J< 0F G 
 --%5n%EF>"!&<&E&Ef&M%;CR%@",D8f):: 
"$. )*@AM Z,-Cf-LMUYY	 	  V$55 'd2 9-.DE R01G&1PQ*ZZ89 	 !%NN!-#O!%#6!!$0.q1	.q1	y  ) 5a 8 $ 9!#@.Racp! 09L%)*@*M*Mf*UV;IGdohnoO

?D1A3352?22-06J[_e  $/(>%78QRO&/H/Q/QRX/Y";<UX^<^"_*/G/G?/Z"3&"32?223PdSYd]cddV.5++F(66??6;R7?O2?223PdSYd]cdd f23FNN#4+??6v~~7O7O6P Q%%+^^%=%=$> ?22 ^^F/V0E0EFm'RXZfhlJm
!/33DLBSTO*?J=AWW'6156S^W]^&7O6667ThW]haghh "2!5!56G!N!-%)<<AWA`A`agAh)?)D&78NOO&/E/N/Nv/V";<RU[<["\*/G/G?/Z"3&"32?223PdSYd]cdd/0@0@/A B++/994[IZ4[+[*\\]_
 	
Q ( c)99:Wb[abcs   ] %^^Nc                     |||}n||}nt        d      |||fD ]  }||t        |j                  <    |||t        |j                  <   t        j                  | ||       y)a  
        Register a new tokenizer in this mapping.

        Args:
            config_class ([`PreTrainedConfig`]):
                The configuration corresponding to the model to register.
            tokenizer_class: The tokenizer class to register (V5 - preferred parameter).
            slow_tokenizer_class: (Deprecated) The slow tokenizer to register.
            fast_tokenizer_class: (Deprecated) The fast tokenizer to register.
        Nz$You need to pass a `tokenizer_class`)exist_ok)r  r   r  r   r  register)config_classr  slow_tokenizer_classfast_tokenizer_classr4  	candidates         r  r5  zAutoTokenizer.register  s     "#/"6%1"6 !GHH.0DoV 	MI$CL,Y-?-?@	M  +0D0PEY#$8$A$AB""<8"Tr  )NNNF)r  r  __qualname____doc__r  classmethodr   r  r   r   r  staticmethodr5  r  r  r  r  r  k  sZ    
 &'>?~
	1	1~
 @ ~
@
 kpU Ur  r  r  )NFNNNFr  )Fr;  r  r  r  r  r  collectionsr   typingr   transformers.utils.import_utilsr   configuration_utilsr   dynamic_module_utilsr   r	   modeling_gguf_pytorch_utilsr
   tokenization_utils_baser   utilsr   r   r   r   r   	utils.hubr   r   encoder_decoderr   auto_factoryr   configuration_autor   r   r   r   r   tokenization_utils_tokenizersr    tokenization_utils_sentencepiecer   
get_loggerr  r  r   dictr  r  __annotations__r   r  r  setr  r  r  CONFIG_TO_TYPEr  r  r  PathLikeboolr  r  r  r  __all__)kvs   00r  <module>rV     s       	 
 #  G 3 \ ? <  / 2 *  BH			H	% 68 d3S	>2 702 c49n- 26+c3:o6g	/F/H+dSg	%<%>/DIg 
(?(A$tLg 
%<%>/DI	g
 
(?(A$tLg 
/F/H+dSg 
,C,E(4Pg 
$;$=4Hg 
'>'@#dKg 
*A*C&Ng 	(g 
$;$=4Hg 
9S9U5[_`g 	3g 	*g  
+B+D'$O!g" 
3J3L/RVW#g$ 	&%g& 
0G0I,tT'g( 	9)g* 
$;$=4H+g, 
&=&??TJ-g. 	,/g0 
$;$=4H1g2 	"3g4 
.E.G*TR5g6 	&7g8 
,C,E4P9g: 	%;g< 
$;$=4H=g> 
'>'@OdK?g@ 	"AgB 
/F/H+dSCgD 
'>'@OdKEgF 
(?(A$tLGgH 
)@)B%MIgJ 
)@)B%MKgL 
(?(A_tLMgN 
-D-F)DQOgP 
"9";FQgR 	&SgT 	"UgV 	3WgX 	.YgZ 
$;$=4H[g\ 
*A*C&N]g^ 
/F/H+dS_g` 	 agb 
0G0I,tTcgd 
*A*CNegf 
1H1J-PTUggh 
'>'@OdKigj 
$;$=4Hkgl 
%<%>/DImgn 	 ogp 
/F/H+dSqgr 
!EXEZ"A`desgt 	*ugv 
%<%>/DIwgx 
-D-F)DQygz 
)@)BoM{g| 
$;$=4H}g~ 	"g@ 
(?(A$tLAgB 
&=&?"TJCgD 
'>'@#dKEgF 
'>'@#dKGgH 
,C,E(4PIgJ 
(?(A$tLKgL 
-D-F)DQMgN 
#:#<$GOgP 
'>'@#dKQgR 
(?(A$tLSgT 
,C,E(4PUgV 
1H1J-PTUWgX 
)@)B%MYgZ 
-D-F)DQ[g\ 
-D-F)DQ]g^ 
*A*C&N_g` 
,C,E(4Pagb 
)C)E%4Pcgd 
$;$=4Hegf 
/F/H+dSggh 
'>'@OdKigj 
+B+D'$Okgl 	:mgn 
$;$=4Hogp 
+B+D'$Oqgr 
.E.G*TRsgt 
4K4M0SWXugv 
4K4M0SWXwgx 
.E.G?TRygz 
(?(A_tL{g| 
*A*C&N}g~ 	+g@ 
+B+D'$OAgB 	&CgD 
(?(A$tLEgF 
)@)B%MGgH 
,C,E4PIgJ 
1H1JoPTUKgL 
)@)B%MMgN 
%<%>/DIOgP 
8O8Q4W[\QgR 
,C,E(4PSgT 
,C,E(4PUgV 
(?(A_tLWgX 
,C,E4PYgZ 
(?(A_tL[g\ 
0G0I,tT]g^ 
0G0I,tT_g` 
.E.G*TRagb 
"9";Fcgd 
0G0I,tTegf 
'>'@#dKggh 
-D-F)DQigj 	"kgl 
(?(A$tLmgn 
)C)E%4Pogp 
(?(A$tLqgr 
)@)B%Msgt 
(B(D$$Ougv 
,C,E(4Pwgx 
&=&?"TJygz 
*A*C&N{g| 	%}g~ 
-D-F/DQg@ 
0G0I,tTAgB 	'CgD 
1H1J-PTUEgF 
/F/H+dSGgJ *, #)@)B%		
IgV *, #)@)B%		
Ugb *, #)@)B%		
agn *, #)@)B%		
mgz *, #)@)B%		
ygD 
&@&B"MEgF 
1H1JoPTUGgH 
0G0I,tTIgJ 
&=&?"TJKgL 
&=&?"TJMgN 	$OgP 
!8!:EQgR 
&=&?]TJSgT 
-D-FMDQUgV 
"9";FWgX 	"YgZ 
3J3L/RVW[g\ 
":Q:S#6Y]^]g^ 
%<%>/DI_g` 
$;$=4Hagb 
(?(A_tLcgd 
*A*CNegf 
(?(A$tLggh 
1H1J-PTUigj 
'>'@#dKkgl 
)@)B%Mmgn 
)@)B%Mogp 
/F/H+dSqgr 
(?(A$tLsgt 
+B+D$Ougv 
)@)BoMwgx 
/F/H+dSygz 
#:#<$G{g| 
&=&?"TJ}g~ 
%<%>/DIg@ 
&=&??TJAgB 
0G0I,tTCgD 
1H1J-PTUEgF 
0G0I,tTGgH 
*A*C&NIgJ 
-D-F)DQKgL 	,MgN 
-D-F)DQOgP 
#:#<$GQgR 	(SgT 
.E.G*TRUgX *, #)@)B%		
Wgb 
(?(A$tLcgd 
/F/H+dSegf 	.ggh 
'>'@OdKigj 
,C,E(4Pkgl 
&=&?"TJmgn 
-D-F)DQogp 
+B+D'$Oqgr 
,C,E(4Psgt 
*A*C&Nugv 
)@)B%Mwgx 
&=&?"TJygz 
*A*C&N{g| 
.E.G*TR}g~ 
*A*C&Ng@ 
*A*C&NAgB 
+B+D'$OCgD 
/F/H+dSEgF 
)@)B%MGgH 
-D-F)DQIgJ 	 KgL 
%<%>/DIMgN 
0G0I,tTOgP 
,C,E(4PQgR 
*A*C&NSgT 
)@)BoMUgV 	(WgX 	5YgZ 	)[g\ 
,C,E(4P]g^ 
'>'@#dK_g` 
$;$=4Hagb 
*A*CNcgd 
3J3L/RVWegf 
6M6O2UYZggh 
-D-F)DQigj 
(B(D$$Okgl 
*A*C&Nmgn 
5O5Q1W[\ogp 
,F,H(dSqgr 	*sgt 
+B+D$Ougv 
,C,E(4Pwgx 
*A*CNygz 
1H1JPTU{g| 
 7 9}tD}g~ 
(?(A$tLg@ 	$AgB 
(B(D$$OCgD 
+B+D'$OEgF 
#:#<$GGgH 
$;$=4HIgJ 
(?(A$tLKgL 	.MgN 	2OgP 
0G0I,tTQgR 
$;$=4HSgT 
+B+D$OUgV 	"WgZ *, #)@)B%		
Ygf *, #)@)B%		
egp 	-qgr 	2sgt 	7ugv 	<wgx 
*A*C&Nygz 
%<%>/DI{g| 
$;$=4H}g~ 	 g@	 
1H1J-PTUA	gB	 
2I2K.QUVC	gD	 
&=&?"TJE	gF	 
(?(A$tLG	gH	 
*A*C&NI	gJ	 
&=&?"TJK	gL	 
%<%>!DIM	gi Z	.7 )3s8 .` < iJ00E\E^.Adh
+i %%9;RS #=#7#=#=#?@41a!Q$@# !#&S)9#9 
&3# 3$s)d2B 3p 04 %)#"]#&S)9#9]R[[%%,] ] #s(^d"	]
 #:] Dj] ] ] 
#s(^]@nU nUb 
0q As   v