Ë
    ÛÓ^j ƒ  ã                   óö  — d Z ddlmZ ddlZddlmZmZmZmZm	Z	 ddl
mZmZ ddlmZ ddlmZmZmZmZmZ dd	lmZ dd
lmZ  ej2                  e«      Zi ddddœ“ddddddddddddœ
“ddddddddddddœ
“ddddddddddddœ
“dddddddddddddd œ“d!ddddddddddd"dd#d$œ“d%ddddddddddd&d'œ“d(dddddddddddœ
“d)dddddddddddddd*œ“d+dddddddddddœ
“d,d-d.d/d0d1œ“d2dddddddddddœ
“d3d4dd5dd6d7œ“d8d9d:d;d<d=d>dd6d?d@ddAœ“dBddddddddCddDœ	“dEd4dFdGdHd5d6dIœ“dJdddddddKdLœ“dddd6ddMdNdOddPœ	dddddddddQddœ
ddddddddddd#ddRœddddddddddd#ddRœdddddddSdddd#ddRœd9d:d;d<d=d>dd6d?d@ddAœdddddddddddœ
i dTd“dUd“dHd“dVd“dWdX“dYd“dZd“d[d“d\d“d]d“d^d“d_d"“d`d“dad“dd“dbdc“ddœ¥Zdedfdgdhdid-d.d/d0djdkœ
dldd-d.d/d0dmœdnœZdodpidqdpidrœZds„ Z G dt„ du«      Z  G dv„ dwe«      Z! G dx„ dye«      Z" G dz„ d{e«      Z# G d|„ d}e«      Z$ G d~„ de«      Z% G d€„ d�e«      Z&i de!“de"“de"“d(e"“d)e"“d2e#“d3e$“d+e$“dBe$“dEe$“dJe$“d8e%“d‚e$“dƒe$“d„e&“d…e&“d†e&“e%e!e!e"d‡œ¥Z'dˆe(d‰e)ee*f   fdŠ„Z+y)‹z
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
with extra methods beings exposed
é    )ÚarrayN)Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEÚUnigramé   )Ú
AddedToken)ÚGemmaConverterÚGPT2ConverterÚLlamaConverterÚQwen2ConverterÚT5Converter)Úlogging)ÚtqdmÚgeneralÚ
model_typeÚ_model_name_or_path)ÚarchitectureÚnameÚllamaÚmax_position_embeddingsÚnum_hidden_layersÚintermediate_sizeÚhidden_sizeÚhead_dimÚ
rope_thetaÚnum_attention_headsÚnum_key_value_headsÚrms_norm_epsÚ
vocab_size)
Úcontext_lengthÚblock_countÚfeed_forward_lengthÚembedding_lengthúrope.dimension_countúrope.freq_baseúattention.head_countúattention.head_count_kvú attention.layer_norm_rms_epsilonr#   ÚmistralÚqwen2Ú	qwen2_moeÚnum_expertsÚnum_experts_per_tok)r$   r%   r&   r'   r(   r)   r*   r+   r,   r#   Úexpert_countÚexpert_used_countÚgpt_ossÚnum_local_expertsÚsliding_window)r$   r%   r&   r'   r(   r)   r*   r+   r,   r#   r2   r3   r6   Úlfm2Úconv_L_cache)r$   r%   r&   r'   r(   r)   r*   r+   r,   r#   zshortconv.l_cacheÚqwen3Ú	qwen3_moe)r$   r%   r&   r'   r(   r)   úattention.key_lengthr*   r+   r,   r#   r2   r3   ÚfalconÚ	tokenizerÚbos_token_idÚeos_token_idÚunk_token_idÚpad_token_id)úggml.bos_token_idúggml.eos_token_idúggml.unknown_token_idúggml.padding_token_idÚphi3ÚbloomÚn_layerÚn_headÚlayer_norm_epsilon)r%   r'   r*   r#   úattention.layer_norm_epsilonÚt5Ún_positionsÚ
num_layersÚd_ffÚd_modelÚd_kvÚ	num_headsÚrelative_attention_num_bucketsÚdecoder_start_token_id)r$   r%   r&   r'   r;   r*   r+   rK   z attention.relative_buckets_countrT   r#   ÚstablelmÚlayer_norm_eps)	r$   r%   r&   r'   r(   r*   r+   rK   r#   Úgpt2Ún_ctxÚn_embdr&   )r%   r$   r'   r&   r*   rK   Ú
starcoder2Únorm_epsilon)r%   r$   r'   r&   r*   r+   rK   Úconv_kernelÚ
state_sizeÚtime_step_rank)	r#   r$   r'   r,   r%   zssm.conv_kernelzssm.state_sizezssm.time_step_rankzssm.inner_sizeÚnorm_eps)r$   r%   r&   r'   r(   r)   r;   r*   r+   r,   zattention.sliding_windowr#   Úglobal_head_dimr$   r%   r'   r(   Ú
rotary_dimr)   r*   r+   r;   zattention.value_lengthr,   r2   r3   Úexpert_feed_forward_lengthÚexpert_gating_funcÚscoring_func)ÚmambaÚnemotronÚgemma2Úgemma3Úgemma4Úumt5ÚdeciÚ
minimax_m2Útokenizer_typeÚtokensÚscoresÚ
token_typeÚmergesÚadd_prefix_space)
ú
ggml.modelzggml.tokenszggml.scoreszggml.token_typezggml.mergesrB   rC   rD   rE   zggml.add_space_prefixÚchat_template)rt   rs   rB   rC   rD   rE   )r=   Útokenizer_configÚnorm_topk_probTÚuse_routing_bias)r:   rl   c                 ó”  — t        |t        «      s|g}t        |«      dk(  r|d   }d }n|d   dk7  rt        d«      ‚|\  }}|dv rt	        | d   «      } | S |dv rt        | d   «      } | S |dk(  rt        | d   «      } | S |dk(  r3t        d	t        | «      «      j                  «       j                  «       } | S |dk(  rt        | |«      } | S )
Né   r   é	   zPReceived multiple types, therefore expected the first type to indicate an array.)r   ry   r   é   é   é   é
   é   )é   é   é   é   ÚB)Ú
isinstanceÚlistÚlenÚ
ValueErrorÚintÚfloatÚboolr   ÚtobytesÚdecodeÚ_gguf_parse_value)Ú_valueÚ	data_typeÚarray_data_types      úi/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/integrations/ggml.pyrŽ   rŽ   ~  sô   € Ü�i¤Ô&Ø�Kˆ	Ü
ˆ9ƒ~˜ÒØ˜a‘Lˆ	Ø‰à�Q‰<˜1ÒÜÐoÓpÐpØ%.Ñ"ˆ	�?àÐ.Ñ.Ü�V˜A‘Y“ˆð €Mð 
�gÑ	Ü�v˜a‘yÓ!ˆð €Mð 
�aŠÜ�f˜Q‘i“ˆð
 €Mð	 
�aŠÜ�sœD ›LÓ)×1Ñ1Ó3×:Ñ:Ó<ˆð €Mð 
�aŠÜ" 6¨?Ó;ˆØ€Mó    c                   ó   — e Zd Zd„ Zy)ÚGGUFTokenizerSkeletonc                 óp  ‡— |j                  «       D ]  \  }}t        | ||«       Œ t        | d«      �s)t        | d«      rt        | d«      st        d«      ‚| j                  }| j
                  }t        |«      D ��ci c]  \  }}|||   “Œ c}}Št        j                  d«       g }t        ‰j                  «       «      D ]k  \  }	}
g }t        dt        |	«      «      D ]*  }|	d | |	|d  }}||v sŒ||v sŒ|j                  |||
f«       Œ, t        |ˆfd„d¬	«      }|j                  |«       Œm t        |d
„ d¬	«      }|D �cg c]  }|d   |d   f‘Œ }}|| _        nt| j                  D �	cg c]  }	t!        |	j#                  d«      «      ‘Œ c}	| _        t        | d«      s1t        t        | j                  «      «      D �cg c]  }d ‘Œ c}| _        t        | d«      sg | _        t        | d«      sd | _        t        | d«      r| j&                  €| j(                  | _        y y y c c}}w c c}w c c}	w c c}w )Nrq   rn   ro   z\tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated.z:Merges were not in checkpoint, building merges on the fly.ry   c                 ó$   •— ‰| d      ‰| d      fS )Nr   ry   © )ÚxÚvocabs    €r’   ú<lambda>z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>¬  s   ø€ °U¸1¸Q¹4±[À%ÈÈ!ÉÁ+Ð4N€ r“   T)ÚkeyÚreversec                 ó   — | d   S )Nr   r˜   )Úvals    r’   r›   z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>®  s
   € °C¸±F€ r“   r   ú Úadded_tokensr@   Úunknown_token_id)ÚitemsÚsetattrÚhasattrrˆ   rn   ro   Ú	enumerateÚloggerÚwarningr   Úranger‡   ÚappendÚsortedÚextendrq   ÚtupleÚsplitr¡   r@   r¢   )ÚselfÚdict_ÚkÚvrn   ro   ÚiÚtrq   ÚmergeÚpiece_scoreÚlocalÚindexÚpiece_lÚpiece_rrŸ   Ú_rš   s                    @r’   Ú__init__zGGUFTokenizerSkeleton.__init__—  s   ø€ Ø—K‘K“Mò 	 ‰DˆAˆqÜ�D˜!˜QÕð	 ô �t˜XÕ&Ü˜4 Ô*´'¸$ÀÔ2IÜ Øróð ð —[‘[ˆFØ—[‘[ˆFÜ.7¸Ó.?×@¡d a¨�Q˜˜q™	‘\Ó@ˆEä�N‰NÐWÔXØˆFÜ&*¨5¯;©;«=Ó&9ò %Ñ"��{Ø�Ü" 1¤c¨%£jÓ1ò F�EØ',¨V¨e }°e¸E¸F°m˜W�GØ &Ò(¨W¸Ò->ØŸ™ g¨w¸Ð%DÕEðFô ˜uÓ*NÐX\Ô]�Ø—‘˜eÕ$ð%ô ˜FÑ(:ÀDÔIˆFØ28Ö9¨3�s˜1‘v˜s 1™vÒ&Ð9ˆFÐ9Ø ˆD�Kà@DÇÁÖL°uœ5 §¡¨SÓ!1Õ2ÒLˆDŒKÜ˜4 Ô*Ü-2´3°t·{±{Ó3CÓ-DÖE¨štÒE�”ä�t˜^Ô,Ø "ˆDÔä�t˜^Ô,Ø $ˆDÔô �4Ð+Ô,°×1BÑ1BÐ1JØ $× 5Ñ 5ˆDÕð 2KÐ,ùó7 Aùò :ùò MùâEs   Á=H#ÅH)Å+!H.Ç 	H3N)Ú__name__Ú
__module__Ú__qualname__r¼   r˜   r“   r’   r•   r•   –  s   „ ó'6r“   r•   c                   ó0   — e Zd Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zy)ÚGGUFLlamaConverterc                 ó’   — t        |«      | _        | j                  | _        i | _        t	        | j                  dd«      dk7  | _        y )Nrm   r   )r•   ÚprotoÚoriginal_tokenizerÚadditional_kwargsÚgetattrÚis_llama_3_tokenizer©r¯   Útokenizer_dicts     r’   r¼   zGGUFLlamaConverter.__init__Â  s>   € Ü*¨>Ó:ˆŒ
Ø"&§*¡*ˆÔØ!#ˆÔÜ$+¨D¯J©JÐ8HÈ'Ó$RÐV]Ñ$]ˆÕ!r“   c                 óT   — t        t        |j                  |j                  «      «      S ©N©r†   Úziprn   ro   ©r¯   rÃ   s     r’   rš   zGGUFLlamaConverter.vocabÈ  ó   € Ü”C˜Ÿ™ e§l¡lÓ3Ó4Ð4r“   c                 ó   — |j                   S rË   ©rq   rÎ   s     r’   rq   zGGUFLlamaConverter.mergesË  ó   € Ø�|‰|Ðr“   c           
      óÄ  — | j                  | j                  «      }| j                  | j                  «      }t        |«      D ���ci c]  \  }\  }}||“Œ }}}}|j                  �|j
                  |j                     nd }t        |dd «      �|j
                  |j                     nd }	t        |dd «      �|j
                  |j                     nd }
t        t        |||dd¬«      «      }g }t        | j                  d«      s^|�|j                  t        |dd¬«      «       |	�|j                  t        |	dd¬«      «       |
�›|j                  t        |
dd¬«      «       n}t        j                  t        j                  | j                  j                   «      dk(  «      d	   }|D ]6  }|j                  t        | j                  j
                  |   dd¬«      «       Œ8 t#        |«      d	k7  r|j%                  |«       t#        | j                  j&                  «      d	k7  r?|j)                  | j                  j&                  D �cg c]  }t        |dd¬«      ‘Œ c}«       || j*                  d
<   |	| j*                  d<   |
| j*                  d<   | j,                  r>d | j*                  d<   d| j*                  d<   d| j*                  d<   d| j.                  _        |S c c}}}w c c}w )Nr>   r?   T)Ú	unk_tokenÚfuse_unkÚbyte_fallbackrp   F©Ú
normalizedÚspecialr{   r   rÔ   Ú	eos_tokenÚ	bos_tokenrr   Úclean_up_tokenization_spacesÚlegacy)rš   rÃ   rq   r¦   r@   rn   rÆ   r>   r   r	   r¥   rª   r   ÚnpÚwherer   rp   r‡   Úadd_special_tokensr¡   Ú
add_tokensrÅ   rÇ   rÄ   rÝ   )r¯   rÃ   Úvocab_scoresrq   r³   ÚwordÚ_scoreÚ	bpe_vocabrÔ   rÛ   rÚ   r=   Úspecial_tokensÚspecial_tokens_idxÚidxÚadded_tokens                   r’   r=   zGGUFLlamaConverter.tokenizerÎ  s¨  € Ø—z‘z $§*¡*Ó-ˆØ—‘˜TŸZ™ZÓ(ˆÜ6?ÀÓ6M×NÐNÑ!2 ¡N T¨6�T˜1‘WÐNˆ	ÒNà8=×8JÑ8JÐ8V�E—L‘L ×!3Ñ!3Ò4Ð\`ˆ	Ü8?ÀÀ~ÐW[Ó8\Ð8h�E—L‘L ×!3Ñ!3Ò4Ðnrˆ	Ü8?ÀÀ~ÐW[Ó8\Ð8h�E—L‘L ×!3Ñ!3Ò4Ðnrˆ	äÜØØØ#ØØ"ôó
ˆ	ð ˆä�t—z‘z <Ô0ØÐ$Ø×%Ñ%¤j°ÀuÐVZÔ&[Ô\àÐ$Ø×%Ñ%¤j°ÀuÐVZÔ&[Ô\àÐ$Ø×%Ñ%¤j°ÀuÐVZÔ&[Õ\ô "$§¡¬"¯(©(°4·:±:×3HÑ3HÓ*IÈQÑ*NÓ!OÐPQÑ!RÐà)ò j�Ø×%Ñ%¤j°·±×1BÑ1BÀ3Ñ1GÐTYÐcgÔ&hÕiðjô ˆ~Ó !Ò#Ø×(Ñ(¨Ô8äˆt�z‰z×&Ñ&Ó'¨1Ò,Ø× Ñ Ø]a×]gÑ]g×]tÑ]tÖuÈk”˜K°EÀ5ÖIÒuôð /8ˆ×Ñ˜{Ñ+Ø.7ˆ×Ñ˜{Ñ+Ø.7ˆ×Ñ˜{Ñ+à×$Ò$Ø9=ˆD×"Ñ"Ð#5Ñ6ØEIˆD×"Ñ"Ð#AÑBà/4ˆD×"Ñ" 8Ñ,Ø-2ˆD×#Ñ#Ô*àÐùôk OùòN vs   ÁKÉKc                 ó0  — t        j                  «       t        j                  «       t        j                  dd«      g}| j                  r|t        j
                  ddd¬«      gz  }|r|t        j                  dd¬«      gz  }t        j                  |«      S )Nõ   â–�r    FT©rr   Útrim_offsetsÚ	use_regexry   ©ÚcontentÚleft)r   ÚByteFallbackÚFuseÚReplacerÇ   Ú	ByteLevelÚStripÚSequence©r¯   Úreplacementrr   Úsequences       r’   ÚdecoderzGGUFLlamaConverter.decoder  sƒ   € ä×!Ñ!Ó#Ü�M‰M‹OÜ×Ñ˜U CÓ(ð
ˆð ×$Ò$Øœ×+Ñ+¸UÐQVÐbfÔgÐhÑhˆHáØœŸ™°¸!Ô<Ð=Ñ=ˆHÜ× Ñ  Ó*Ð*r“   c                 ó   — | j                  | j                  «      }| j                  | j                  «      }|�||_        d}d}t        | j                  d«      r| j                  j
                  }| j                  ||«      }|�||_        | j                  ||«      |_        | j                  «       }|r||_        | j                  r7t        j                  ddd¬«      |_        t        j                  g «      |_        |S )Nrë   Trr   Frì   )r=   rÃ   Ú
normalizerr¥   rÄ   rr   Úpre_tokenizerrû   Úpost_processorrÇ   r   rõ   r   r÷   )r¯   r=   rý   rù   rr   rþ   rÿ   s          r’   Ú	convertedzGGUFLlamaConverter.converted  sò   € à—N‘N 4§:¡:Ó.ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;Ð8HÓIˆØÐ$Ø&3ˆIÔ#à ŸL™L¨Ð6FÓGˆ	ÔØ×,Ñ,Ó.ˆÙØ'5ˆIÔ$ð ×$Ò$Ü&4×&>Ñ&>Ø!&°UÀdô'ˆIÔ#ô
 $/×#7Ñ#7¸Ó#;ˆIÔ àÐr“   N)	r½   r¾   r¿   r¼   rš   rq   r=   rû   r   r˜   r“   r’   rÁ   rÁ   Á  s"   „ ò^ò5òò8òt+ó!r“   rÁ   c                   ó*   ‡ — e Zd Zd„ Zdefˆ fd„Zˆ xZS )ÚGGUFQwen2Converterc                 ó2   — t        |«      | _        i | _        y rË   ©r•   rÄ   rÅ   rÈ   s     r’   r¼   zGGUFQwen2Converter.__init__;  ó   € Ü"7¸Ó"GˆÔØ!#ˆÕr“   Úreturnc           	      ó4  •— t        | j                  j                  «      D ��ci c]  \  }}||“Œ
 }}}| j                  j                  }t        ‰| �  ||«      }|j                  t        ddd¬«      t        ddd¬«      t        ddd¬«      g«       |S c c}}w )Nú<|endoftext|>FTr×   z<|im_start|>z
<|im_end|>)r¦   rÄ   rn   rq   Úsuperr   rà   r   ©r¯   r³   rã   rš   rq   r=   Ú	__class__s         €r’   r   zGGUFQwen2Converter.converted?  s–   ø€ Ü(1°$×2IÑ2I×2PÑ2PÓ(Q×R™W˜Q ��q‘ÐRˆÑRØ×(Ñ(×/Ñ/ˆÜ‘GÑ% e¨VÓ4ˆ	à×$Ñ$ä˜?°uÀdÔKÜ˜>°eÀTÔJÜ˜<°EÀ4ÔHðô	
ð Ðùó Ss   ¤B©r½   r¾   r¿   r¼   r   r   Ú__classcell__©r  s   @r’   r  r  :  s   ø„ ò$ð˜9÷ ñ r“   r  c                   ó6   — e Zd Zd„ Zd„ Zd„ Zd„ Zd„ Zdefd„Z	y)	ÚGGUFPhi3Converterc                 óT   — t        |«      | _        | j                  | _        i | _        y rË   ©r•   rÃ   rÄ   rÅ   rÈ   s     r’   r¼   zGGUFPhi3Converter.__init__O  s"   € Ü*¨>Ó:ˆŒ
Ø"&§*¡*ˆÔØ!#ˆÕr“   c                 óT   — t        t        |j                  |j                  «      «      S rË   rÌ   rÎ   s     r’   rš   zGGUFPhi3Converter.vocabT  rÏ   r“   c                 ó   — |j                   S rË   rÑ   rÎ   s     r’   rq   zGGUFPhi3Converter.mergesW  rÒ   r“   c                 ó  — | j                  | j                  «      }| j                  | j                  «      }t        |«      D ���ci c]  \  }\  }}||“Œ }}}}t	        t        ||«      «      }|j                  t        ddddd¬«      t        ddd¬«      t        dddd¬«      t        d	ddd¬«      t        d
ddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      g«       |j                  �|j                  |j                     nd | j                  d<   |j                  �|j                  |j                     nd | j                  d<   |j                  �|j                  |j                     nd | j                  d<   |j                  �|j                  |j                     nd | j                  d<   |S c c}}}w )Nú</s>TF)ÚrstripÚlstriprØ   rÙ   r  r×   z<|assistant|>)r  rØ   rÙ   z<|placeholder1|>z<|placeholder2|>z<|placeholder3|>z<|placeholder4|>z
<|system|>z<|end|>z<|placeholder5|>z<|placeholder6|>z<|user|>rÔ   rÚ   rÛ   Ú	pad_token)rš   rÃ   rq   r¦   r   r	   rà   r   r@   rn   rÅ   r?   r>   rA   )	r¯   rÃ   râ   rq   r³   rã   rä   rå   r=   s	            r’   r=   zGGUFPhi3Converter.tokenizerZ  s   € Ø—z‘z $§*¡*Ó-ˆØ—‘˜TŸZ™ZÓ(ˆÜ6?ÀÓ6M×NÐNÑ!2 ¡N T¨6�T˜1‘WÐNˆ	ÒNäœc )¨VÓ4Ó5ˆ	à×$Ñ$ä˜6¨$°uÈÐX\Ô]Ü˜?°uÀdÔKÜ˜?°4ÀEÐSWÔXÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[Ü˜<°ÀÐPTÔUÜ˜9¨T¸eÈTÔRÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[Ü˜:¨d¸uÈdÔSðô	
ð$ 16×0BÑ0BÐ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð 16×0BÑ0BÐ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð 16×0BÑ0BÐ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð 16×0BÑ0BÐ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð ÐùôE Os   ÁG:c                 óà   — t        j                  «       t        j                  «       t        j                  |d«      g}|r|t        j                  dd¬«      gz  }t        j
                  |«      S )Nr    ry   rï   )r   rò   ró   rô   rö   r÷   rø   s       r’   rû   zGGUFPhi3Converter.decoder�  s\   € ä×!Ñ!Ó#Ü�M‰M‹OÜ×Ñ˜[¨#Ó.ð
ˆñ ØœŸ™°¸!Ô<Ð=Ñ=ˆHÜ× Ñ  Ó*Ð*r“   r  c                 óÊ   — | j                  | j                  «      }d}d}t        | j                  d«      r| j                  j                  }| j                  ||«      |_        |S )Nrë   Trr   )r=   rÃ   r¥   rÄ   rr   rû   )r¯   r=   rù   rr   s       r’   r   zGGUFPhi3Converter.convertedŒ  s\   € Ø—N‘N 4§:¡:Ó.ˆ	àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà ŸL™L¨Ð6FÓGˆ	ÔàÐr“   N)
r½   r¾   r¿   r¼   rš   rq   r=   rû   r   r   r˜   r“   r’   r  r  N  s(   „ ò$ò
5òò%òN	+ð
˜9ô 
r“   r  c                   ó*   ‡ — e Zd Zd„ Zdefˆ fd„Zˆ xZS )ÚGGUFGPTConverterc                 ó2   — t        |«      | _        i | _        y rË   r  rÈ   s     r’   r¼   zGGUFGPTConverter.__init__š  r  r“   r  c                 óÄ   •— t        | j                  j                  «      D ��ci c]  \  }}||“Œ
 }}}| j                  j                  }t        ‰| �  ||«      }|S c c}}w rË   )r¦   rÄ   rn   rq   r	  r   r
  s         €r’   r   zGGUFGPTConverter.convertedž  s]   ø€ Ü(1°$×2IÑ2I×2PÑ2PÓ(Q×R™W˜Q ��q‘ÐRˆÑRØ×(Ñ(×/Ñ/ˆÜ‘GÑ% e¨VÓ4ˆ	ØÐùó Ss   ¤Ar  r  s   @r’   r  r  ™  s   ø„ ò$ð˜9÷ ñ r“   r  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zd„ Zdefd„Zy)ÚGGUFT5Converterc                 óÜ   — dg|d<   t        |«      | _        t        | j                  j                  «      D ��ci c]  \  }}||“Œ
 c}}| _        | j                  | _        i | _        y c c}}w ©Nz
dummy textrq   )r•   rÃ   r¦   rn   Útoken2idrÄ   rÅ   )r¯   rÉ   r²   r±   s       r’   r¼   zGGUFT5Converter.__init__¦  s]   € à$0 >ˆ�xÑ ä*¨>Ó:ˆŒ
Ü*3°D·J±J×4EÑ4EÓ*F×G¡$ ! Q˜˜A™ÓGˆŒØ"&§*¡*ˆÔØ!#ˆÕùó Hs   ¹A(c                 óT   — t        t        |j                  |j                  «      «      S rË   rÌ   rÎ   s     r’   rš   zGGUFT5Converter.vocab¯  rÏ   r“   c                 óø   — t        | j                  dd«      rcg }t        | j                  dd«      r|t        j                  d¬«      gz  }|t        j                  dd¬«      gz  }t        j
                  |«      S y )NrÝ   Trr   rë   )Úprependr    )Úpatternrð   )rÆ   rÄ   r   ÚPrependrô   r÷   )r¯   rÃ   rú   s      r’   rý   zGGUFT5Converter.normalizer²  sr   € Ü�4×*Ñ*¨H°dÔ;ØˆHÜ�t×.Ñ.Ð0BÀDÔIØœ[×0Ñ0¸Ô?Ð@Ñ@�Øœ×,Ñ,°SÀ%ÔHÐIÑIˆHÜ×'Ñ'¨Ó1Ð1Ør“   c                 óZ   — t        j                  ddgg d¢d| j                  d   fg¬«      S )Nú$Ar  )r+  r  z$Br  )ÚsingleÚpairræ   )r   ÚTemplateProcessingr$  )r¯   s    r’   rÿ   zGGUFT5Converter.post_processor»  s5   € Ü×,Ñ,Ø˜&�>Ú-à˜Ÿ™ vÑ.Ð/ðô
ð 	
r“   r  c                 óÐ  — | j                  | j                  «      }t        t        || j                  j                  d¬«      «      }| j                  | j                  «      }|�||_        d}d}t        | j                  d«      r| j                  j                  }| j                  ||«      }|�||_	        | j                  ||«      |_
        | j                  «       }|r||_        |S )NF©Úunk_idrÖ   rë   Trr   )rš   rÃ   r   r
   r@   rý   r¥   rÄ   rr   rþ   rû   rÿ   )r¯   râ   r=   rý   rù   rr   rþ   rÿ   s           r’   r   zGGUFT5Converter.convertedÄ  sÝ   € Ø—z‘z $§*¡*Ó-ˆÜÜØØ—z‘z×.Ñ.Ø#ôó
ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;Ð8HÓIˆØÐ$Ø&3ˆIÔ#à ŸL™L¨Ð6FÓGˆ	ÔØ×,Ñ,Ó.ˆÙØ'5ˆIÔ$àÐr“   N)	r½   r¾   r¿   r¼   rš   rý   rÿ   r   r   r˜   r“   r’   r!  r!  ¥  s"   „ ò$ò5òò
ð˜9ô r“   r!  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zd„ Zdefd„Zy)ÚGGUFGemmaConverterc                 ó`   — dg|d<   t        |«      | _        | j                  | _        i | _        y r#  r  rÈ   s     r’   r¼   zGGUFGemmaConverter.__init__å  s.   € à$0 >ˆ�xÑ ä*¨>Ó:ˆŒ
Ø"&§*¡*ˆÔØ!#ˆÕr“   c                 óL  — t        t        |j                  |j                  «      «      }g }|D ]s  \  }}|dk(  r|j	                  d|f«       Œd|v r>t        |j                  «       «      dk(  r"dt        |«      z  }|j	                  ||f«       Œa|j	                  ||f«       Œu |S )Nz<0x09>ú	r    r   rë   )r†   rÍ   rn   ro   rª   r‡   Ústrip)r¯   rÃ   Úoriginal_vocabÚupdated_vocabÚtokenÚscoreÚunderscoress          r’   rš   zGGUFGemmaConverter.vocabí  s¡   € Üœc %§,¡,°·±Ó=Ó>ˆØˆà*ò 	5‰LˆE�5Ø˜Ò Ø×$Ñ$ d¨E ]Õ3Ø˜‘¤# e§k¡k£mÓ"4¸Ò"9Ø#¤c¨%£jÑ0�Ø×$Ñ$ k°5Ð%9Õ:à×$Ñ$ e¨U ^Õ4ð	5ð Ðr“   c                 ó.   — t        j                  dd«      S )Nr    rë   )r   rô   rÎ   s     r’   rý   zGGUFGemmaConverter.normalizerü  s   € Ü×"Ñ" 3¨Ó.Ð.r“   c                 óà   — t        j                  dd«      t        j                  «       t        j                  «       g}|r|t        j                  dd¬«      gz  }t        j
                  |«      S )Nrë   r    ry   rï   )r   rô   rò   ró   rö   r÷   rø   s       r’   rû   zGGUFGemmaConverter.decoderÿ  s\   € ä×Ñ˜U CÓ(Ü×!Ñ!Ó#Ü�M‰M‹Oð
ˆñ ØœŸ™°¸!Ô<Ð=Ñ=ˆHÜ× Ñ  Ó*Ð*r“   r  c                 ó²  — | j                  | j                  «      }t        t        || j                  j                  | j
                  ¬«      «      }| j                  | j                  «      }|�||_        d}d}t        | j                  d«      r| j                  j                  }| j                  ||«      |_
        | j                  ||«      }|�||_        |S )Nr0  rë   Trr   )rš   rÃ   r   r
   r@   Úhandle_byte_fallbackrý   r¥   rÄ   rr   rû   rþ   )r¯   râ   r=   rý   rù   rr   rþ   s          r’   r   zGGUFGemmaConverter.converted
  sÊ   € Ø—z‘z $§*¡*Ó-ˆÜÜØØ—z‘z×.Ñ.Ø"×7Ñ7ôó
ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà ŸL™L¨Ð6FÓGˆ	ÔØ×*Ñ*¨;Ð8HÓIˆØÐ$Ø&3ˆIÔ#àÐr“   N)	r½   r¾   r¿   r¼   rš   rý   rû   r   r   r˜   r“   r’   r3  r3  ä  s"   „ ò$òò/ò	+ð˜9ô r“   r3  re   rf   rg   Úgemma3_textÚgemma4_text)rj   rk   Údecilmrl   r   r  c                 ó^   — | }t        |   |«      }|j                  «       }||j                  fS )a6  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        architecture (`str`): The model architecture derived from gguf file.
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    )ÚGGUF_TO_FAST_CONVERTERSr   rÅ   )r   rÉ   Útokenizer_class_nameÚ	converterÚfast_tokenizers        r’   Úconvert_gguf_tokenizerrI  >  s9   € ð (ÐÜ'Ð(<Ñ=¸nÓM€IØ×(Ñ(Ó*€NØ˜9×6Ñ6Ð6Ð6r“   ),Ú__doc__r   ÚnumpyrÞ   Ú
tokenizersr   r   r   r   r   Útokenizers.modelsr	   r
   Ú r   Úconvert_slow_tokenizerr   r   r   r   r   Úutilsr   Úutils.loggingr   Ú
get_loggerr½   r§   ÚGGUF_CONFIG_MAPPINGÚGGUF_TOKENIZER_MAPPINGÚGGUF_CONFIG_DEFAULTS_MAPPINGrŽ   r•   rÁ   r  r  r  r!  r3  rE  Ústrr­   ÚdictrI  r˜   r“   r’   ú<module>rX     sü  ðñõ
 ã ß SÕ Sß *å ß oÕ oÝ Ý  ð 
ˆ×	Ñ	˜HÓ	%€ðqØØ$Ø%ñðqð
 Ø3Ø*Ø2Ø)à *Ø&Ø 5Ø#8Ø,:Ø"ñðqð$ Ø3Ø*Ø2Ø)à *Ø&Ø 5Ø#8Ø,:Ø"ñð%qð> Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñð?qðV Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"Ø%Ø2ñðWqðr Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"Ø+Ø2Ø*ñðsqðP Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"Ø+ñðQqðj Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñðkqðB Ø3Ø*Ø2Ø)Ø $Ø&Ø *Ø 5Ø#8Ø,:Ø"Ø%Ø2ñðCqð` Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñðaqðx Ø+Ø+Ø!/Ø!/ñ	ðyqðD Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñðEqð\ Ø Ø)Ø (Ø"Ø(<ñð]qðj 	Ø'Ø#Ø%Ø%Ø &Ø +Ø#8Ø(<Ø,LØ":Ø"ñðkqðD Ø3Ø*Ø2Ø)Ø $Ø 5Ø#8Ø(8Ø"ñ
ðEqðZ Ø Ø!Ø$Ø4Ø (Ø(<ñð[qðj Ø*Ø3Ø)Ø2Ø 5Ø#8Ø(6ñðkqð~ #Ø3Ø)Ø,@Ø*Ø(Ø&Ø.Ø-ñ
ð 4Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,6Ø"ñð 4Ø*Ø2Ø)Ø $Ø&ð !+Ø 5Ø#8Ø,:Ø$4Ø"ñð" 4Ø*Ø2Ø)Ø $Ø&ð !+Ø 5Ø#8Ø,:Ø$4Ø"ñð" 4Ø*Ø2Ø)Ø $Øð !2Ø 5Ø#8Ø,:Ø$4Ø"ñð$ (Ø#Ø%Ø%Ø &Ø +Ø#8Ø(<Ø,LØ":Ø"ñð 4Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñðØÐ3ðàÐ*ðð 	Ð2ðð 	˜Mð	ð
 	 ðð 	˜,ðð 	Ð 5ðð 	"Ð#8ðð 	 
ðð 	! $ðð 	+¨Nðð 	Ð+ðð 	Ð2ðð 	% dðð 	�lðð  	˜nð!òqÐ ðj	 'ØØØ'ØØ+Ø+Ø!/Ø!/Ø!3ñð )Ø"Ø+Ø+Ø!/Ø!/ññÐ ð: 	˜$ð	ð 	˜Dð	ñ Ð ò ÷0(6ñ (6ôVv˜ô vôr˜ô ô(H˜ô HôV	�}ô 	ô<�kô <ô~>˜ô >ðBØÐðàÐðð Ð#ðð Ðð	ð
 Ð#ðð Ððð Ððð Ððð Ð ðð Ððð Ð"ðð 	ˆ/ðð Ððð Ð ðð Ð ðð  Ð%ð!ð" Ð%ð#ð$ ØØ Ø$ò+Ð ð27¨ð 7ÀÀyÐRVÀÑAWô 7r“   