^jnddlZddlmZddlZddlmZddlmZm Z m Z m Z ddl m Z ddlmZddlmZe rddlZeeZd d d d d ddde de dide didZeedj/ZGddeZGddZGddeZGddeZGddeZGdd eZGd!d"eZGd#d$eZ Gd%d&eZ!Gd'd(eZ"Gd)d*eZ#Gd+d,eZ$Gd-d.eZ%eeeeeeee e!e"e#e#e$e%d/Z&d0Z' d7d1ed2e(dzd3e)dzd4e(fd5Z*d8d6Z+y)9N) NamedTuple)tqdm)GGUF_CONFIG_DEFAULTS_MAPPINGGGUF_CONFIG_MAPPINGGGUF_TOKENIZER_MAPPING_gguf_parse_value)is_torch_available)is_gguf_available) get_loggerversion tensor_countkv_count)r rr file_typequantization_version)rr)GGUFgeneral tokenizertokenizer_config)ignoreconfigrrrc@eZdZUejed<eed<eed<y) GGUFTensorweightsnamemetadataN)__name__ __module__ __qualname__npndarray__annotations__strdicts/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/modeling_gguf_pytorch_utils.pyrr8s ZZ INr&rcLeZdZd dZdedefdZdeeefdededefd Zd Zy) TensorProcessorNc|xsi|_yNr)selfrs r'__init__zTensorProcessor.__init__?s l r&hf_namereturnc|S)zP Preprocesses the tensor name to ease loading the GGUF tensors. r%r-r/s r'preprocess_namezTensorProcessor.preprocess_nameBs r&gguf_to_hf_name_mapsuffix qual_namecy)z Called when get_gguf_hf_weights_map fails to map a HF parameter (tensor) and corresponding GGUF one. This is particularly useful to resolve one-to-many HF-GGUF mappings sometimes appear in some MoE models. Nr%)r-r4r5r6r/s r'perform_fallback_tensor_mappingz/TensorProcessor.perform_fallback_tensor_mappingHs r&c t||iSr+rr-rrkwargss r'processzTensorProcessor.processTs'4,,r&r+) rrrr.r#r3r$r8r=r%r&r'r)r)>sL#ss  #'S>  ;>  KN  Y\  -r&r)c peZdZd fd ZdZ d dej dededzdej fdZxZ S) LlamaTensorProcessorNc&t||yNr,superr.r-r __class__s r'r.zLlamaTensorProcessor.__init__Y 'r&c d|vsd|vrx|jjd}|jjd}d||fvr t||iSd|vr|j|||}nd|vr|j|||}t||iS)Nz.attn_k.z.attn_q.num_attention_headsnum_key_value_heads)rgetr_reverse_permute_weights)r-rrr< num_heads num_kv_headss r'r=zLlamaTensorProcessor.process\s  t!3 (=>I;;??+@AL <00!'444T!77IVt#77LY'4,,r&rn_headrMr0c|||k7r|}|jd|zdz}|j||dg|jdd}|jddj|jS)Nrr)shapereshapeswapaxes)r-rrNrMdimws r'rKz-LlamaTensorProcessor._reverse_permute_weightsisr  #,(>!FmmA&(A- GOOFC ?W]]12-> ?zz!Q'' 66r&r+) rrrr.r=r r!intrK __classcell__rEs@r'r?r?XsE( -LP 7zz 7+. 7>ADj 7  7r&r?ceZdZejdZejdZejdZdfd Zde de fdZ de e e fd e d e de fd Z d e fd Z dejde e e fde de fdZxZS)Qwen2MoeTensorProcessorzmlp.experts.\d+.z7model\.layers\.(?P\d+)\.mlp\.experts\.gate_up_proj3(?P.*\.ffn_(?Pgate|down|up)_exps)\.weight$c&t||yrArBrDs r'r.z Qwen2MoeTensorProcessor.__init__{rFr&r/r0cDtj|jd|SNz mlp.experts.resubHF_EXPERT_RENAME_PATTERNr2s r'r3z'Qwen2MoeTensorProcessor.preprocess_name~vvd33^WMMr&r4r5r6ctj|j|x}r"||z}||d|dd|<||d|dd|<yy)Nblk.bid.ffn_gate_exps .ffn_up_exps)r` fullmatchHF_MOE_W13_PATTERNr-r4r5r6r/m full_hf_names r'r8z7Qwen2MoeTensorProcessor.perform_fallback_tensor_mappingsg T44g> >1 >$w.LKW $qxjvh G HIU $qxj VH E F ?r&rc 2tj|j|x}rN|jd}|jd}|r*|j ||||d|dt |diSd|vrt j|d}t ||iS)Ntensor_key_mappingparsed_parametersrrUffn_gate_inp_shexpraxis)r`riGGUF_MOE_WEIGHTS_PATTERNrJ_set_moe_expert_tensorrr expand_dimsr-rrr<rlrorps r'r=zQwen2MoeTensorProcessor.processs T::DA A1 A!',@!A  & +> ? !++G5FHZ[\]c[dHeghilgmn!'444 4 'nnW15G'4,,r&rrprUctjtj|}|dk(r ||d|<yt |j }d}||}|dz||<||dvr'tj ||j|d|<|d|} |dk(r| j|d|} n| j|||} | j|yNdowntensorsrrP)dtypegater torch from_numpyr copylistrQzerosr|narrowcopy_ r-rrpr/rU torch_weightsrQ shard_dim shard_sizeouts r'ruz.Qwen2MoeTensorProcessor._set_moe_expert_tensors(()9: ;4A i ( 1 'EIy)J)A~E) / ::8= EQ^QdQd8e!),W5 1) #$^_)rzz*`a(NsNsNV#'S>V;>VKNVY\V -S -%bjj%TRUW[R[_%gj%or%r&rZc eZdZdZej dZej dZdfd Zde fdZ de jde d e d e d e f d Zde jde d e d e fd ZxZS)GptOssTensorProcessoraK Tensor processor for GPT-OSS models (MoE with 128 experts). Handles: - Splitting stacked expert tensors (down_proj, gate_proj, up_proj) into individual experts. - Interleaving gate and up projections if stored in a combined tensor (gate_up_projs). - Bias tensors (1D) are passed through without transpose. z\d+)\.ffn_(?Pdown|gate|up)_projs\.weight$z-blk\.(?P\d+)\.ffn_gate_up_projs\.weight$c&t||yrArBrDs r'r.zGptOssTensorProcessor.__init__rFr&rc |jj|x}rN|jd}|jd}|r*|r(|j|||d|d|t |diS|j j|x}rJ|jd}|jd}|r&|r$|j |||d|t |diSd|vr%t|jdk(r t ||iSt ||iS)Nrorprfproj.biasr) rtmatchrJ_split_moe_expert_tensorrGGUF_MOE_COMBINED_PATTERN_interleave_gate_up_tensorlenrQrws r'r=zGptOssTensorProcessor.processs--33D9 91 9!',@!A  & +> ? !&7--g7H!E(TUV\T]_qr!'444..44T: :1 :!',@!A  & +> ? !&7//9JAeHVhi!'444 d?s7==1Q6gtR0 0'4,,r&rrprfrrocD|jjdd}tt||jdD]_}||}d|d|d|d} |j D]\} } | | vs | j | | } tj|d |d | <ay ) z:Split a stacked MoE tensor into individual expert tensors.num_local_expertsr model.layers..block_sparse_moe.experts..z _proj.weightTrr{N) rrJrangeminrQitemsreplacertensor) r-rrprfrro num_expertsi expert_weightr/key mapped_keys r'rz.GptOssTensorProcessor._split_moe_expert_tensorskkoo&93? s; a(89: [A#AJM%cU*DQCql[G#5#;#;#= ?Z'>%ooc:>G ?5:LLUY4Z i ( 1 [r&cH|jjdd}|jd}|dz}|ddd|ddf}|dd|dddf} tt ||jdD]} || j } | | j } d|d| d } d|d| d }|j D]2\}}|| vr| j||} ||vs!|j||}4tj| d |d | <tj| d |d |<y)a Process a combined gate+up tensor. Expected shape: [num_experts, intermediate_size, hidden_size]. Interleaving: gate occupies first half of intermediate dimension, up occupies second half. Transpose to [hidden, half_inter] per expert. rrrrPNrrrz.gate_proj.weightz.up_proj.weightTrr{) rrJrQrrTrrrr)r-rrprfror inter_size half_inter gate_partup_partr gate_weight up_weight gate_nameup_namerrs r'rz0GptOssTensorProcessor._interleave_gate_up_tensorsLkkoo&93? ]]1% 1_ A{ {A-. !Z[!+,s; a(89: WA#A,..K I'u,FqcIZ[I%cU*DQCWG$6#;#;#= ?Z)# ) 1 1#z BI'>%ooc:>G  ? 7sHZZ_ E}}%j '4%66r&r+rrrr.r=rWrXs@r'rr:s (7r&rc&eZdZdfd ZdZxZS)GPT2TensorProcessorc&t||yrArBrDs r'r.zGPT2TensorProcessor.__init__HrFr&c d|vs d|vsd|vsd|vr |j}|dk(rDd}|jdi}tjt j ||d|<d}t ||iS) Nzattn_qkv.weightzffn_down.weightz ffn_up.weightzattn_output.weight output.weightzlm_head.weightrpr{)rrJrrr rr)r-rrr<rps r'r=zGPT2TensorProcessor.processKs  % D($&#t+iiG ? "$D & +> C 161A1A"'''BR1S i ( .D'4,,r&r+rrXs@r'rrGs (-r&rc&eZdZdfd ZdZxZS)MambaTensorProcessorc&t||yrArBrDs r'r.zMambaTensorProcessor.__init__brFr&c d|vrtj|d}d|vrtj| }t||iS)Nzssm_conv1d.weightrrrssm_a)r rvlogrr;s r'r=zMambaTensorProcessor.processesD $ &nnW15G d?ffgX&G'4,,r&r+rrXs@r'rras ( -r&rc&eZdZdfd ZdZxZS)NemotronTensorProcessorc&t||yrArBrDs r'r.z NemotronTensorProcessor.__init__rrFr&c .d|vr|dz }t||iSNz norm.weightrr:r;s r'r=zNemotronTensorProcessor.processv" D kG'4,,r&r+rrXs@r'rrqs (-r&rc&eZdZdfd ZdZxZS)Gemma2TensorProcessorc&t||yrArBrDs r'r.zGemma2TensorProcessor.__init__}rFr&c .d|vr|dz }t||iSrr:r;s r'r=zGemma2TensorProcessor.processrr&r+rrXs@r'rr|s ( -r&rc&eZdZdfd ZdZxZS)Lfm2TensorProcessorc&t||yrArBrDs r'r.zLfm2TensorProcessor.__init__rFr&c Rd|vrtj|d}t||iS)Nzshortconv.conv.weightrrr)r rvrr;s r'r=zLfm2TensorProcessor.processs) "d *nnW15G'4,,r&r+rrXs@r'rrs (-r&rc$eZdZejdZejdZejdZejdZdfd Z de de fdZ d e e e fd e d e de fd Z d e fdZdej de e e fde de fdZxZS)MiniMaxM2TensorProcessorzmlp\.experts\.\d+\.z<(?:model\.)?layers\.(?P\d+)\.mlp\.experts\.gate_up_projr[z>(?:model\.)?layers\.(?P\d+)\.mlp\.e_score_correction_biasc&t||yrArBrDs r'r.z!MiniMaxM2TensorProcessor.__init__rFr&r/r0cDtj|jd|Sr^r_r2s r'r3z(MiniMaxM2TensorProcessor.preprocess_namercr&r4r5r6ctj|j|x}r"||z}||d|dd|<||d|dd|<ytj|j|x}r||z|d|dd<yy)Nrerfrgrhz.exp_probs_b.bias)r`rirjHF_BIAS_PATTERNrks r'r8z8MiniMaxM2TensorProcessor.perform_fallback_tensor_mappings T44g> >1 >$w.LKW $qxjvh G HIU $qxj VH E F,,t33W= =Q =FORYFY $qxj0A B C>r&rc tj|j|x}rN|jd}|jd}|r|j ||||d|dt |diSt ||iS)NrorprrU)r`rirtrJrurrws r'r=z MiniMaxM2TensorProcessor.processs T::DA A1 A!',@!A  & +> ? !++G5FHZ[\]c[dHeghilgmngtR0 0'4,,r&rrprUctjtj|}|dk(r ||d|<yt |j }d}||}|dz||<||dvr'tj ||j|d|<|d|} |dk(r| j|d|} n| j|||} | j|yryr~rs r'ruz/MiniMaxM2TensorProcessor._set_moe_expert_tensors(()9: ;4A i ( 1'EIy)J)A~E) / ::8= EQ^QdQd8e!),W5 1)  Z;> ZKN ZY\ Z-S-%bjj%TRUW[R[_%gj%or%r&r)llamaqwen2moegpt_ossqwen3moebloomt5 t5encodergpt2mambanemotrongemma2gemma3lfm2 minimax-m2c||jvrgS|j|}|jDcgc]%}t|j||j'c}Scc}wr+)fieldsdatar partstypes)readerfieldvalue _data_indexs r' read_fieldr sP FMM! MM% EX]XbXb c ekk+6 D cc cs*A processor model_type num_layersr6c trtr ddlm}m}n t j dtd||jjn|}||jjn|}|dk(rd}n7|dk(rd }n/|d k(rd }n'|d k(rd }n|dk(rd}n|dk(rd}n|dk(rd}n|dk(rd}d}|jD]\}} | |k(s |}n|td|d|||} i} |j} | D]} |j| } | d}}| jds| jdr| j!dd\}}d|z}| j#|}||j%| ||| ||| z| ||z<|j'x}rX|D]S\}}t)||||||d}|jDcic] \}}|| vs ||}}}| j+|U| Scc}}w)aY GGUF uses this naming convention for their tensors from HF checkpoint: `blk.N.BB.weight` and `blk.N.BB.bias` where N signifies the block number of a layer, and BB signifies the attention/mlp layer components. See "Standardized tensor names" in https://github.com/ggerganov/ggml/blob/master/docs/gguf.md for details. r)MODEL_ARCH_NAMESget_tensor_name_mapLoading a GGUF checkpoint in PyTorch, requires both PyTorch and GGUF>=0.10.0 to be installed. Please see https://pytorch.org/ and https://github.com/ggerganov/llama.cpp/tree/master/gguf-py for installation instructions.KPlease install torch and gguf>=0.10.0 to load a GGUF checkpoint in PyTorch.Ncoherez command-r qwen2_moer qwen3_moer gemma3_textr gemma4_textgemma4umt5r minimax_m2rrgpt-osszUnknown gguf model_type: z in gguf-py. This might because you're using an outdated version of gguf-py package, you can install `gguf` package from source refer to https://github.com/ggerganov/llama.cpp/tree/master/gguf-py#developmentz.weightrrr)r6)r r ggufrrloggererror ImportErrorrr num_hidden_layersrNotImplementedError state_dictr3endswithrsplitget_namer8named_childrenget_gguf_hf_weights_mapupdate)hf_modelr r rr6rrarchrr name_mapr4r$r/rr5 gguf_namer(childsub_maprrs r'r)r)s13>>  A ghh/9/A++zJ6@6H22jJX { " { " } $ } $ v  | #! y  D&,,. U J D  |!' |4U U  #44H$$&J F++G4f   I &'*:*:7*C">>#q1LD&6\F%%d+    5 56I6S\^e f 2;g2EI./ F""0022~2) 0KD%-y*jykRVQWWXDYG)0 X11DW;Wq!tXGX  & &w /  0 Ys G7G7c /0trtr ddlm}m}n t j dtd||}|j}t|j}tD cic]} | i} } t|dd} t|d} d} d| vrd | vrd } nId | vsd | vr?d | d d<| r%d| djvrd} d | vrdg| d d<nd | vr dg| d d<d } n| } d| vrd} nd| vsd| vrd} n d| vrd} nd| vrd} d| vrShd/d0t/fd|jD}t0fd|jD}|| d d <| | d d!<| t vr| t vrt#d"| d#d$d%g}t%d&|jDxs| |v| d d'<t'j(| t'j(| xsi}|j+D]\}}| d j-|||jj+D]-\}}|j/| | }|j1d(}|d}d(j3|d)d}|j4Dcgc]%}t7|j8||j:'}}t=|d)k(r|d}t?|t@r| |vr|j/| | }tj+D]@\}}||vs |||vs|||}|d*k(r!||| ||<||vs0|jC|B||vst jEd+|d,|0| d d-d.k(rd/| d d-<| d d-d0k(rd1| d d-<gd2| d d3<| d j)d-dk(rBd4d5d6d7}| d j)d8}t?|tFr|j)|d5| d d8<| d d-d9k(rK| d d:}tI|| d d:<d;| d d<<tK|D !cgc] \} }!|!dkDs | c}!} | d d=<| dk(rdRd>}"|"|d?}#|#d@|#i}$|jD]}|jMdAs|t=dAd}%|%dBk(r)|j|j8d}t?|tNr|jQdC}|%dDvr tS|}n|%dEvrdF}%tG|}n ||$|%<|$| d dG<dH| d vr3| dI}&dJ|&vrt=|&dJ| d dH<nt jUdK|ri| dL<| j)d i}'tVj)| tX}(|(|'M})t[||)}*t]|jdNOD]}+|+j^},||+j4|+j`}-|)jc|-|,|*| P}.|.jd}-|.j^},|,|*vr^|*|,},tgjhtkjl|-}+||+jo|}+|+| dL|,<t=|dkDrt jEdQ|| Scc} wcc}wcc}!} w)SaF Load a GGUF file and return a dictionary of parsed parameters containing tensors, the parsed tokenizer and config attributes. Args: gguf_checkpoint_path (`str`): The path the to GGUF file to load return_tensors (`bool`, defaults to `False`): Whether to read the tensors from the file and return them. Not doing so is faster and only loads the metadata in memory. model_to_load (`nn.Module`, *optional*): The model to load the weights into. This is used to map GGUF tensor names to Transformers parameter names. torch_dtype (`torch.dtype`, *optional*): The desired `torch.dtype` for the loaded tensors. If provided, tensors will be converted to this dtype immediately after dequantization to save memory. r) GGUFReader dequantizerrzgeneral.architecturez general.nameNrmistralrrTr is_gated_actrUMT5EncoderModel architecturesT5EncoderModelrrrrrrrrstablelm> attn_k.bias attn_q.bias attn_v.biasffn_normc3HK|]}D]}||jvywr+r).0r bias_nameattn_bias_names r' z'load_gguf_checkpoint..s)mF^lmQZyFKK/m/ms"c3:K|]}|jvywr+r?)r@r ffn_norm_names r'rCz'load_gguf_checkpoint..s#^VMV[[$@#^s use_qkv_biasuse_parallel_residualzGGUF model with architecture z is not supported yet.falconrc3:K|]}|jdk7yw)rNr?)r@rs r'rCz'load_gguf_checkpoint..s HvFKK? * Hstie_word_embeddingsrrz1Some keys were not parsed and added into account z | r rrrr)rj2 eos_token_idnonesoftmaxsigmoid)rrrP scoring_funcrrIFblock_auto_adjust_ff_dimfull_attn_idxscd|}||jvr?|j|jd}t|tr|j d}|S|S)Nzgpt-oss.rutf-8)rr isinstancebytesdecode)rr5defaultrvals r' read_gpt_keyz*load_gguf_checkpoint..read_gpt_keysUVH%Cfmm#mmC(..q1c5)**W-C Nr&zrope.scaling.type rope_typezgpt-oss.rope.scaling.typerV)factorattention_factor beta_fast beta_slow)original_context_length original_max_position_embeddingsrd rope_scaling vocab_sizertokenszCan't find a way to retrieve missing config vocab_size from tokenizer parameters. This will use default value from model config class and cause unexpected behavior.r{r,z,Converting and de-quantizing GGUF tensors...)desc)rrrorpz0Some keys of the GGUF file were not considered: r+)8r r rr2r3rr r!rrkeysGGUF_TO_TRANSFORMERS_MAPPINGr loweranyr{GGUF_SUPPORTED_ARCHITECTURES ValueErrorallrrJr setdefaultrrjoinrr rrrrWr#removeinforVmax enumerate startswithrXrYfloatwarningTENSOR_PROCESSORSr)r)rr tensor_typer=rrrr rto)1gguf_checkpoint_pathreturn_tensors model_to_load torch_dtyper2r3rr reader_keysrrp architecture model_nameupdated_architecturerrG exceptionsconfig_defaultsrr gguf_keyrrprefix config_keyr  parameterparameter_renamesrenamed_config_key_gating_func_map_scoringgguf_num_key_value_headsrrMr\r]rer5tokenizer_parametersrProcessorClassr rorrrresultrBrEs1 @@r'load_gguf_checkpointr<s$13//  A ghh , -F ]]Fv{{}%K(DE1BEEf&<=a@LFN3J,9 #:(   !<6:(#N3 &JqM$7$7$99#) l*@R?S!(+O<l*@P?Q!(+O<#' +\!* l "i<&?( | #*  %+ \!F" mfnnmm ##^v~~#^ ^6>(#N3CX?X(#$;<77>|LRPRO&++-; U(#..sE:;"==..0b%##L2FGs#qXXeABi( ]b]g]ghk"5;;{#;U[[Ihh u:?!HE eS !le&;MM,0DEE,H,N,N,P 1 (I(**z=Nv=V/V%6v%>z%J"%+%1GL%i01CD{*&&x0 1 { " KKKH:UXY^X_` a7b<"<0H<4A(#L1"<0H<4A(#L17C(#N3"&&|4 D%) B$X.22>B h $:J:N:NxYb:c h ' 7"<0F:#4X#>?T#U =@AY=Z(#$9:BG(#$>? &//G%H9 !!\L[\L\A9 (#$45y( !)<=  '3L}} -~~&=>S!89;<V# c*003eU+!LL1EUU!%LE^^?FJE', V$% -(;G h ' 7,X660= + +8;>0^_ 8F;;D f.@.@AG&&#5"3 'FnnG;;D--%d+D%%bggg&67F&;/17 i ( .- 80 ;! F{mTU kFLif9 s5 Y8*Y" Y')Y')NNr)FNN),r`typingrnumpyr tqdm.autor integrationsrrrr utilsr utils.import_utilsr utils.loggingr rrrrjrrirmrr)r?rZrrrrrrrrrryr r#rVr)rr%r&r'rs   &1% H  !*" "-F\] "5kBC$&<=O&PQ  $$@$J$O$O$QR --47?7<5%o5%pbWObWJ$?$N 7 7-/-4 -? - -o- -O --/-2%2%l"'$' ! "  !'## *$d"! WWd Wd W  Wttr&