^j2ddlZddlmcmZddlmZddlmZddlm Z m Z ddl m Z ddl mZmZddlmZdd lmZdd lmZmZdd lmZmZdd lmZd dlmZmZd dlm Z m!Z!d dl"m#Z#m$Z$m%Z%m&Z&m'Z'ddl(m)Z)GddeZ*GddeZ+GddejXZ-Gdde$Z.GddejXZ/Gdde Z0Gdd e#Z1eGd!d"e!eZ2eGd#d$e&Z3Gd%d&e%Z4gd'Z5y)(N)nn)initialization)Cache DynamicCache)create_causal_mask)MoeCausalLMOutputWithPastMoeModelOutputWithPast)PreTrainedModel)Unpack)TransformersKwargsauto_docstring)can_return_tuplemerge_with_config_defaults)capture_outputs)GraniteRMSNormGraniteRotaryEmbedding)LlamaAttentionLlamaPreTrainedModel)MixtralDecoderLayerMixtralExpertsMixtralForCausalLM MixtralModelload_balancing_loss_func)GraniteMoeConfigc eZdZy)GraniteMoeRMSNormN__name__ __module__ __qualname__|/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/granitemoe/modular_granitemoe.pyrr)r%rc eZdZy)GraniteMoeRotaryEmbeddingNr r$r%r&r)r)-r'r%r)ceZdZdZdeffd ZdejdeejejejffdZ xZ S)GraniteMoeTopKRouteraTop-k gating that returns the routing decisions without grouping tokens by expert. Returns ``(top_k_index, top_k_weights, router_logits)``; the grouping/scattering used to live here (via ``expert_size.tolist()``, which broke fullgraph compile) and now happens inside the experts forward via ``use_experts_implementation`` so the default ``grouped_mm`` / ``batched_mm`` paths can compile cleanly. configct||j|_|j|_t jtj|j|j|_ yN) super__init__num_local_experts num_expertsnum_experts_per_toktop_kr Parametertorchempty hidden_sizeweightselfr, __class__s r&r0zGraniteMoeTopKRouter.__init__:sP !33// ll5;;t/?/?ASAS#TU r% hidden_statesreturnctj||jj}|j |j d\}}t j|dj|}|||fS)N)dim) Flinearr9floattopkr4r6softmaxtype_as)r;r= router_logits top_k_logits top_k_index top_k_weightss r&forwardzGraniteMoeTopKRouter.forward@se <BBD $1$6$6tzzr$6$J! k l;CCMR M=88r%) r!r"r#__doc__rr0r6TensortuplerL __classcell__r<s@r&r+r+1sIV/V 9U\\9eELL%,,X]XdXdc|j\}}}|jd|}|j|\}}}|j|||} | j |||j S)Nr@)sizereshaperXrYviewrW) r;rZbszlengthemb_sizer=rJrK_ layer_outputs r&rLzGraniteMoeMoE.forwardTsi + 0 0 2VX#++B9 (, M(B% ]A||M; N   fdoo>>r%) r!r"r#rMrr0r6rNrLrPrQs@r&rUrUKs.S1/1 ?5<<?ELL?r%rUc(eZdZdedeffd ZxZS)GraniteMoeAttentionr, layer_idxcJt|||||j|_yr.)r/r0attention_multiplierscalingr;r,rfr<s r&r0zGraniteMoeAttention.__init__]s! vy122 r%)r!r"r#rintr0rPrQs@r&rere\s3/3C33r%rec eZdZdedeffd Z d dejdejdzdedzde ejejfdzd ejf d Z xZ S) GraniteMoeDecoderLayerr,rfcHt|||t|||_t ||_t |j|j|_ t |j|j|_ |` t ||_|j|_ y)N)r,rfeps) r/r0re self_attnrUblock_sparse_moerr8 rms_norm_epsinput_layernormpost_attention_layernormmlpresidual_multiplierrjs r&r0zGraniteMoeDecoderLayer.__init__cs +,FiP -f 501C1CI\I\](9&:L:LRXReRe(f% H -f 5#)#=#= r%Nr=attention_maskpast_key_valuesposition_embeddingsr>c |}|j|}|jd||||d|\}}|||jzz}|}|j|}|j |}|||jzz}|S)N)r=rxryrzr$)rtrqrwrurr)r;r=rxryrzkwargsresidualrbs r&rLzGraniteMoeDecoderLayer.forwardms!,,]; )4>> ')+ 3     q!=43K3K#KK  55mD --m<  =43K3K#KK r%)NNN) r!r"r#rrkr0r6rNrrOrLrPrQs@r&rmrmbs>/>C>/3(,HL || t+  #5<<#=>E  r%rmcfeZdZUeed<dZdZdgZdgZdZ dZ dZ e jdZy)GraniteMoePreTrainedModelr,modelTrmryctj||t|trmt j |j d|jjt j |jd|jjyt|tr7t j |jd|jjyy)Ng)meanstd) r _init_weights isinstancerSinitnormal_ gate_up_projr,initializer_range down_projr+r9)r;modules r&rz'GraniteMoePreTrainedModel._init_weightss%%dF3 f/ 0 LL,,3DKKQ>QR $*$?$?! isBN input_idsrx position_idsry inputs_embeds use_cacher|r>c X|du|duz r td|r|t|j}||j|}|V||j nd}t j |jd|j|z}|jd}t|j||||} ||jz}|} |j| |} |jd|jjD]} | | f| | |||d|} |j| } t!| |S) Nz:You must specify exactly one of input_ids or inputs_embeds)r,rr)device)r,rrxryr)rzrxrryr)last_hidden_statery) ValueErrorrr, embed_tokensget_seq_lengthr6arangeshaper unsqueezerr rotary_embrrrr ) r;rrxrryrrr|past_seen_tokens causal_maskr=rz decoder_layers r&rLzGraniteMoeModel.forwards\  -t"; <YZ Z 0*$++>O   --i8M  CRC^==?de  << (;(;A(>}G[G[\_ooL'11!4L(;;')+%  &(A(AA % #oom\J![[)H4;;+H+HI M)$7*) /# M  -0 %++  r%)NNNNNN)r!r"r#rr0rrrr6 LongTensorrNr FloatTensorboolr r r rLrPrQs@r&rrs@/@ .2.204(,26!%5 ##d*5  t+5 &&- 5  5 ((4/ 5 $;5 +,5  5  5 r%rc$eZdZdeffd Zee ddejdzdejdzdejdzde dzdejdzd ejdzd e dzd e ejzd eezfd ZxZS)GraniteMoeForCausalLMr,cft||t||_|j|_yr.)r/r0rrlogits_scalingr:s r&r0zGraniteMoeForCausalLM.__init__s*  $V, $33r%Nrrxrryrlabelsoutput_router_logitslogits_to_keepr>c ||n|jj}|jd|||||d| } | j} t |t r t | dn|} |j| dd| ddf} | |jjz } d}|*|j| |fd|jji| }d}|rYt| j|j|j|}|+||j|j!|j"zz }t%||| | j&| j(| j*| jS)al labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from transformers import AutoTokenizer, GraniteMoeForCausalLM >>> model = GraniteMoeForCausalLM.from_pretrained("ibm/PowerMoE-3b") >>> tokenizer = AutoTokenizer.from_pretrained("ibm/PowerMoE-3b") >>> prompt = "Hey, are you conscious? Can you talk to me?" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=30) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." ```N)rrxrryr vocab_size)lossaux_losslogitsryr= attentionsrHr$)r,rrrrrkslicelm_headr loss_functionrrrHr2r3router_aux_loss_coeftorr ryr=r)r;rrxrryrrrrr|outputsr= slice_indicesrrrs r&rLzGraniteMoeForCausalLM.forwards|J%9$D $++JjJj $** )%+'     11 8B>SV8W~ot4]k mA}a,?@A$++444  %4%% ;;11 D /%%  (( H !11HKK 4LLL(#33!//))!//  r%)NNNNNNNr)r!r"r#rr0rrr6rrNrrrrkrOr rLrPrQs@r&rrs4/4 .2.204(,26*.,0-.Q ##d*Q  t+Q &&- Q  Q ((4/ Q   4'Q #TkQ ell*Q  * *Q Q r%r)rrr)6r6torch.nn.functionalr functionalrBrr cache_utilsrr masking_utilsrmodeling_outputsr r modeling_utilsr processing_utilsr utilsr r utils.genericrrutils.output_capturingrgranite.modeling_graniterrllama.modeling_llamarrmixtral.modeling_mixtralrrrrrconfiguration_granitemoerrr)Moduler+rSrUrermrrr__all__r$r%r&rs  &./Q-&7I5MG7   6 92999,  ?BII?"3.3 !0!HU 4oUU(A lA A HY .Y x Tr%