^j*DdZddlmZddlZddlmZddlmZddlm Z m Z ddl m Z dd l mZdd lmZdd lmZmZdd lmZdd lmZmZmZddlmZddlmZddlmZm Z m!Z!m"Z"m#Z#m$Z$ddl%m&Z&m'Z'm(Z(ddl)m*Z*ddl+m,Z,ejZe.Z/Gdde!Z0Gdde"Z1GddeZ2GddeZ3Gdde&Z4Gd d!e*Z5Gd"d#ejlZ7Gd$d%e Z8eGd&d'eZ9eGd(d)e(Z:Gd*d+e'e Z;gd,Z)__name__ __module__ __qualname__r& __classcell__r*s@r+r"r"-s ++r,r"c eZdZy)OlmoeRotaryEmbeddingNr-r.r/r,r+r3r32r,r3c eZdZy)OlmoeMLPNr4r5r,r+r8r86r6r,r8ceZdZd dededzffd Z d dejdeejejfdejdzde dzd e e d eejejdzeejdzff d Z xZ S) OlmoeAttentionNconfig layer_idxct|||t|j|j|_t|j|j z|jz|j|_y)Nr)) r%r&r"r( rms_norm_epsq_normnum_attention_headsnum_key_value_headsk_normr'r;r<r*s r+r&zOlmoeAttention.__init__;s` +"6#5#56;N;NO "   6#=#= =A[A[ [agatat  r, hidden_statesposition_embeddingsattention_maskpast_key_valueskwargsreturnc |jdd}g|d|j}|j|j|}|j |j |} |j |} |jj|j|jj |jj| j|jj |jj| j|jj |jj|j|jdd}| j|jdd} | j|jdd} |\} } t|| | | \}} | |j| | |j\} } tj |jj"t$} | ||| | |f|j&sdn |j(|j*t-|jddd|\}}|j.g|dj1}|j3|}||fS)N)minmaxrrsliding_window)dropoutscalingrP)shapehead_dimr@q_projrCk_projv_projr;clip_qkvclamp_view transposerupdater<r get_interface_attn_implementationrtrainingattention_dropoutrRgetattrreshape contiguouso_proj)r'rErFrGrHrI input_shape hidden_shape query_states key_states value_statescossinattention_interface attn_output attn_weightss r+forwardzOlmoeAttention.forwardBsR$))#2. 88b8$--8 {{4;;}#=> [[]!;< {{=1 ;;   +   T[[%9%9$9t{{?S?S  T   4;;#7#7"7T[[=Q=Q  R   T[[%9%9$9t{{?S?S  T(|((,7AA!QG $Z__l3==aC (|((,7AA!QG &S#7 jRUWZ#[ j  &'6'='=j,X\XfXf'g $J (?(M(M KK , ,.E) %8      % $}}C$2H2HLL"4;;0@$G %  % ! \*k));;;;FFH kk+. L((r,r$)r-r.r/r intr&torchTensortuplerrrror0r1s@r+r:r::s { sTz )- /)||/)#5<<#=>/) t+ /)  /) +, /) u||U\\D0% 2E2LL M/)r,r:c eZdZy) OlmoeExpertsNr4r5r,r+rurutr6r,ruc eZdZy)OlmoeTopKRouterNr4r5r,r+rwrwxr6r,rwcVeZdZfdZdej dej fdZxZS)OlmoeSparseMoeBlockcbt|t||_t ||_yr$)r%r&rwgateruexpertsr'r;r*s r+r&zOlmoeSparseMoeBlock.__init__}s& #F+ #F+ r,rErJc|j\}}}|jd|}|j|\}}}|j|||j |||}|S)NrL)rSrZr{r|rb) r'rE batch_sizesequence_length hidden_dim_ top_k_weights top_k_indexfinal_hidden_statess r+rozOlmoeSparseMoeBlock.forwardsh2?2E2E/ OZ%**2z: (, -(@%=+"ll=+}U]]  #"r,)r-r.r/r&rqrrror0r1s@r+ryry|s#, #U\\#ell#r,ryc(eZdZdedeffd ZxZS)OlmoeDecoderLayerr;r<c$t||||j|_t|||_t ||_t|j|j|_ t|j|j|_ y)N)r;r<r>) r%r&r(r: self_attnrymlpr"r?input_layernormpost_attention_layernormrDs r+r&zOlmoeDecoderLayer.__init__sp +!--'vK&v.+F,>,>FDWDWX(4V5G5GVM`M`(a%r,)r-r.r/r rpr&r0r1s@r+rrsb{bsbbr,rceZdZUeed<dZdZdgZdgZdZ dZ e e de edZdZej$d Zy ) OlmoePreTrainedModelr;modelTrrHr)index) router_logitsrE attentionsctj||t|trmt j |j d|jjt j |jd|jjyt|tr7t j |jd|jjyy)NrO)meanstd) r _init_weights isinstanceruinitnormal_ gate_up_projr;initializer_range down_projrwweight)r'modules r+rz"OlmoePreTrainedModel._init_weightss%%dF3 fl + LL,,3DKKr;)r%r&r Embedding vocab_sizer( padding_idx embed_tokens ModuleListrangenum_hidden_layersrlayersr"r?normr3 rotary_embrDs r+r&zOlmoeModel.__init__s  LL):):FO   --i8M  CRC^==?de  << (;(;A(>}G[G[\_ooL'11!4L(;;')+%  & #oom\J![[)H4;;+H+HI M)$7*) /# M  -0 %++  r,)NNNNNN)r-r.r/r r&rq LongTensorrrr FloatTensorboolrrr ror0r1s@r+rrs>{>.2.204(,26!%5 ##d*5  t+5 &&- 5  5 ((4/ 5 $;5 +,5  5 r,rc0eZdZddiZfdZfdZxZS)OlmoeForCausalLMzlm_head.weightzmodel.embed_tokens.weightcft||t||_|j|_yr$)r%r&rr num_expertsr}s r+r&zOlmoeForCausalLM.__init__s*  ' !--r,c "t|di|S)u labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from transformers import AutoTokenizer, OlmoeForCausalLM >>> model = OlmoeForCausalLM.from_pretrained("allenai/OLMoE-1B-7B-0924") >>> tokenizer = AutoTokenizer.from_pretrained("allenai/OLMoE-1B-7B-0924") >>> prompt = "Hey, are you conscious? Can you talk to me?" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=30) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] 'Hey, are you conscious? Can you talk to me?\nI’m not sure if you’re conscious of this, but I’m' ``` r5)r%ro)r' super_kwargsr*s r+rozOlmoeForCausalLM.forwards0w...r,)r-r.r/_tied_weights_keysr&ror0r1s@r+rrs*,GH. //r,r)rrr)=__doc__collections.abcrrqrrr cache_utilsrr generationr masking_utilsr modeling_outputsr modeling_utilsr r processing_utilsrutilsrrrutils.output_capturingrgemma.modeling_gemmarllama.modeling_llamarrrrrrmixtral.modeling_mixtralrrrqwen2_moe.modeling_qwen2_moerconfiguration_olmoer get_loggerr-loggerr"r3r8r:rurwModuleryrrrr__all__r5r,r+rs$ &.)/6F&@@4+XW=,   H %+<+  /  x 7)^7)t >  (  #")) # b)bU?UU4? ? ? D /)? /F Er,