^jrddlmZddlZddlmZddlmZddlmZddl m Z m Z ddl m Z dd lmZdd lmZmZdd lmZmZdd lmZmZdd lmZmZddlmZddlmZm Z m!Z!ddl"m#Z#ddl$m%Z%ddl&m'Z'm(Z(ddl)m*Z*m+Z+m,Z,ddl-m.Z.ddl/m0Z0m1Z1ddl2m3Z3e!jhe5Z6Gdde+Z7Gdde*Z8GddejrZ:Gdd e.Z;Gd!d"e0Z<Gd#d$ejrZ=Gd%d&eZ>Gd'd(eZ?e>e?d)Z@Gd*d+eZAe Gd,d-eAZBGd.d/e1ZCGd0d1eeAZDgd2ZEy)3)CallableN)nn)initialization)ACT2FN)Cache DynamicCache)lazy_load_kernel)force_accelerate_hooks)create_causal_maskcreate_recurrent_attention_mask) GenericForSequenceClassificationGradientCheckpointingLayer)MoeCausalLMOutputWithPastMoeModelOutputWithPast)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringlogging)merge_with_config_defaults)resolve_internal_import)OutputRecordercapture_outputs)LlamaAttention LlamaRMSNormeager_attention_forward) MistralMLP)MixtralExpertsMixtralForCausalLM) JambaConfigc eZdZy) JambaRMSNormN__name__ __module__ __qualname__r/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/jamba/modular_jamba.pyr&r&/r,r&ceZdZdedeffd Z d dejdejdzdedzde e d e ejejdzff d Z xZ S) JambaAttentionconfig layer_idxct|||tj|j|j |j zd|_tj|j|j|j zd|_ tj|j|j|j zd|_ tj|j |j z|jd|_ yNFbias) super__init__rLinear hidden_sizenum_attention_headshead_dimq_projnum_key_value_headsk_projv_projo_proj)selfr1r2 __class__s r-r8zJambaAttention.__init__4s +ii 2 2F4N4NQUQ^Q^4^ejk ii 2 2F4N4NQUQ^Q^4^ejk ii 2 2F4N4NQUQ^Q^4^ejk ii : :T]] JFL^L^ejk r,N hidden_statesattention_maskpast_key_valueskwargsreturnc |jdd}g|d|j}|j|j|j dd}|j |j|j dd}|j |j|j dd} | |j|| |j\}} tj|jjt} | |||| |f|jsdn |j|j d|\} } | j"g|dj%} |j'| } | | fS)Nr#r)dropoutscaling)shaper<r=view transposer?r@updater2r get_interfacer1_attn_implementationrtrainingattention_dropoutrMreshape contiguousrA) rBrDrErFrG input_shape hidden_shape query_states key_states value_statesattention_interface attn_output attn_weightss r-forwardzJambaAttention.forward;sq$))#2. 88b8$--8 {{=166|DNNqRST [[/44\BLLQPQR {{=166|DNNqRST  &'6'='=j,X\XfXf'g $J (?(M(M KK , ,.E) %8      % $}}C$2H2HLL %  % ! \*k));;;;FFH kk+. L((r,NN)r(r)r*r$intr8torchTensorrrrtupler` __classcell__rCs@r-r0r03sl{lsl/3(, ")||") t+") ") +, ") u||U\\D00 1 ")r,r0ceZdZdZdeffd Z d dejdedzdejdzfdZ d dedzdejdzfd Z e d  d dedzdejdzfd Z xZS) JambaMambaMixeru Compute ∆, A, B, C, and D the state space parameters and compute the `contextualized_states`. A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective) ∆, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4, and is why Mamba is called **selective** state spaces) r1c t|||_||_|j|_|j |_|j|_|j|jz|_ |j|_ |j|_|j|_t#j$|j|j|j|j|j|jdz |_|j(|_t,|j(|_t#j0|j|jdz|j |_t#j0|j|j|j dzzd|_t#j0|j|jd|_t9j:d|j dzdddf}|j=|jdj?}t#j@t9jB||_"t#j@t9jF|j|_$t#j0|j|j|j |_%tM|j|jN|_(tM|j |jN|_)tM|j |jN|_*tWd }tY|d da-tY|d da.tWd }t_|d a0tY|dda1tY|dda2tgt`tbt\tZtdfa4thstjjmd|jn||_8y)Nr#) in_channels out_channelsr6 kernel_sizegroupspaddingrr5FTrJepsz causal-conv1dcausal_conv1d_updatecausal_conv1d_fnz mamba-ssmz8ops.triton.selective_state_update.selective_state_update) chained_pathselective_scan_fnmamba_inner_fnaThe fast path is not available because on of `(selective_state_update, selective_scan_fn, causal_conv1d_fn, causal_conv1d_update, mamba_inner_fn)` is None. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d.)9r7r8r1r2r: mamba_d_statessm_state_size mamba_d_convconv_kernel_size mamba_expandintermediate_size mamba_dt_ranktime_step_rankmamba_conv_bias use_conv_biasmamba_proj_biasuse_biasrConv1dconv1d hidden_act activationractr9in_projx_projdt_projrcarangeexpandrW ParameterlogA_logonesDout_projr& rms_norm_eps dt_layernorm b_layernorm c_layernormr getattrrrrsrselective_state_updaterurvallis_fast_path_availablelogger warning_once layer_types layer_type)rBr1r2A causal_conv1d mamba_ssmrCs r-r8zJambaMambaMixer.__init__hs  "!--$22 & 3 3!'!4!4v7I7I!I$22#33.. ii..//##--))))A-   !++&++,yy!1!143I3IA3MTXTaTab ii 6 68K8KdNaNadeNe8elqr yy!4!4d6L6LSWX  LLD//!3 4T1W = HHT++R 0 ; ; =\\%))A,/ ejj)?)?@A $"8"8$:J:JQUQ^Q^_ ()<)<&BUBUV'(;(;ATATU'(;(;ATATU)9 &}6LdS"=2DdK%[1 !8 $^" $I/BDI ,jj@}t1jB5t1jD|j>jj@|j>j_ ddd|j?| j dd}t1jB5||j>j_ dddt1jF|jHjK }||jKnd}|rjtM|j|jjNd|d|d||dddf|dddf|jP| d|d j d}nptS||||j dd|j dd|jPjK| |dd \}}|||jU||j|jW|j dd}|S#1swYxYw#1swYXxYw) Nr#rdimrrJ)r).rT) dt_softplus)delta_softplusreturn_last_state),rNhas_previous_stater2rrPchunk unsqueezerweightrOsizerrsqueezelayers conv_statesr6rr functionalpadrzupdate_conv_statersrrcsplitr~rxrrrrdatano_grad zeros_likeexprfloatrrecurrent_statesrruupdate_recurrent_stater)rBrDrrE batch_sizeseq_len_use_precomputed_statesprojected_statesgate conv_weightsrssm_parameters time_stepBCtime_proj_biasdiscrete_time_stepr scan_outputs ssm_statecontextualized_statess r-cuda_kernels_forwardz$JambaMambaMixer.cuda_kernels_forwardsB "/!4!4 GQ  $ i)H)H)X i]dhi]i  << 6@@AF/44QA4> t  %)N,D,DQ,GGM{{))..t{{/A/A/F/Fq/I4;;K]K]KbKbcdKef !0%%b)##DNN3??B    M*33B7M' mm// @U@UXeXkXklnXo@oqr?st ..{DNNK,]L$++JZJZgkgvgvwM  %)N,D,DQ,GGM]%<%>q!D ]]_ 4%3DLL   " 4YYtzz'') * *3A3M--/SW !1##DNN3DDQGf%"6*!Q$!Q$V   im '8" Aq! Aq! #"& ' #L)$)A33It~~N!% l.D.DQ.J K$$S N N 4 4sA R7S7SSc b |j\}}}|j}|j|jdd}|j dd\} } || |j dz} |P|j |jr5|j|jjdj} n9tj||j|jf| j|} |m|j |jr|dk(r|j!| |jd|j" df} tj$| |j&j(dddddfzd} |j*r| |j&j,z } |j/| j1|j d} nt2j4j7| |j"| jdz df} |j!| |jd|j" df} |j/|j'| dd|f} n'|j/|j'| dd|f} || |j dz} |j9| jdd} tj:| |j<|j|jgd\}}}|j?|}|jA|}|jC|}|jE|}t2j4jG|jdd}tjH|jJjM }tjH|ddddddf|dddddddfz}|dddddddf|dddddddfjMz}|| dddddddfjMz}g}tO|D]}}|dddd|ddf| z|dddd|ddfz} tjP| j1||dd|ddfj d}|jS|dddddftjT|d}|| |jVddddfzz}||j/| z}||jY| |j|j[|jdd}|S)Nr#rrr)devicedtype.rJ).rNrrrPrrrr2rrclonerczerosr|rxrrrzsumrrrr6rtorrrrrr~rrrrsoftplusrrrrangematmulappendstackrrr)rB input_statesrrErrrrrrDrr conv_staterrrrrr discrete_A discrete_BdeltaB_uri scan_outputrs r- slow_forwardzJambaMambaMixer.slow_forwards!-!3!3 GQ""<< 5??1E.44QA4> t  %)N,D,DQ,GGM  # (G(G(W$++DNN;LLQOUUWI T33T5H5HI$++5I  #..t~~>7a<);;M4>>Z[^aeavav`v`w[wx % *t{{7I7I!QPQ'7R*RXZ [ %%!T[[%5%55M $ 7 : :5 A K KB O ]]..!**]-@-@-DDaH *;;JWX[^b^s^s]s]tXtu $])CC'M)R S  HHT[[%?XgX %NOM  %)N,D,DQ,GGM]%<%> J!% k.C.CAq.I J$$r,rcV|jjrXtr,d|jjj j vr&tjdd|j_|jjr|j|||S|j|||S)NcudazFast Mamba kernels are not available. Make sure that they are installed and that the mamba module is on a CUDA device. Turning off the fast path `config.use_mamba_kernels=False` and falling back to the slow path.F) r1use_mamba_kernelsrrrrtyperrrr)rBrDrrEs r-r`zJambaMambaMixer.forward`s ;; ( (&& 8J8J8Q8Q8V8V*V   V  -2DKK ) ;; ( (,,]L.Y Y   nMMr,ra)r(r)r*__doc__r$r8rcrdr LongTensorrrr r`rfrgs@r-riri`sC8{C8P&*26 c%||c%dlc%((4/ c%LJ%ut|J%\a\l\los\sJ%ZH%&*26 NdlN((4/ N&Nr,ric eZdZy)JambaMLPNr'r+r,r-rrvr.r,rc eZdZy) JambaExpertsNr'r+r,r-rrzr.r,rcfeZdZdZdeffd ZdZdejdejfdZ xZ S)JambaSparseMoeBlocka This implementation is strictly equivalent to standard MoE with full capacity (no dropped tokens). It's faster since it formulates MoE operations in terms of block-sparse operations to accommodate imbalanced assignments of tokens to experts, whereas standard MoE either (1) drop tokens at the cost of reduced performance or (2) set capacity factor to number of experts and thus waste computation and memory on padding. r1c,t||j|_|j|_|j |_|j|_tj|j|j d|_ t||_ yr4)r7r8r: hidden_dimr|ffn_dim num_expertsnum_experts_per_toktop_krr9routerrexpertsrBr1rCs r-r8zJambaSparseMoeBlock.__init__sm  ,,// !--// ii1A1AN #F+ r,ctjjj|dtj}tj ||j d\}}||j|jfS)NrJ)rrr) rcrrsoftmaxrtopkrrr)rBrD router_logitsrouting_weights top_k_weights top_k_indexs r-route_tokens_to_expertsz+JambaSparseMoeBlock.route_tokens_to_expertssb((--55mSXS^S^5_%*ZZQS%T" {M,,]-@-@AAAr,rDrHc|j\}}}|jd|}|j|}|j||\}}|j |||}|j |||}|S)NrJ)rNrOrrrrV)rBrDrsequence_lengthrrrrs r-r`zJambaSparseMoeBlock.forwardsx2?2E2E/ OZ%**2z:  M2 %)%A%A-Q^%_" ] ]KO %--j/:V r,) r(r)r*rr$r8rrcrdr`rfrgs@r-rr~s5 ,{,B U\\ellr,rceZdZdedeffd Z d dejdejdzdejdzde dzd e dzd e e d ejfd ZxZS)JambaAttentionDecoderLayerr1r2cRt||jr|j|nd}t|||_|dkDrt nt }|||_t|j|j|_ t|j|j|_ y)Nr#rp) r7r8layers_num_expertsr0 self_attnrr feed_forwardr&r:rinput_layernormpre_ff_layernormrBr1r2rffn_layer_classrCs r-r8z#JambaAttentionDecoderLayer.__init__s >D>W>Wf// :]^ ' :1,>FDWDWX ,V-?-?VEXEX Yr,NrDrE position_idsrF use_cacherGrHc |}|j|}|jd|||||d|\}}||z}|}|j|}|j|}||z}|S)N)rDrEr rFr r+)rrrr) rBrDrEr rFr rGresidualrs r-r`z"JambaAttentionDecoderLayer.forwards!,,]; )4>> ')%+     q!=0  --m< ))-8  =0 r,)NNNF)r(r)r*r$rbr8rcrdrrboolrr FloatTensorr`rfrgs@r-rrsZ{ZsZ/304(,!& || t+&&-    $; +,   r,rc eZdZdedeffd Z d dejdejdzdejdzde dzd e e d ejf d Z xZS) JambaMambaDecoderLayerr1r2cTt||jr|j|nd}t|||_|dkDrt nt }|||_t|j|j|_ t|j|j|_ y)Nr#)r1r2rp) r7r8rrimambarrrr&r:rrrrs r-r8zJambaMambaDecoderLayer.__init__s >D>W>Wf// :]^ $FiH 1,>FDWDWX ,V-?-?VEXEX Yr,NrDrEr rFrGrHc |}|j|}|j|||}||z}|}|j|}|j|}||z}|S)N)rDrrE)rrrr)rBrDrEr rFrGr s r-r`zJambaMambaDecoderLayer.forwardsv!,,];  '()# !=0  --m< ))-8  =0 r,)NNN)r(r)r*r$rbr8rcrdrrrrrr`rfrgs@r-rrsZ{ZsZ/304(, || t+&&-    +,    r,r) attentionrceZdZUeed<dZdZddgZdgZdZ dZ dZ dZ e egeeej$dd Zej*fd ZxZS) JambaPreTrainedModelr1modelTrrrFr) layer_name)rD attentionsrct||t|trt j d|j dzdddf}|j|jdj}tj|jt j|tj|jyt|t rmtj"|j$d|j&j(tj"|j*d|j&j(yy)Nr#rJrK)meanstd)r7 _init_weights isinstancerircrrxrr|rWinitcopy_rrones_rrnormal_ gate_up_projr1initializer_range down_proj)rBmodulerrCs r-rz"JambaPreTrainedModel._init_weightss f% fo . Q 5 5 9:47CA1126AACA JJv||UYYq\ 2 JJvxx  - LL,,3DKK,>FDWDWX&+# r,N input_idsrEr rF inputs_embedsr rGrHc t|du|duz r td||j|}|r|t|j}|V||j nd}t j |jd|j|z}|jd}t|x} ts)|j||||d} td i| td i| d} |} t|jD]-\} } | | f| |jj | |||d|} /|j#| } t%| | S) Nz:You must specify exactly one of input_ids or inputs_embeds)r1rr#)r)r1rDrErFr )full_attentionlinear_attention)rEr rFr )last_hidden_staterFr+) ValueErrorr9r r1get_seq_lengthrcrrNrrrdictr r enumeraterrr>r)rBrCrEr rFrDr rGpast_seen_tokenscausal_mask_mapping mask_kwargsrDr decoder_layers r-r`zJambaModel.forwardss  -t"; <YZ Z   --i8M 0*$++>O  CRC^==?de  << (;(;A(>}G[G[\_ooL'11!4L?-F++!."0#2 , K#5"C{"C$C$Rk$R# & )$++ 6  A})24;;3J3J13MN) /#   M ,,]; %++  r,)NNNNNN)r(r)r*r$r8rrrrcrrdrrrrrrr`rfrgs@r-r3r3 s{$ .2.204(,26!%6 ##d*6  t+6 &&- 6  6 ((4/ 6 $;6 +,6  6  6 r,r3c$eZdZdeffd Z ddej dzdejdzdej dzdedzdejdzd ej dzd e dzd e dzd e ejzd e e deffd ZxZS)JambaForCausalLMr1cFt|||j|_y)N)r7r8rrs r-r8zJambaForCausalLM.__init__[s  !--r,NrCrEr rFrDlabelsr output_router_logitslogits_to_keeprGrHc 2t ||||||||| fi| S)aj labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Example: ```python >>> from transformers import AutoTokenizer, JambaForCausalLM >>> model = JambaForCausalLM.from_pretrained("ai21labs/Jamba-v0.1") >>> tokenizer = AutoTokenizer.from_pretrained("ai21labs/Jamba-v0.1") >>> prompt = "Hey, are you conscious? Can you talk to me?" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=30) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." ```)r7r`) rBrCrEr rFrDrTr rUrVrGrCs r-r`zJambaForCausalLM.forward_s9Fw            r,) NNNNNNNNr)r(r)r*r$r8rcrrdrrrrbrrrr`rfrgs@r-rRrRZs.{. .2.204(,26*.!%,0-.- ##d*-  t+- &&- -  - ((4/ -   4'- $;- #Tk- ell*- +,-  #- - r,rRc eZdZy)JambaForSequenceClassificationNr'r+r,r-rYrYr.r,rY)rRrYr3r)Fcollections.abcrrcrrr  activationsr cache_utilsrr integrationsr integrations.accelerater masking_utilsr r modeling_layersrrmodeling_outputsrrmodeling_utilsrrprocessing_utilsrutilsrrr utils.genericrutils.import_utilsrutils.output_capturingrrllama.modeling_llamarrrmistral.modeling_mistralr mixtral.modeling_mixtralr!r"configuration_jambar$ get_loggerr(rr&r0Modulerirrrrrr;rr3rRrY__all__r+r,r-rps6&% &!.,=P[QF&@@79EXX1I,   H % < *)^*)ZSNbiiSNl z  > ""))"J#!;#L7B)CMcdX?X:L %L L ^2 )2 j %EG[  gr,