^j0}^dZddlZddlmZddlZddlmZddlmZddl m Z ddl m Z dd l mZdd lmZmZmZmZmZmZmZd d lmZej2eZdad ZGddej<j>Z d'dZ!d'dZ"GddejFZ$GddejFZ%Gdde Z&eGddeZ'edeGddeZ(edeGdd eZ)eGd!d"e'Z*ed#Gd$d%e'e Z+gd&Z,y)(zPyTorch RWKV model.N) dataclass)nn)initialization)GenerationMixin)GradientCheckpointingLayer)PreTrainedModel) ModelOutputauto_docstringis_bitsandbytes_availableis_kernels_availableis_ninja_availableis_torch_cuda_availablelogging) RwkvConfigcdts tdddlm}|dda|t_y)NzFkernels is not installed, please install it with `pip install kernels`r) get_kernelzkernels-community/rwkvr)version)r ImportErrorintegrations.hub_kernelsrrwkv_cuda_kernelmax_seq_length)context_lengthrs q/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/rwkv/modeling_rwkv.pyload_wkv_cuda_kernelr-s.  !bcc6!":AF&4#c0eZdZeddZeddZy)RwkvLinearAttentionNc |j\}}} |tjkDr td|dtjd|| zt | dzdk7rtd|d| dt | dd |j |_|jjd k7sK|jjd k7s2|jjd k7s|jjd k7r td tj|jj }|j tjk(r0|j}|j}|j}|j}|j}|j}tj|tj } |s||Vtj || d tj"|jtj}|dddddfxxdzcc<nBtj$|D cgc]} | j'dc} dj}|j tj(k(rtj*} ntj,} | ||||| |nI|j tj(k(rtj.ntj0} | ||||| |j3||||| |4tj4|d dD cgc]} | j7d}} | j9|j |fScc} wcc} w)NzCannot process a batch with z+ tokens at the same time, use a maximum of z with this model. rzThe product of batch size (z) and hidden size (z") needs to be a round multiple of .cudazUCalling the CUDA kernel for wkv attention requires all tensors to be on CUDA devices. memory_formatr)dtypedevicer%籡*G)dim)sizerr ValueErrorminr& input_dtyper'typetorchexpfloat contiguousfloat16 empty_likecontiguous_formatzerosfloat32cat unsqueezebfloat16forward_with_state_bf16forward_with_state forward_bf16forwardsave_for_backwardchunksqueezeto) ctx time_decay time_firstkeyvaluestate return_state batch_sizeseq_len hidden_sizeoutputs forward_funcs rr?zRwkvLinearAttention.forward9s+.88:( G[ %44 4.wi7b#2233DF   #c+r&: :a ?-j\9L[MZ";34A7  ))    " "f ,  %%/zz&(||  F*tu uii 0 0 2 = = ?@@ 99 %#))+J))+CKKME**, nn  "!!#U5L5LM 5,} --::"'"9"9 aAg$& 5"Aa1;;q>"AqITTVyyENN*/GG /BB ZeVU K>$A%..u}} '' %BYBYZ   E4K4KL""58O8OP %-- '~~'H:E:W(66]m]v]v          !      OOK ( OOK ( HH[ ! JJ{ #     rNFN)__name__ __module__ __qualname__ staticmethodr?rTrrrr8s)<1<1|% % rrc|j\}}}tj|}|tj|dddftj} tj|dddftj} tj|dddftjdz } n|\} } } tj| }t |D]} |dd| fj } |dd| f}tj| | |z}tj| |z }tj| |z|z }|| z||zz}|| z|z}||z j|j|dd| f<tj| |z| }tj| |z|z }tj| |z }|| z||zz} || z|z} |}  |s|| | | g}||fS)Nr)r&r)) r+r0 zeros_liker8r1ranger2maximumrCr&)rErFrGrHrIrJ_ seq_lengthrN num_state den_state max_state current_index current_key current_valuemax_for_outpute1e2 numerator denominator max_for_states rrwkv_linear_attention_cpurusxxzAz1   c "F }$$SAYemmD $$SAYemmD $$SAYemmDtK */' 9i ))J''Jz*" !]*+113 a./ y+ 2JK YYy>1 2 YY{Z/.@ ANR-%77 9nr) $- $;#?#? #Mq-  i*& YY{]2 3NR-%77 NR' ! %"(u(Iy1 5=rctd||||fD}|jddk(}t|s|rt||||||Stj ||||||S)Nc3NK|]}|jjdk7yw)r#N)r'r/).0ts r z(rwkv_linear_attention..sXa!((--6)Xs#%rrIrJ)anyr+rrurapply)rErFrGrHrIrJno_cuda one_tokens rrwkv_linear_attentionrsmXJ CQV3WXXG q I7i(ZeSXgstt"((ZeUT`aarc0eZdZdfd ZddZddZxZS)RwkvSelfAttentioncrt|||_tduxrtj|j k(}t r"tr|s t|j ||_ |j}|j |jn|}||_tj t#j$||_tj t#j$||_tj t#j$dd||_tj t#j$dd||_tj t#j$dd||_tj0d|_tj4||d|_tj4||d|_tj4||d|_tj4||d|_y#t$rtjdYwxYw)Nz9Could not load the custom CUDA kernel for RWKV attention.rrrrFbias)super__init__configrrrrrr Exceptionloggerinfolayer_idrMattention_hidden_sizer Parameterr0emptyrErF time_mix_keytime_mix_valuetime_mix_receptance ZeroPad2d time_shiftLinearrGrH receptancerN)selfrr kernel_loadedrMr __class__s rrzRwkvSelfAttention.__init__s  (4q9I9X9X\b\q\q9q  $;$=m Y$V%:%:;! (( ,2,H,H,TF ( (Ze &;",,u{{3H'IJ,,u{{3H'IJLLQ;)GH ll5;;q![+IJ#%<< Aq+0N#O ,,}599[*?eLYY{,AN ))K1FUSii 5{O ) Y WX YsHH65H6cp|jddk(r||ddddd|jf}n3|j|}| |ddddd|jf|dddf<||jz|d|jz zz}||jz|d|jz zz}||j z|d|j z zz}|j |}|j|}tj|j|}| |dddf|ddddd|jf<||||fSNrrr) r+rrrrrrGrHr0sigmoidr)rhiddenrIshiftedrGrHrs rextract_key_valuez#RwkvSelfAttention.extract_key_values< ;;q>Q 5#4Ahq!T]]23Goof-G %aAt}})< =1 t(((7a$:K:K6K+LL,,,w!d>Q>Q:Q/RRd666AH`H`D`9aa hhsm 5!]]4??:#>?  ,21b5ME!HQ4==( )3u,,rcj||\}}}}|tfd|ddDnd}tjj||||\}}|T|d|dddddj f<|d|dddddj f<|d|dddddj f<j ||z|fS) NrIc3JK|]}|ddddjfywr]r)rxrOrs rrzz,RwkvSelfAttention.forward.. s!FqAaDMM12Fs #r(r{rrr)rtuplerrErFrrN) rrrI use_cacherrGrH layer_staterwkvs ` rr?zRwkvSelfAttention.forward s(,(>(>vU(>(S% CJOJ[eFE!"IFFae 1 OO OO  "  k  ",7NE!HQ4==( ),7NE!HQ4==( ),7NE!HQ4==( ){{:,-u44rrr]r\)r^r_r`rrr? __classcell__rs@rrrsP<-&5rrc(eZdZdfd ZddZxZS)RwkvFeedForwardcBt|||_||_|j}|j |j nd|jz}t jd|_t jtjdd||_ t jtjdd||_ t j||d|_t j||d|_t j||d|_y)NrrrFr)rrrrrMintermediate_sizerrrrr0rrrrrGrrH)rrrrMrrs rrzRwkvFeedForward.__init__ s    (( (.(@(@(LF $ $RSV\VhVhRh ,,}5LLQ;)GH#%<< Aq+0N#O 99[*;%H))K5IYY0+EJ rcz|jddk(r||ddddd|jf}n3|j|}| |ddddd|jf|dddf<||jz|d|jz zz}||jz|d|jz zz}t j t j|j|}|j|}t j|j|}| |dddf|ddddd|jf<||z|fSr) r+rrrrr0squarerelurGrHrr)rrrIrrGrrHs rr?zRwkvFeedForward.forward1s) ;;q>Q 5#4Ahq!T]]23Goof-G %aAt}})< =1 t(((7a$:K:K6K+LLd666AH`H`D`9aa ll5::dhhsm45 3]]4??:#>?  ,21b5ME!HQ4==( )E!5((rrr]r^r_r`rr?rrs@rrrsK")rrc&eZdZfdZddZxZS) RwkvBlockct|||_||_|dk(r0t j |j |j|_t j |j |j|_ t j |j |j|_ t|||_ t|||_y)Nr)eps)rrrrr LayerNormrMlayer_norm_epsilonpre_lnln1ln2r attentionr feed_forward)rrrrs rrzRwkvBlock.__init__Fs    q=,,v'9'9v?X?XYDK<< 2 28Q8QR<< 2 28Q8QR*68<+FH=rc|jdk(r|j|}|j|j|||\}}||z}|j |j ||\}}||z}||f}|r||fz }|S|dz }|S)Nr)rIrrr])rrrrrr)rrrIroutput_attentionsrroutputss rr?zRwkvBlock.forwardTs ==A [[(F>>$((6*:%S\>] 5)#"//0@/N e,&5/   | #G w Gr)NFFrrs@rrrEs  >rrceZdZUeed<dZdgZddgZdZdZ e jde jffd ZxZS) RwkvPreTrainedModelrrrrErFTmodulec  t||t|tr|j}|j j }|j j}|j}||dz z }d||z z }tjt|Dcgc]}||z  c}|jj|jj} | ddddf} t|D cgc]} dd| |dz z dd|zzzzz} } tj| |jj|jj} tjt|Dcgc] }|dzd zdz c}|j j|j jd z} t#j$|j| t#j$|j tj&|j t)j*d z| zt#j$|jtj,| |t#j$|j.tj,| |d |zzt#j$|j0tj,| d |zyt|t2r |j}|j j }|j j}d||z z }tjt|Dcgc]}||z  c}|jj|jj} | ddddf} t#j$|jtj,| |t#j$|j0tj,| |yt|t4j6r|j8j:} d}d}|j<t#j>|j<| d | dkDrt)j@| d | dz }| d |j jBk(r| d|j jk(rd }||z}t#jD|j8| yt|t4jFr`|j8j:} dt)j@tI| d | dz}t#jD|j8| yycc}wcc} wcc}wcc}w)zInitialize the weights.rg?r&r'Ngffffff?g?rg?g333333?r)gaing-C6?)%r _init_weights isinstancerrrnum_hidden_layersrMrr0tensorrerr&r'rErFinitcopy_ ones_likemathlogpowrrrrrweightshaperzeros_sqrt vocab_size orthogonal_ Embeddingmax)rrrrrMr ratio_0_to_1ratio_1_to_almost0i time_weighth decay_speedzigzagrrscalers rrz!RwkvPreTrainedModel._init_weightspsS f% f/ 0H & ? ?  --33K$*$@$@ !#'81'<=L!$3D(D!E ,,*/ *<=Q[=))//**11K &dD!m4K45Q!4q89sS L*>s U'U #U3 U)r^r_r`r__annotations__base_model_prefix_no_split_modules_keep_in_fp32_modulessupports_gradient_checkpointing _is_statefulr0no_gradrModulerrrs@rrrgsU $ )<8&*#LU]]_G7BIIG7G7rrz+ Class for the RWKV model outputs. ) custom_introceZdZUdZdZej dzed<dZe ej dzed<dZ e ej dfdzed<dZ e ej dfdzed<y) RwkvOutputa  state (list of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`): The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to avoid providing the old `input_ids`. Nlast_hidden_staterI. hidden_states attentions) r^r_r`__doc__rr0 FloatTensorrrIlistrrrrbrrrrsw 37u((4/6,0E4!! "T )0:>M5**C/047>7;Je'',-4;rrzK Base class for causal language model (or autoregressive) outputs. ceZdZUdZdZej dzed<dZej dzed<dZ e ej dzed<dZ e ej dfdzed<dZ e ej dfdzed<y) RwkvCausalLMOutputap loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction). logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). state (list of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`): The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to avoid providing the old `input_ids`. NlosslogitsrI.rr)r^r_r`rrr0rrrrIrrrrrbrrrrs&*D%  d ")'+FE   $+,0E4!! "T )0:>M5**C/047>7;Je'',-4;rrc eZdZfdZdZdZe ddejdzdejdzdejdzde ejdzd e dzd e dzd e dzd e dzd e e zfdZdZdZxZS) RwkvModelc t||tj|j|j |_tjt|jDcgc]}t||c}|_ tj|j |_ d|_d|_|j!ycc}w)NrF)rrrrrrM embeddings ModuleListrerrblocksrln_outlayers_are_rescaledgradient_checkpointing post_init)rridxrs rrzRwkvModel.__init__s  ,,v'8'8&:L:LMmmPUV\VnVnPo$pYv%D$pq ll6#5#56 #( &+# %qs&C c|jSr]rrs rget_input_embeddingszRwkvModel.get_input_embeddingss rc||_yr]rrnew_embeddingss rset_input_embeddingszRwkvModel.set_input_embeddingss (rN input_idsattention_mask inputs_embedsrIrroutput_hidden_states return_dictreturnc *||n|jj}||n|jj}||n#|js|jjnd}||n|jj }|t jd|j|jk(r|j| | td| | td||j|}|r||jd|jj|jjf} tdD cgc]A} t!j"| | dkr |j$nt j&|j(d C}} |d xxd zcc<|j*r%|jr|rt jd d}|} |rd nd} |rd nd}t-|j.D]o\}}|| |||\} }}|jr=|jj0dkDr$|dz|jj0zdk(r| dz } |r|| fz}|sj| |fz} q|j3| } |r|| fz}|st5d| ||| fDSt7| ||| Scc} w)a input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) state (tuple of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`, *optional*): If passed along, the model uses the previous state in all the blocks (which will give the output for the `input_ids` provided as if the model add `state_input_ids + input_ids` as context). use_cache (`bool`, *optional*): If set to `True`, the last state is returned and can be used to quickly generate the next logits. NFz<`attention_mask` was passed, but it is unused in this model.zDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embedsrrrrgꌠ9Y>)FzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...rb)rIrrr(c3&K|] }|| ywr]rb)rxxs rrzz$RwkvModel.forward..^stqfgfsts)rrIrr)rrrtrainingrrr warning_oncer_rescale_layersr,rr+rMrrer0r7r&r8r'r enumerater rescale_everyrrr)rrrrrIrrrrkwargsrrrall_self_attentionsall_hidden_statesrblockrs rr?zRwkvModel.forwards@2C1N-TXT_T_TqTq$8$D $++JjJj "+!6IZ^ZgZgT[[=R=Rmr %0%cd d  =#8TU U   OOI6M "''*DKK,C,CT[[EbEbcE q  a-"5"5U]][h[o[oE !H H  & &4==##p" % $5b4"6BD#DKK0 JJC/4UiSd0 ,M5* ((KK--11W 9 99Q> - 1 #$58H$H! &9ZM&I#! J$ M2  1]4D D t]E;LNa$btt t++*   [s5AJc |j|j k(ry|jjdkDrt j 5t |jD]\}}|jr|jjjjdt||jjzz|jjjjdt||jjzzt|jjjdr|jjjj j#dt||jjzz|jjjj j#dt||jjzzt|jjjdrN|j%|jj||j%|jj||jjjj#dt||jjzz|jjjj#dt||jjzz ddd|j |_y#1swYxYw)Nrr(SCB quant_state)rrrrr0rrrrrNrmul_intrrHhasattrr"div_ _bnb_4bit_dequantize_and_rescale)rblock_idr s rrzRwkvModel._rescale_layersgs`  # #DMM(9 :  ;; $ $q ( r'0'=rOHe}}..55::1HPTP[P[PiPiDi@j;jk**0077<>qCTXT_T_TmTmHmDn?no!..44;;@@c(VZVaVaVoVoJoFpApqr r"(,}}#4 # r rs KL77Mcts tdddl}|jj |j j |j j}|jdt||jjzz|jj|jddj|j}t!|d|y) z Perform the dequantization and rescaling of the weights of a given layer. After that operation the layer will be quantized again. z/Please install bitsandbytes to use this method.rNr(cpuF) requires_gradr)r r bitsandbytes functionaldequantize_4bitrdatar#r'r%rrr Params4bitrCr'setattr)r target_layerr)bnbdequant_weights quant_weights rr(z*RwkvModel._bnb_4bit_dequantize_and_rescales )*OP P"..889L9L9Q9QS_SfSfSrSrsQ#h$++2K2K&K"LLMvv((););E)BRW(X[[\k\r\rs  h 5r)NNNNNNNN)r^r_r`rr r r r0 LongTensorrrboolrrr?rr(rrs@rrrs ).2262604!%)-,0#'h ##d*h ((4/h ((4/ h E%%&- h $; h  $;h #Tkh D[h   h h T506rrz The RWKV Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). cFeZdZddiZfdZdZdZe ddejdzdejdzd ejdzd e ejdzd ejdzd e dzd e dzde dzde dzde ejzdeezfdZxZS)RwkvForCausalLMz head.weightzrwkv.embeddings.weightct||t||_t j |j |jd|_|jy)NFr) rrrrrrrMrheadr)rrrs rrzRwkvForCausalLM.__init__sH  f% IIf00&2C2C%P  rc|jSr]r<rs rget_output_embeddingsz%RwkvForCausalLM.get_output_embeddingss yyrc||_yr]r>r s rset_output_embeddingsz%RwkvForCausalLM.set_output_embeddingss " rNrrrrIlabelsrrrrlogits_to_keeprc | | n|jj} |j||||||| } | d} t| tr t | dn| }|j | dd|ddf}d}|*|jd|||jjd| }| s|f| ddz}||f|zS|St||| j| j| jS)aJ input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) state (tuple of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`, *optional*): If passed along, the model uses the previous state in all the blocks (which will give the output for the `input_ids` provided as if the model add `state_input_ids + input_ids` as context). labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` use_cache (`bool`, *optional*): If set to `True`, the last state is returned and can be used to quickly generate the next logits. N)rrIrrrrr)rrBrr)rrrIrrrb) rrrrr%slicer< loss_functionrrrIrr)rrrrrIrBrrrrrCr rwkv_outputsr slice_indicesrrrNs rr?zRwkvForCausalLM.forwardsL&1%SV8W~ot4]k =M1)<=>  %4%%pVFt{{OeOepiopDYab!11F)-)9TGf$ Ev E!$$&44#..   r) NNNNNNNNNr)r^r_r`_tied_weights_keysrr?rAr r0r7rrr8r%Tensorrrr?rrs@rr:r:s(()AB#.2262604*.!%)-,0#'-.D ##d*D ((4/D ((4/ D E%%&- D   4' D $;D  $;D #TkD D[D ell*D  # #D D rr:)r:rrr\)-rr dataclassesrr0rrr generationrmodeling_layersrmodeling_utilsr utilsr r r r rrrconfiguration_rwkvr get_loggerr^rrrautogradFunctionrrurrrrrrrrrr:__all__rbrrrVs ! &)9-+   H %5g %..11g T)XbC5 C5L#)bii#)L*DP7/P7P7f   < <  <  <<  <$k6#k6k6\ V )?V  V r Br