
    ^jƛ                        d Z ddlmZ ddlZddlmZ ddlmZ ddlm	Z	 ddl
mZmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZmZ ddlmZ ddlmZ ddlmZmZmZmZ ddl m!Z!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/ ddl0m1Z1m2Z2m3Z3m4Z4m5Z5 ddl6m7Z7  ejp                  e9      Z: G d ded      Z; G d de.      Z<d3dZ=e" G d  d!e*             Z> G d" d#e1      Z? G d$ d%ej                        ZA G d& d'e,      ZB G d( d)e-      ZC G d* d+e(      ZDe G d, d-e             ZEe G d. d/eE             ZF G d0 d1e+      ZGg d2ZHy)4zPyTorch Bamba model.    )	TypedDictN)nn   )initialization)ACT2FN)CacheDynamicCache)force_accelerate_hooks)lazy_load_kernel)create_causal_maskcreate_recurrent_attention_mask)BaseModelOutputWithPastCausalLMOutputWithPast)PreTrainedModel)Unpack)auto_docstringcan_return_tupleis_torchdynamo_compilinglogging)merge_with_config_defaultsno_inherit_decorator)resolve_internal_import)capture_outputs   )JambaAttentionDecoderLayer)LlamaAttentionLlamaForCausalLMLlamaMLPLlamaRMSNormLlamaRotaryEmbeddingrotate_half)MambaRMSNormGatedapply_mask_to_padding_statespad_tensor_by_sizereshape_into_chunkssegment_sum   )BambaConfigc                       e Zd ZU dZej
                  ed<   ej
                  ed<   eed<   eed<   ej                  ed<   y)BambaFlashAttentionKwargsaU  
    Keyword arguments for advanced Flash Attention, causal-conv1d, and mamba_ssm kernel usage.
    Use cases include padding-free training and fewer `torch.compile` graph breaks.

    cu_seq_lens_q (`torch.LongTensor`):
        Gets cumulative sequence length for query state.
    cu_seq_lens_k (`torch.LongTensor`):
        Gets cumulative sequence length for key state.
    max_length_q (`int`):
        Maximum sequence length for query state.
    max_length_k (`int`):
        Maximum sequence length for key state.
    seq_idx (`torch.IntTensor`):
        Index of each packed sequence.
    cu_seq_lens_qcu_seq_lens_kmax_length_qmax_length_kseq_idxN)	__name__
__module____qualname____doc__torch
LongTensor__annotations__int	IntTensor     r/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/bamba/modular_bamba.pyr*   r*   =   s7      ######__r:   r*   F)totalc                       e Zd Zy)BambaRotaryEmbeddingNr0   r1   r2   r9   r:   r;   r>   r>   U       r:   r>   c                 h   |j                  |      }|j                  |      }|j                  d   }| dd|f   | d|df   }}|dd|f   |d|df   }	}||z  t        |      |z  z   }
||z  t        |      |z  z   }t        j                  |
|gd      }
t        j                  ||	gd      }|
|fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Removes the interleaving of cos and sin from GLM

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    .Ndim)	unsqueezeshaper!   r4   cat)qkcossinunsqueeze_dim
rotary_dimq_rotq_passk_rotk_passq_embedk_embeds               r;   apply_rotary_pos_embrT   Z   s    ( --
&C
--
&C 2Jc;J;&'3
+;)<6Ec;J;&'3
+;)<6E s{{51C78Gs{{51C78G ii&)r2Gii&)r2GGr:   c                       e Zd Zy)BambaAttentionNr?   r9   r:   r;   rV   rV      s    r:   rV   c                       e Zd Zy)BambaRMSNormGatedNr?   r9   r:   r;   rX   rX      r@   r:   rX   c            
       <    e Zd ZdZdedef fdZ	 	 	 ddej                  de	dz  dej                  dz  d	ej                  dz  fd
Z	 	 dde	dz  dej                  dz  fdZ ed      	 	 	 dde	dz  dej                  dz  d	ej                  dz  fd       Z xZS )
BambaMixeruP  
    Compute ∆, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    ∆, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)

    The are a few differences between this and Mamba2Mixer:
    - The variable use_precomputed_states is slightly different due to the hybrid cache structure
    - There's a few non-obvious bugs fixed with batching in the slow path that exist in main
    - Some extra variables that our layer doesn't need have been removed
    - We ported most of the refactors in https://github.com/huggingface/transformers/pull/35154, which is (as of Dec 18, 2024) unmerged
    config	layer_idxc           	         t         |           |j                  | _        |j                  | _        |j
                  | _        |j                  | _        t        |j                  | j                  z        | _        || _        |j                  | _        |j                  | _        t"        |j                     | _        |j&                  | _        |j*                  | _        |j.                  | _        |j2                  | _        |j6                  | _        |j:                  | _        |j<                  | _        |j>                  | _        | j                  d| j0                  z  | j                  z  z   | _         tC        jD                  | j@                  | j@                  |j                  | j                  | j@                  | j                  dz
        | _#        | j                  | j@                  z   | j                  z   }tC        jH                  | j                  || j(                        | _%        tC        jL                  tO        jP                  | j                              | _)        tO        jT                  d| j                  dz         }tC        jL                  tO        jV                  |            | _,        t[        | j                  | j,                        | _.        tC        jL                  tO        jP                  | j                              | _/        tC        jH                  | j                  | j                  | j(                        | _0        tc        d      }te        |dd       a3te        |dd       a4tc        d	      }tk        |d
      a6tk        |d      a7tk        |d      a8ts        tl        tn        tp        th        tf        f      a:tt        stv        jy                  d       ntv        jy                  d       |jz                  |   | _>        y )Nr   r'   )in_channelsout_channelsbiaskernel_sizegroupspadding)r`   epszcausal-conv1dcausal_conv1d_updatecausal_conv1d_fnz	mamba-ssmz8ops.triton.selective_state_update.selective_state_update)chained_pathz1ops.triton.ssd_combined.mamba_chunk_scan_combinedz8ops.triton.ssd_combined.mamba_split_conv1d_scan_combineda  The fast path is not available because one of `(selective_state_update, causal_conv1d_fn, causal_conv1d_update)` is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1dzDThe fast path for Bamba will be used when running the model on a GPU)?super__init__mamba_n_heads	num_headshidden_sizemamba_d_statessm_state_sizemamba_d_convconv_kernel_sizer7   mamba_expandintermediate_sizer\   mamba_conv_biasuse_conv_bias
hidden_act
activationr   actmamba_proj_biasuse_biasrms_norm_epslayer_norm_epsilonmamba_n_groupsn_groupsmamba_d_headhead_dimmamba_chunk_size
chunk_sizetime_step_limittime_step_mintime_step_maxconv_dimr   Conv1dconv1dLinearin_proj	Parameterr4   onesdt_biasarangelogA_logrX   normDout_projr   getattrrf   rg   r   selective_state_updatemamba_chunk_scan_combined mamba_split_conv1d_scan_combinedallis_fast_path_availableloggerwarning_oncelayer_types
layer_type)selfr[   r\   projection_sizeAcausal_conv1d	mamba_ssm	__class__s          r;   rj   zBambaMixer.__init__   s,   --!--$22 & 3 3!$V%8%84;K;K%K!L"#33 ++&++,.."("5"5--++ 11%55#11#11..T]]1BTEXEX1XXii''--==))A-
 004==@4>>Qyy
 ||EJJt~~$>? LLDNNQ./\\%))A,/
%d&<&<$BYBYZ	ejj89		$"8"8$:J:JQUQ^Q^_ )9&}6LdS"=2DdK %[1	!8$^"
 %<$W%
! ,C$^,
(
 "%&)0 $"
 &>  fg ,,Y7r:   Nhidden_statescache_paramsattention_maskr/   c                    t        ||      }| j                  |      }|j                  \  }}}| j                  | j                  z  }	|d uxr |j                  | j                        }
|
rL|j                  | j                     j                  d   }|j                  | j                     j                  d   }|
r|dk(  r|j                  d      j                  | j                  | j                  | j                  gd      \  }}}t        || j                   j"                  j                  d      | j                   j$                  | j&                        }t)        j                  || j                  |	|	gd      \  }}}t)        j*                  | j,                  j/                                }|d d d df   d d d d d f   j1                  d| j2                  | j                        j5                  t(        j6                        }|d d d d d f   j1                  dd| j2                        }| j8                  d d d df   j1                  d| j2                        }| j:                  d d d df   j1                  d| j2                        }|j=                  || j                  |j                  d   | j                  z        }|j=                  || j                  |j                  d   | j                  z        }|j=                  || j                  | j2                        }t?        ||||||d |d
      }|j=                  || j                  | j2                  z        }| jA                  ||      }| jC                  |      d d d df   }|S t)        j*                  | j,                  j/                                }| jD                  d	t/        d
      fk(  ri nd| jD                  i}| jF                  r|tI        || j                   j"                  j                  d      | j                   j$                  | j8                  |f| j:                  | jJ                  || j&                  | j@                  j"                  | j@                  jL                  | jB                  j"                  | jB                  j$                  | j2                  | j                  ddd|}|S |j                  | j                  | j                  | j                  gd      \  }}}|jO                  dd      }|
rt)        jP                  |gd      }|XtR        jT                  jW                  || jX                  |j                  d   z
  df      }|j[                  || j                         | j&                  dvr5| j]                  | j!                  |      dd |j                  d   f         }nQt_        || j                   j"                  j                  d      | j                   j$                  | j&                  |      }|
r|d d d d | d f   }|jO                  dd      }t        ||      }t)        j                  || j                  |	|	gd      \  }}}ta        |j=                  ||d| j2                        |||j=                  ||| j                  d      |j=                  ||| j                  d      f| jJ                  | j:                  d |d| j8                  d|
rnd d|\  }}|||jc                  || j                        }|j=                  ||d      }| jA                  ||      }| jC                  |      }|S )Nr   r'   rB   rC   .dtypeT)zr   dt_softplusg        infdt_limitF)r   r   r/   rw   rmsnorm_weightrmsnorm_epsoutproj_weightoutproj_biasheaddimngroupsnorm_before_gatereturn_final_statesr   )siluswish)xweightr`   rw   r/   )r   r   r   r/   r   r   r   initial_states)2r#   r   rF   r~   ro   has_previous_stater\   layersconv_statesrecurrent_statessqueezesplitrs   r   rl   rf   r   r   r`   rw   r4   expr   floatexpandr   tofloat32r   r   viewr   r   r   r   trainingr   r   variance_epsilon	transposerG   r   
functionalpadrq   update_conv_staterx   rg   r   update_recurrent_state)r   r   r   r   r/   projected_states
batch_sizeseq_len_groups_time_state_sizeuse_precomputed_states
conv_staterecurrent_stategatehidden_states_B_CdtBCr   r   r   hidden_states_reshapedoutdt_limit_kwargsr   scan_output	ssm_states                              r;   cuda_kernels_forwardzBambaMixer.cuda_kernels_forward   s    5]NS<<6 "/!4!4
GQ!%1D1D!D!-T!9!ml>]>]^b^l^l>m!%,,T^^<HHKJ*11$..ARRSTUO "gl*:*B*B1*E*K*K''GR +L +'D#R
 !5!""**1-  ! #(++!'')?AWX#M1a 4::++-..A!T3,1d
+222t}}dFYFYZ]]didqdq]rAAq$J&&r2t}}=Bll1dC<077DMMJGq$|$++B>Az4==!''!*2MNAz4==!''!*2MNA%2%7%7
DNNTXTaTa%b"2& M *..z4>>DMM;YZM IImT:M --.q$|<C@ 
{ 4::++-..A$($8$8S%,<O$ObV`bfbvbvUwO }}!56$KK&&..q1KK$$LL ff####'99#3#3 $		 : :#'==#7#7!%!3!3 MM MM%*(-#$ &%p 
E /?.D.D++T]]DNNKQS /E /+'
 %6$?$?1$E!) ).		:?P2QWY(Z%+"$--"3"3)..1B1H1H1LLaP#K !22;O??*;;(,=N1OPSUrWhWnWnoqWrUrPr1s(t%(8+#{{1199!<![[--#'?? ')% *(9!Q	/(J%$5$?$?1$E!$@ARTb$c!&+kk%++-CE[\'#q! *C!&&z7BNFF:wrBFF:wrB*  $ff#(, LL $6L?RV* &*&Y$ (\-E , C CIt~~ ^I)..z7BG"iiT: mmK0
r:   c                    |j                   \  }}}|j                  }t        ||      }| j                  |      }|j	                  | j
                  | j                  | j                  gd      \  }	}
}|
j                  dd      }
|d uxr |j                  | j                        }|r&|j                  | j                     j                  d   }|r|dk(  r|j                  |
| j                        d| j                   d f   }t        j                   || j"                  j$                  j'                  d      z  d      }
| j(                  r|
| j"                  j*                  z   }
| j-                  |
      }
n|rt        j.                  |
gd      }
|Xt0        j2                  j5                  |
| j                  |
j                   d   z
  df      }|j                  || j                         | j-                  | j#                  |
      dd |
j                   d   f         }
|r
|
d| d f   }
|
j                  dd      }
t        |
|      }
t        j                  |
| j
                  | j6                  | j8                  z  | j6                  | j8                  z  gd      \  }}}t        j:                  | j<                  j?                                }|r|dk(  r|j                  | j                     j@                  d   jB                  }|d d dd d f   d d d df   }|j                  dd      jE                  ||j                   d   | jF                        }| jH                  d   jE                  | jH                  j                   d   | jF                        }t        j0                  j2                  jK                  ||jM                  |j                        z         }t        jN                  || jP                  d   | jP                  d         }|d   jE                  | j                  | jF                  | j8                        jM                  t        jR                  	      }t        j:                  |d   |z        jM                  |
      }|jU                  || j6                  d      dd d d f   }|jE                  || j6                  | j                  | j6                  z  |j                   d         jW                         }|jU                  |d|j                   d         }|d   |dd d d f   z  }|jU                  |d| jF                        }||d   z  jM                  |
      }|j                  | j                     j@                  d   |z  |z   }|jY                  || j                        }|jU                  || j6                  d      dd d d f   }|jE                  || j6                  | j                  | j6                  z  |j                   d         jW                         }|jU                  |d|j                   d         }|jM                  |jB                  |j                        }|j[                  || j                  z  | jF                  | j8                        }|j[                  || j                  z  | j8                  d      }t        j\                  ||      }|j[                  || j                  | jF                        }| j^                  d   jE                  | j^                  j                   d   | jF                        }|||z  z   jM                  |j                        }|jU                  |d      d d d df   }nt0        j2                  jK                  || jH                  z         }t        jN                  || jP                  d   | jP                  d         }|jU                  ||d| jF                        j?                         }|jU                  ||d| j8                        j?                         }|jU                  ||d| j8                        j?                         }|ja                  | j                  | j6                  z  d| j                        }|ja                  | j                  | j6                  z  d| j                        }| jb                  || jb                  z  z
  | jb                  z  }| j^                  d   te        ||      z  }||d   z  }|jM                  |j                        |z  }||||fD cg c]  }tg        ||| jb                         c}\  }}}}|ji                  dddd      }t        jj                  |d      } t        j:                  tm        |            }!|d d d d d d d d d d d f   |d d d d d d d d d d d f   z  }"|"j!                  d      }#|#d   |!ji                  ddddd      d   z  }$|$j!                  d      }%|%d   |d d d d d f   z  j!                  d      }&t        j:                  | d d d d d d dd f   | z
        }'||'ji                  dddd      d   z  }(|(dd d d f   |d   z  j!                  d      })|rR|j                  | j                     j@                  d   d d d f   jM                  |)j                  |)jB                        nt        jn                  |)d d d df         }*t        j.                  |*|)gd      })t        j:                  tm        t0        j2                  j5                  | d d d d d d df   d                  }+|+j                  dd      }+|+d   |)d d d d d df   z  j!                  d      },|,d d d df   |,d d df   }-})t        j:                  |       }.|dd d d f   |)d d d d d df   z  }/|.ji                  dddd      }0|/j!                  d      |0d   z  }1|&|1z   }|jU                  |d| j                  | jF                        }||z   }|dkD  r|d d d |d d d d f   }|jU                  ||d      }|-||jY                  |-| j                        }-| jq                  ||	      }2| js                  |2jM                  |            }3|3S c c}w )NrB   rC   r'   r   r   .).N).NNr   device)r   r   )rD   output_sizer      )r   r   )r'   r   ):rF   r   r#   r   r   rs   r   rl   r   r   r\   r   r   r   rq   r4   sumr   r   r   ru   r`   rx   rG   r   r   r   r~   ro   r   r   r   r   r   r   r   r   softplusr   clampr   r   reshape
contiguousr   r   bmmr   repeat_interleaver   r$   r%   permutecumsumr&   
zeros_liker   r   )4r   input_statesr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   cache_devicer   dAdBdBx
ssm_statesssm_states_reshaped
C_reshapedyr   pad_size
D_residualtA_cumsumLG_intermediateGM_intermediateMY_diagdecay_statesB_decaystatesprevious_statesdecay_chunk
new_statesr   state_decay_outC_times_statesstate_decay_out_permutedY_offr   contextualized_statess4                                                       r;   torch_forwardzBambaMixer.torch_forward  s    ".!3!3
GQ"" 4L.Q<<5&6&<&<''GR '= '
# .77!<!-T!9!ml>]>]^b^l^l>m!%,,T^^<HHKJ "gl&889JDNN[\_bfbwbwawax\xyK %		dkk0088;;! !!$58H8H$H! $): ;%$)IIz;L.MSU$V!' mm//%(=(=@Q@W@WXZ@[([]^'_ ..{DNNK $5F)GMjO`OfOfgiOjMjHj)k l%$5cG89n$E! 1 ; ;Aq A89JN[#kk##T]]T5H5H%H$--Z^ZmZmJmn
q! YYtzz'')**!gl'..t~~>OOPQRYYL Aq!GQc\*Ba#**:rxx|T]]SBll9-44T\\5G5G5JDMMZG$$--b7::bhh3G.GHBR!5!5a!8$:N:Nq:QRB/"))$..$--I\I\]``glgtgt`uA))ByMA-.22,2GB
 		*dmmR8dAFAT]]DNNdmm4SUVU\U\]_U`allnA		*b!''"+6AI3a<0B *11*b$--PMi0044L4IC &,,T^^<MMaPSUUX[[J%<<ZXJ 		*dmmR8dAFAT]]DNNdmm4SUVU\U\]_U`allnA		*b!''"+6A $ahhaggFJ",//*t~~2Mt}}^b^q^q"r
T^^ ;T=P=PRSTJ		-z:Az4>>4==AA y!((a$--HA]Q&&**1773A 		*b)!T3,7A ''T\\(9:BR!5!5a!8$:N:Nq:QRB)11*gr4==Y__aM		*gr43F3FGMMOA		*gr43F3FGMMOA##DNNdmm$CX\XfXf#gA##DNNdmm$CX\XfXf#gA'DOO*CCtVH	*-?x-XXJ *ByM9M](()B.A cpqrtuwxay%z\]&9!Xt&W%z"M1a 		!Q1%A||A2.H 		+a.)A q!Qa23a1dAq!8K6LLN""r"*A y\AIIaAq!,DY,OON""r"*A 	l]1a:%>>CCCJF !99XaArsl%;h%FGL,..q"b!<YGGGc4l+mI.FFKKPQKRF * ##DNN3DDQG4PSSZ`ZfZfouo|o|S}%%fQUm4 
 YY8a@F))K0A0A(1aQRTV;BWY_0`$abK%//15K%o61dC9PPUUZ[U\J *1crc6 2Jq"u4EIF $ii1OT1oq!T30GGN'6'>'>q!Q'J$#''+.Fy.QQE A		*b$..$--HAJA!|a'1a'(		*gr2A $)A(??	4>>Z	ii4(
 !%knnU.C D$$I &{s   <r?r   c                    t         rJd| j                  j                  j                  j                  v rt               s| j                  ||||      S |t        d      |j                  }|B|j                  d   dkD  r0|j                  d   dkD  r||d d d d d f   z  j                  |      }| j                  |||      S )Ncudaz\`seq_idx` support requires fast path support. Please install `mamba_ssm` and `causal_conv1d`r'   r   )r   r   r   r   typer   r   NotImplementedErrorr   rF   r   r  )r   r   r   r   r/   kwargsr   s          r;   forwardzBambaMixer.forwardb  s     "f0C0C0J0J0O0O&OXpXr,,]L.Zabb%n  ##%.*>*>q*AA*E.J^J^_`JadeJe*^Aq$J-GGKKERM!!-~NNr:   )NNN)NN)r0   r1   r2   r3   r(   r7   rj   r4   Tensorr   r8   r   r  r
   r  __classcell__r   s   @r;   rZ   rZ      s   \8{ \8s \8B &*.2*.b||b dlb t+	b
 4'bP &*.2	D% dlD% t+	D%N H% &*.2*.O dlO t+	O
 4'O &Or:   rZ   c                       e Zd Zy)BambaMLPNr?   r9   r:   r;   r  r  y  r@   r:   r  c                       e Zd Zy)BambaRMSNormNr?   r9   r:   r;   r  r  }  r@   r:   r  c                   J    e Zd Zddededef fdZ	 	 	 	 	 ddej                  dej                  dz  dej                  dz  d	e
dz  d
edz  deej                  ej                  f   dz  dee   deej                  eej                  ej                  f   dz  f   fdZ xZS )BambaDecoderLayerr[   r\   r   c                     t         |   ||       | `d}|dk(  rt        nd } ||      | _        || _        |dk(  rt        ||      | _        y |dk(  rt        ||      | _        y t        d|      )Nr'   linear_attention)r[   r\   full_attentionzInvalid layer_type: )
ri   rj   	self_attnr  feed_forward
block_typerZ   mambarV   
ValueError)r   r[   r\   r   num_expertsffn_layer_classr   s         r;   rj   zBambaDecoderLayer.__init__  s    +N&1Q&6(D+F3$++#6YGDJ+++FI>DN3J>BCCr:   Nr   r   position_idspast_key_values	use_cacheposition_embeddingsr  returnc           
      2   |}| j                  |      }| j                  dk(  r | j                  d|||d|}d }	n+| j                  dk(  r | j                  d||||||d|\  }}	||z   }|}| j	                  |      }| j                  |      }||z   }|	fS )Nr   )r   r   r   r!  )r   r   r)  r*  r+  r,  r9   )input_layernormr$  r%  r"  pre_ff_layernormr#  )
r   r   r   r)  r*  r+  r,  r  residualself_attn_weightss
             r;   r  zBambaDecoderLayer.forward  s     !,,];??00&DJJ +,- 	M !%__ 00/=t~~ 0+-) /#$70 0,M, !=0 --m<))-8 =0///r:   )r   )NNNFN)r0   r1   r2   r(   r7   strrj   r4   r  r5   r   booltupler   r*   FloatTensorr  r  r  s   @r;   r  r    s    D{ Ds D D( /304(,!&HL(0||(0 t+(0 &&-	(0
 (0 $;(0 #5<<#=>E(0 23(0 
u  %(9(95;L;L(L"MPT"TT	U(0r:   r  c                        e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZeedZ ej"                          fd       Z xZS )BambaPreTrainedModelr[   modelTr  r*  )r   
attentionsc           
      j   t         |   |       t        |t              rt	        j
                  |j                         t	        j                  |j                  t        j                  t        j                  d|j                  dz                      t	        j
                  |j                         y y )Nr'   )ri   _init_weights
isinstancerZ   initones_r   copy_r   r4   r   r   rl   r   )r   moduler   s     r;   r<  z"BambaPreTrainedModel._init_weights  sq    f%fj)JJv~~&JJv||UYYu||Av?O?ORS?S/T%UVJJvxx  *r:   )r0   r1   r2   r(   r6   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_is_stateful_can_compile_fullgraphr  rV   _can_record_outputsr4   no_gradr<  r  r  s   @r;   r8  r8    sg    &*#,-#4"5NL!*$
 U]]_! !r:   r8  c                        e Zd Zdef fdZeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   defd                     Z xZS )
BambaModelr[   c           	      Z   t         |   |       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                        | _        g }t        |j                        D ],  }|j                  t        |||j                  |                . t        j                  |      | _        |j                   | _        t#        |j                  |j$                        | _        t)        |      | _        d| _        | j/                          y )N)r\   r   rd   r[   F)ri   rj   pad_token_idpadding_idx
vocab_sizer   	Embeddingrm   embed_tokensrangenum_hidden_layersappendr  layers_block_type
ModuleListr   _attn_implementationr  r{   final_layernormr>   
rotary_embgradient_checkpointing	post_init)r   r[   decoder_layersir   s       r;   rj   zBambaModel.__init__  s     !.. ++LL):):F<N<NPTP`P`av//0 	rA!!"3FaTZTlTlmnTo"pq	rmmN3$*$?$?!+F,>,>FDWDWX.f=&+#r:   N	input_idsr   r)  r*  inputs_embedsr+  r  r-  c           
      p   |d u |d uz  rt        d      || j                  |      }|}|r|t        | j                        }|=t	        j
                  |j                  d   |j                        j                  d      }t        |x}	t              s)| j                  ||||d}
t        di |
t        di |
d}	| j                  ||      }t        | j                        D ]1  \  }} ||f|	| j                  j                   |      ||||d	|\  }}3 | j#                  |      }t%        ||
      S )Nz:You must specify exactly one of input_ids or inputs_embedsrO  r'   r   r   )r[   rb  r   r*  r)  )r!  r   )r)  )r   r)  r*  r+  r,  )last_hidden_stater*  r9   )r&  rT  r	   r[   r4   r   rF   r   rE   r=  dictr   r   r\  	enumerater   rX  r[  r   )r   ra  r   r)  r*  rb  r+  r  r   causal_mask_mappingmask_kwargsr,  r`  decoder_layerattn_weightss                  r;   r  zBambaModel.forward  sn    -t";<YZZ  --i8M%0*$++>O <<(;(;A(>}G[G[\ffghiL?-F ++!."0#2 ,K #5"C{"C$C$Rk$R# #oom,oW )$++ 6 		A}*7+24;;3P3PQR3ST) /#$7+ +'M<		 ,,];&++
 	
r:   )NNNNNN)r0   r1   r2   r(   rj   r   r   r   r4   r5   r  r   r6  r4  r   r*   r   r  r  r  s   @r;   rM  rM    s    { &   .2.204(,26!%7
##d*7
 t+7
 &&-	7

 7
 ((4/7
 $;7
 237
 
!7
    7
r:   rM  c                   0    e Zd Z fdZee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de	dz  dej                  dz  dej                  dz  d	edz  d
eej                  z  defd              Z	 	 	 	 	 	 d fd	Z xZS )BambaForCausalLMc                 f    t         |   |       |j                  | _        | j                          y )N)ri   rj   z_loss_coefficientr^  )r   r[   r   s     r;   rj   zBambaForCausalLM.__init__*  s*     "(";"; 	r:   Nra  r   r)  r*  rb  labelsr+  logits_to_keepr-  c	           
      L    | j                   d
||||||d|	}
|
j                  }t        |t              rt	        | d      n|}| j                  |dd|ddf         }d}| | j                  d
||| j                  j                  d|	}| j                  dkD  r[|j                  d      j                  |j                        j                  d      j                         }|| j                  |z  z   }t        |||
j                   |
j"                  |
j$                  	      S )aJ  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, BambaForCausalLM

        >>> model = BambaForCausalLM.from_pretrained("...")
        >>> tokenizer = AutoTokenizer.from_pretrained("...")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)ra  r   r)  r*  rb  r+  N)logitsro  rR  r   rB   rC   r   r   )lossrr  r*  r   r:  r9   )r9  rd  r=  r7   slicelm_headloss_functionr[   rR  rn  	logsumexpr   r   powmeanr   r*  r   r:  )r   ra  r   r)  r*  rb  ro  r+  rp  r  outputsr   slice_indicesrr  rs  z_losss                   r;   r  zBambaForCausalLM.forward1  s6   H ,64:: ,
)%+',
 ,
  118B>SV8W~ot4]kmA}a,?@A%4%%pVFt{{OeOepiopD&&*))b)1444::4FJJ1MRRTd55>>%#33!//))
 	
r:   c           
      h    | j                   j                  |d<   t        
|   |f||||||d|}	|	S )Nrp  )r*  r   rb  r)  r+  is_first_iteration)r[   num_logits_to_keepri   prepare_inputs_for_generation)r   ra  r*  r   rb  r)  r+  r~  r  model_inputsr   s             r;   r  z.BambaForCausalLM.prepare_inputs_for_generationr  sU     $(;;#A#A w<	
+)'%1	
 	
 r:   )NNNNNNNr   )NNNNTF)r0   r1   r2   rj   r   r   r4   r5   r  r   r6  r4  r7   r   r  r  r  r  s   @r;   rl  rl  )  s      .2.204(,26*.!%-.=
##d*=
 t+=
 &&-	=

 =
 ((4/=
   4'=
 $;=
 ell*=
 
 =
  =
D   r:   rl  )rM  rl  r8  )r'   )Ir3   typingr   r4   r    r   r>  activationsr   cache_utilsr   r	   integrations.accelerater
   integrations.hub_kernelsr   masking_utilsr   r   modeling_outputsr   r   modeling_utilsr   processing_utilsr   utilsr   r   r   r   utils.genericr   r   utils.import_utilsr   utils.output_capturingr   jamba.modeling_jambar   llama.modeling_llamar   r   r   r   r    r!   mamba2.modeling_mamba2r"   r#   r$   r%   r&   configuration_bambar(   
get_loggerr0   r   r*   r>   rT   rV   rX   ModulerZ   r  r  r  r8  rM  rl  __all__r9   r:   r;   <module>r     sD  &     & ! . = 8 P O - & X X M 9 5 =   - 
		H	%	 0	/ 	
#L 	^ 	 		) 	
lO lO^	x 		< 	:02 :0z !? ! !0 N
% N
 N
b`' `F Er:   