
    ^j                     4   d dl mZ d dlmZmZ d dlZd dlmZ ddlmZ	 ddl
mZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZ ddlmZ ddlmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3 ddl4m5Z5  e,jl                  e7      Z8 G d ded      Z9 G d dejt                        Z;d Z<d ejz                  d!e>d"ejz                  fd#Z?	 dHd$ejt                  d%ejz                  d&ejz                  d'ejz                  d(ejz                  dz  d)e@d*e@d+e&e(   fd,ZAdId-ZB G d. d/ejt                        ZC G d0 d1ej                  jt                        ZDd2ejz                  d3e>fd4ZEd5 ZFd6 ZGd7 ZH G d8 d9ejt                        ZI G d: d;ejt                        ZJ ed<       G d= d>ejt                               ZK G d? d@e      ZLe) G dA dBe$             ZMe) G dC dDeM             ZNe) G dE dFeMe             ZOg dGZPy)J    )Callable)Optional	TypedDictN)nn   )initialization)ACT2FN)CacheDynamicCache)GenerationMixin)use_kernel_forward_from_hub)force_accelerate_hooks)lazy_load_kernel)create_causal_maskcreate_recurrent_attention_mask)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tupleis_torchdynamo_compilinglogging)maybe_autocastmerge_with_config_defaults)resolve_internal_import)capture_outputs   )BambaConfigc                       e Zd ZU dZej
                  ed<   ej
                  ed<   eed<   eed<   ej                  ed<   y)BambaFlashAttentionKwargsaU  
    Keyword arguments for advanced Flash Attention, causal-conv1d, and mamba_ssm kernel usage.
    Use cases include padding-free training and fewer `torch.compile` graph breaks.

    cu_seq_lens_q (`torch.LongTensor`):
        Gets cumulative sequence length for query state.
    cu_seq_lens_k (`torch.LongTensor`):
        Gets cumulative sequence length for key state.
    max_length_q (`int`):
        Maximum sequence length for query state.
    max_length_k (`int`):
        Maximum sequence length for key state.
    seq_idx (`torch.IntTensor`):
        Index of each packed sequence.
    cu_seq_lens_qcu_seq_lens_kmax_length_qmax_length_kseq_idxN)	__name__
__module____qualname____doc__torch
LongTensor__annotations__int	IntTensor     s/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/bamba/modeling_bamba.pyr&   r&   7   s7      ######__r6   r&   F)totalc                        e Zd ZU ej                  ed<   ddef fdZe	 	 	 ddedz  de	d   de
dz  ded	ef   fd
       Z ej                         ed               Z xZS )BambaRotaryEmbeddinginv_freqNconfigc                    t         |           |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |      \  }| _
        | j                  d|d       | j                  d|j                         d       y )N	rope_typedefaultr;   F)
persistentoriginal_inv_freq)super__init__max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr<   rope_parametersr>   compute_default_rope_parametersr   attention_scalingregister_bufferclone)selfr<   devicerope_init_fnr;   	__class__s        r7   rC   zBambaRotaryEmbedding.__init__R   s    "("@"@$*$B$B!44[A!%!E!E>>Y&.t~~>L+7V+L($(ZeD0(..2BuUr6   rM   ztorch.deviceseq_lenreturnztorch.Tensorc                    | j                   d   }t        | dd      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                        j                  |t        j                        |z  z  z  }||fS )	a  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        
rope_thetahead_dimNg      ?r      dtyperM   rW   )	rG   getattrhidden_sizenum_attention_headsr0   arangeint64tofloat)r<   rM   rP   basedimattention_factorr;   s          r7   rH   z4BambaRotaryEmbedding.compute_default_rope_parametersb   s    & %%l3fj$/c63E3EIcIc3c U\\!S!5;;?BB&X]XcXcBdgjjk
 )))r6   c                 N   | j                   d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d      5  |j                         |j                         z  j                  dd      }t        j                  ||fd	      }|j                         | j                  z  }|j                         | j                  z  }	d d d        j	                  |j                   
      	j	                  |j                   
      fS # 1 sw Y   AxY w)Nr   r#   mpscpuF)device_typeenabledrU   ra   rV   )r;   r_   expandshaper^   rM   
isinstancetypestrr   	transposer0   catcosrI   sinrW   )
rL   xposition_idsinv_freq_expandedposition_ids_expandedrg   freqsembrq   rr   s
             r7   forwardzBambaRotaryEmbedding.forward   sR    !MM$4-8>>@GGHZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfkUC 	5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C		5 vvAGGv$cff177f&;;;	5 	5s   BFF$NNNN)r,   r-   r.   r0   Tensorr2   r$   rC   staticmethodr   r3   tupler_   rH   no_gradr   ry   __classcell__rO   s   @r7   r:   r:   O   s    llV{ V  %)+/"*d"*(* t* 
~u$	%	* *: U]]_<  <r6   r:   c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..Nrd   rU   ri   )rk   r0   rp   )rs   x1x2s      r7   rotate_halfr      sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r6   hidden_statesn_reprQ   c                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r#   N)rk   rj   reshape)r   r   batchnum_key_value_headsslenrT   s         r7   	repeat_kvr      so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TTr6   modulequerykeyvalueattention_maskscalingdropoutkwargsc                    t        || j                        }t        || j                        }	t        j                  ||j	                  dd            |z  }
||
|z   }
t
        j                  j                  |
dt        j                        j                  |j                        }
t
        j                  j                  |
|| j                        }
t        j                  |
|	      }|j	                  dd      j                         }||
fS )NrU   r   rd   )ra   rW   )ptrainingr#   )r   num_key_value_groupsr0   matmulro   r   
functionalsoftmaxfloat32r^   rW   r   r   
contiguous)r   r   r   r   r   r   r   r   
key_statesvalue_statesattn_weightsattn_outputs               r7   eager_attention_forwardr      s     3 ; ;<JUF$?$?@L<<z';';Aq'ABWLL!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|\:K''1-88:K$$r6   c                 h   |j                  |      }|j                  |      }|j                  d   }| dd|f   | d|df   }}|dd|f   |d|df   }	}||z  t        |      |z  z   }
||z  t        |      |z  z   }t        j                  |
|gd      }
t        j                  ||	gd      }|
|fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Removes the interleaving of cos and sin from GLM

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    rd   .Nri   )	unsqueezerk   r   r0   rp   )qkrq   rr   unsqueeze_dim
rotary_dimq_rotq_passk_rotk_passq_embedk_embeds               r7   apply_rotary_pos_embr      s    ( --
&C
--
&C 2Jc;J;&'3
+;)<6Ec;J;&'3
+;)<6E s{{51C78Gs{{51C78G ii&)r2Gii&)r2GGr6   c                        e Zd ZdZdedef fdZ	 	 	 ddej                  de	ej                  ej                  f   dz  dej                  dz  d	e
dz  d
ee   de	ej                  ej                  f   fdZ xZS )BambaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr<   	layer_idxc                 d   t         |           || _        || _        t	        |d|j
                  |j                  z        | _        |j                  |j                  z  | _	        | j                  dz  | _
        |j                  | _        d| _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j                  | j                  z  |j
                  |j                        | _        y )NrT   g      Tbias)rB   rC   r<   r   rY   rZ   r[   rT   r   r   r   attention_dropout	is_causalr   Linearattention_biasq_projk_projv_projo_proj)rL   r<   r   rO   s      r7   rC   zBambaAttention.__init__   sM   "
F4F4F&JdJd4de$*$>$>&B\B\$\!}}d*!'!9!9ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii&&68J8JQWQfQf
r6   Nr   position_embeddingsr   past_key_valuesr   rQ   c                 
   |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }	| j                  |      j                  |      j	                  dd      }
|\  }}t        ||	||      \  }}	| |j                  |	|
| j                        \  }	}
t        j                  | j                  j                  t              } || ||	|
|f| j                  sdn| j                   | j"                  d|\  }} |j$                  g |d j'                         }| j)                  |      }||fS )Nrd   r#   rU           )r   r   )rk   rT   r   viewro   r   r   r   updater   r   get_interfacer<   _attn_implementationr   r   r   r   r   r   r   )rL   r   r   r   r   r   input_shapehidden_shapequery_statesr   r   rq   rr   attention_interfacer   r   s                   r7   ry   zBambaAttention.forward   s    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST&S#7jRUWZ#[ j&'6'='=j,X\XfXf'g$J(?(M(MKK,,.E)
 %8	%
  $}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((r6   r{   )r,   r-   r.   r/   r$   r3   rC   r0   r|   r~   r
   r   r   ry   r   r   s   @r7   r   r      s    G
{ 
s 
4 IM.2(,&)||&) #5<<#=>E&) t+	&)
 &) +,&) 
u||U\\)	*&)r6   r   c                   (     e Zd Zd fd	ZddZ xZS )BambaRMSNormGatedc                     t         |           t        j                  t	        j
                  |            | _        || _        y rz   rB   rC   r   	Parameterr0   onesweightvariance_epsilonrL   rZ   epsrO   s      r7   rC   zBambaRMSNormGated.__init__'  s/    ll5::k#:; #r6   c                    |j                   }|j                  t        j                        }|?|t        j
                  j                  |j                  t        j                              z  }|j                  d      j                  dd      }|t        j                  || j                  z         z  }| j                  |j                  |      z  S NrU   rd   T)keepdim)rW   r^   r0   r   r   r   silupowmeanrsqrtr   r   )rL   r   gateinput_dtypevariances        r7   ry   zBambaRMSNormGated.forward,  s    #))%((7)BMM,>,>twwu}}?U,VVM $$Q',,R,>%Ht?T?T4T(UU{{]--k:::r6   gư>rz   r,   r-   r.   rC   ry   r   r   s   @r7   r   r   &  s    $
	;r6   r   input_tensorpad_sizec                     t        | j                        dk(  r
ddddd|ddfnddd|ddf}t        j                  j                  j                  | |dd      S )z
    Padding x tensor with `pad_size` on the seq_len dim (dim=1)

    Assumes that we only have tensors of either size 4 or 3
       r   constant)moder   )lenrk   r0   r   r   pad)r   r   	pad_shapes      r7   pad_tensor_by_sizer   ;  sf     47|7I7I3Ja3OAq!Q!Q/VWYZ\]_gijlmUnI88""<ST"UUr6   c                    t        | |      } t        | j                        dk(  r.| j                  | j                  d   d|| j                  d         S | j                  | j                  d   d|| j                  d   | j                  d         S )z
    Padding input_tensor with `pad_size` on the seq_len dim (dim=1) and
    simultaneously splitting it into chunk sequences.

    Assumes that we only have tensors of either size 4 or 3
    r   r   rd   rU   )r   r   rk   r   )r   r   
chunk_sizes      r7   reshape_into_chunksr   F  s     &lH=L
<!###L$6$6q$92z<K]K]^_K`aa ##q!2z<3E3Ea3H,J\J\]^J_
 	
r6   c                 "   | j                  d      } | d   j                  g | j                         | } t        j                  t        j                  ||| j
                  t        j                        d      }| j                  | d      } t        j                  | d      }t        j                  t        j                  ||| j
                  t        j                        d      }|j                  | t        j                         }|S )zo
    More stable segment sum calculation. Uses cumulative sums and masking instead of direct subtractions.
    rd   .NrX   )diagonalr   ri   )
sizerj   r0   trilr   rM   boolmasked_fillcumsuminf)r   r   masktensor_segsums       r7   segment_sumr   Z  s     ""2&J 2<	*11S<3D3D3FS
SL::ejjZ@S@S[`[e[efqstD++TE15LLL26M ::ejjZ@S@S[`[e[efqrsD!--teeiiZ@Mr6   c                     |N|j                   d   dkD  r<|j                   d   dkD  r*| j                  }| |dddddf   z  j                  |      } | S )zm
    Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
    Nr#   r   )rk   rW   r^   )r   r   rW   s      r7   apply_mask_to_padding_statesr   n  sa    
 !n&:&:1&=&AnFZFZ[\F]`aFa##&1d
)CCGGNr6   c            
       <    e Zd ZdZdedef fdZ	 	 	 ddej                  de	dz  dej                  dz  d	ej                  dz  fd
Z	 	 dde	dz  dej                  dz  fdZ ed      	 	 	 dde	dz  dej                  dz  d	ej                  dz  fd       Z xZS )
BambaMixeruP  
    Compute ∆, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    ∆, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)

    The are a few differences between this and Mamba2Mixer:
    - The variable use_precomputed_states is slightly different due to the hybrid cache structure
    - There's a few non-obvious bugs fixed with batching in the slow path that exist in main
    - Some extra variables that our layer doesn't need have been removed
    - We ported most of the refactors in https://github.com/huggingface/transformers/pull/35154, which is (as of Dec 18, 2024) unmerged
    r<   r   c           	         t         |           |j                  | _        |j                  | _        |j
                  | _        |j                  | _        t        |j                  | j                  z        | _        || _        |j                  | _        |j                  | _        t"        |j                     | _        |j&                  | _        |j*                  | _        |j.                  | _        |j2                  | _        |j6                  | _        |j:                  | _        |j<                  | _        |j>                  | _        | j                  d| j0                  z  | j                  z  z   | _         tC        jD                  | j@                  | j@                  |j                  | j                  | j@                  | j                  dz
        | _#        | j                  | j@                  z   | j                  z   }tC        jH                  | j                  || j(                        | _%        tC        jL                  tO        jP                  | j                              | _)        tO        jT                  d| j                  dz         }tC        jL                  tO        jV                  |            | _,        t[        | j                  | j,                        | _.        tC        jL                  tO        jP                  | j                              | _/        tC        jH                  | j                  | j                  | j(                        | _0        tc        d      }te        |dd       a3te        |dd       a4tc        d	      }tk        |d
      a6tk        |d      a7tk        |d      a8ts        tl        tn        tp        th        tf        f      a:tt        stv        jy                  d       ntv        jy                  d       |jz                  |   | _>        y )NrU   r#   )in_channelsout_channelsr   kernel_sizegroupspaddingr   r   zcausal-conv1dcausal_conv1d_updatecausal_conv1d_fnz	mamba-ssmz8ops.triton.selective_state_update.selective_state_update)chained_pathz1ops.triton.ssd_combined.mamba_chunk_scan_combinedz8ops.triton.ssd_combined.mamba_split_conv1d_scan_combineda  The fast path is not available because one of `(selective_state_update, causal_conv1d_fn, causal_conv1d_update)` is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1dzDThe fast path for Bamba will be used when running the model on a GPU)?rB   rC   mamba_n_heads	num_headsrZ   mamba_d_statessm_state_sizemamba_d_convconv_kernel_sizer3   mamba_expandintermediate_sizer   mamba_conv_biasuse_conv_bias
hidden_act
activationr	   actmamba_proj_biasuse_biasrms_norm_epslayer_norm_epsilonmamba_n_groupsn_groupsmamba_d_headrT   mamba_chunk_sizer   time_step_limittime_step_mintime_step_maxconv_dimr   Conv1dconv1dr   in_projr   r0   r   dt_biasr\   logA_logr   normDout_projr   rY   r  r  r!   selective_state_updatemamba_chunk_scan_combined mamba_split_conv1d_scan_combinedallis_fast_path_availableloggerwarning_oncelayer_types
layer_type)rL   r<   r   projection_sizeAcausal_conv1d	mamba_ssmrO   s          r7   rC   zBambaMixer.__init__  s,   --!--$22 & 3 3!$V%8%84;K;K%K!L"#33 ++&++,.."("5"5--++ 11%55#11#11..T]]1BTEXEX1XXii''--==))A-
 004==@4>>Qyy
 ||EJJt~~$>? LLDNNQ./\\%))A,/
%d&<&<$BYBYZ	ejj89		$"8"8$:J:JQUQ^Q^_ )9&}6LdS"=2DdK %[1	!8$^"
 %<$W%
! ,C$^,
(
 "%&)0 $"
 &>  fg ,,Y7r6   Nr   cache_paramsr   r+   c                    t        ||      }| j                  |      }|j                  \  }}}| j                  | j                  z  }	|d uxr |j                  | j                        }
|
rL|j                  | j                     j                  d   }|j                  | j                     j                  d   }|
r|dk(  r|j                  d      j                  | j                  | j                  | j                  gd      \  }}}t        || j                   j"                  j                  d      | j                   j$                  | j&                        }t)        j                  || j                  |	|	gd      \  }}}t)        j*                  | j,                  j/                                }|d d d df   d d d d d f   j1                  d| j2                  | j                        j5                  t(        j6                        }|d d d d d f   j1                  dd| j2                        }| j8                  d d d df   j1                  d| j2                        }| j:                  d d d df   j1                  d| j2                        }|j=                  || j                  |j                  d   | j                  z        }|j=                  || j                  |j                  d   | j                  z        }|j=                  || j                  | j2                        }t?        ||||||d |d
      }|j=                  || j                  | j2                  z        }| jA                  ||      }| jC                  |      d d d df   }|S t)        j*                  | j,                  j/                                }| jD                  d	t/        d
      fk(  ri nd| jD                  i}| jF                  r|tI        || j                   j"                  j                  d      | j                   j$                  | j8                  |f| j:                  | jJ                  || j&                  | j@                  j"                  | j@                  jL                  | jB                  j"                  | jB                  j$                  | j2                  | j                  ddd|}|S |j                  | j                  | j                  | j                  gd      \  }}}|jO                  dd      }|
rt)        jP                  |gd      }|XtR        jT                  jW                  || jX                  |j                  d   z
  df      }|j[                  || j                         | j&                  dvr5| j]                  | j!                  |      dd |j                  d   f         }nQt_        || j                   j"                  j                  d      | j                   j$                  | j&                  |      }|
r|d d d d | d f   }|jO                  dd      }t        ||      }t)        j                  || j                  |	|	gd      \  }}}ta        |j=                  ||d| j2                        |||j=                  ||| j                  d      |j=                  ||| j                  d      f| jJ                  | j:                  d |d| j8                  d|
rnd d|\  }}|||jc                  || j                        }|j=                  ||d      }| jA                  ||      }| jC                  |      }|S )Nr   r#   rd   ri   .rV   T)zr%  dt_softplusr   r   dt_limitF)r)  r   r+   r  rmsnorm_weightrmsnorm_epsoutproj_weightoutproj_biasheaddimngroupsnorm_before_gatereturn_final_statesrU   )r   swish)rs   r   r   r  r+   )r   r)  r:  r+   rD  r%  r;  initial_states)2r   r$  rk   r  r  has_previous_stater   layersconv_statesrecurrent_statessqueezesplitr  r!  r
  r  r#  r   r   r  r0   expr'  r_   rj   rT   r^   r   r%  r)  r   r+  r(  r*  r  r   r-  r   r   ro   rp   r   r   r   r  update_conv_stater  r  r,  update_recurrent_state)rL   r   r8  r   r+   projected_states
batch_sizerP   _groups_time_state_sizeuse_precomputed_states
conv_staterecurrent_stater   hidden_states_B_CdtBCr5  r%  r)  hidden_states_reshapedoutdt_limit_kwargsrI  scan_output	ssm_states                              r7   cuda_kernels_forwardzBambaMixer.cuda_kernels_forward  s    5]NS<<6 "/!4!4
GQ!%1D1D!D!-T!9!ml>]>]^b^l^l>m!%,,T^^<HHKJ*11$..ARRSTUO "gl*:*B*B1*E*K*K''GR +L +'D#R
 !5!""**1-  ! #(++!'')?AWX#M1a 4::++-..A!T3,1d
+222t}}dFYFYZ]]didqdq]rAAq$J&&r2t}}=Bll1dC<077DMMJGq$|$++B>Az4==!''!*2MNAz4==!''!*2MNA%2%7%7
DNNTXTaTa%b"2& M *..z4>>DMM;YZM IImT:M --.q$|<C@ 
{ 4::++-..A$($8$8S%,<O$ObV`bfbvbvUwO }}!56$KK&&..q1KK$$LL ff####'99#3#3 $		 : :#'==#7#7!%!3!3 MM MM%*(-#$ &%p 
E /?.D.D++T]]DNNKQS /E /+'
 %6$?$?1$E!) ).		:?P2QWY(Z%+"$--"3"3)..1B1H1H1LLaP#K !22;O??*;;(,=N1OPSUrWhWnWnoqWrUrPr1s(t%(8+#{{1199!<![[--#'?? ')% *(9!Q	/(J%$5$?$?1$E!$@ARTb$c!&+kk%++-CE[\'#q! *C!&&z7BNFF:wrBFF:wrB*  $ff#(, LL $6L?RV* &*&Y$ (\-E , C CIt~~ ^I)..z7BG"iiT: mmK0
r6   c                    |j                   \  }}}|j                  }t        ||      }| j                  |      }|j	                  | j
                  | j                  | j                  gd      \  }	}
}|
j                  dd      }
|d uxr |j                  | j                        }|r&|j                  | j                     j                  d   }|r|dk(  r|j                  |
| j                        d| j                   d f   }t        j                   || j"                  j$                  j'                  d      z  d      }
| j(                  r|
| j"                  j*                  z   }
| j-                  |
      }
n|rt        j.                  |
gd      }
|Xt0        j2                  j5                  |
| j                  |
j                   d   z
  df      }|j                  || j                         | j-                  | j#                  |
      dd |
j                   d   f         }
|r
|
d| d f   }
|
j                  dd      }
t        |
|      }
t        j                  |
| j
                  | j6                  | j8                  z  | j6                  | j8                  z  gd      \  }}}t        j:                  | j<                  j?                                }|r|dk(  r|j                  | j                     j@                  d   jB                  }|d d dd d f   d d d df   }|j                  dd      jE                  ||j                   d   | jF                        }| jH                  d   jE                  | jH                  j                   d   | jF                        }t        j0                  j2                  jK                  ||jM                  |j                        z         }t        jN                  || jP                  d   | jP                  d         }|d   jE                  | j                  | jF                  | j8                        jM                  t        jR                  	      }t        j:                  |d   |z        jM                  |
      }|jU                  || j6                  d      dd d d f   }|jE                  || j6                  | j                  | j6                  z  |j                   d         jW                         }|jU                  |d|j                   d         }|d   |dd d d f   z  }|jU                  |d| jF                        }||d   z  jM                  |
      }|j                  | j                     j@                  d   |z  |z   }|jY                  || j                        }|jU                  || j6                  d      dd d d f   }|jE                  || j6                  | j                  | j6                  z  |j                   d         jW                         }|jU                  |d|j                   d         }|jM                  |jB                  |j                        }|j[                  || j                  z  | jF                  | j8                        }|j[                  || j                  z  | j8                  d      }t        j\                  ||      }|j[                  || j                  | jF                        }| j^                  d   jE                  | j^                  j                   d   | jF                        }|||z  z   jM                  |j                        }|jU                  |d      d d d df   }nt0        j2                  jK                  || jH                  z         }t        jN                  || jP                  d   | jP                  d         }|jU                  ||d| jF                        j?                         }|jU                  ||d| j8                        j?                         }|jU                  ||d| j8                        j?                         }|ja                  | j                  | j6                  z  d| j                        }|ja                  | j                  | j6                  z  d| j                        }| jb                  || jb                  z  z
  | jb                  z  }| j^                  d   te        ||      z  }||d   z  }|jM                  |j                        |z  }||||fD cg c]  }tg        ||| jb                         c}\  }}}}|ji                  dddd      }t        jj                  |d      } t        j:                  tm        |            }!|d d d d d d d d d d d f   |d d d d d d d d d d d f   z  }"|"j!                  d      }#|#d   |!ji                  ddddd      d   z  }$|$j!                  d      }%|%d   |d d d d d f   z  j!                  d      }&t        j:                  | d d d d d d dd f   | z
        }'||'ji                  dddd      d   z  }(|(dd d d f   |d   z  j!                  d      })|rR|j                  | j                     j@                  d   d d d f   jM                  |)j                  |)jB                        nt        jn                  |)d d d df         }*t        j.                  |*|)gd      })t        j:                  tm        t0        j2                  j5                  | d d d d d d df   d                  }+|+j                  dd      }+|+d   |)d d d d d df   z  j!                  d      },|,d d d df   |,d d df   }-})t        j:                  |       }.|dd d d f   |)d d d d d df   z  }/|.ji                  dddd      }0|/j!                  d      |0d   z  }1|&|1z   }|jU                  |d| j                  | jF                        }||z   }|dkD  r|d d d |d d d d f   }|jU                  ||d      }|-||jY                  |-| j                        }-| jq                  ||	      }2| js                  |2jM                  |            }3|3S c c}w )Nrd   ri   r#   rU   r   .r   ).NNrV   rM   rX   )ra   output_sizer   r   r   )rW   rM   )r#   r   ):rk   rW   r   r$  rL  r  r!  r
  ro   rG  r   rH  rI  rN  r  r0   sumr#  r   rK  r  r   r  rp   r   r   r   r  r  rM  r'  r_   rJ  rM   rj   rT   r%  softplusr^   clampr  r   r   r   rO  r   bmmr)  repeat_interleaver   r   r   permuter   r   
zeros_liker(  r*  )4rL   input_statesr8  r   rQ  rP   rR  rW   rP  r   rW  rX  rT  rU  rI  r   rY  rZ  r5  cache_devicer%  dAdBdBx
ssm_statesssm_states_reshaped
C_reshapedyr)  r   
D_residualtA_cumsumLG_intermediateGM_intermediateMY_diagdecay_statesB_decaystatesprevious_statesdecay_chunk
new_statesr_  state_decay_outC_times_statesstate_decay_out_permutedY_offr^  contextualized_statess4                                                       r7   torch_forwardzBambaMixer.torch_forward  s    ".!3!3
GQ"" 4L.Q<<5&6&<&<''GR '= '
# .77!<!-T!9!ml>]>]^b^l^l>m!%,,T^^<HHKJ "gl&889JDNN[\_bfbwbwawax\xyK %		dkk0088;;! !!$58H8H$H! $): ;%$)IIz;L.MSU$V!' mm//%(=(=@Q@W@WXZ@[([]^'_ ..{DNNK $5F)GMjO`OfOfgiOjMjHj)k l%$5cG89n$E! 1 ; ;Aq A89JN[#kk##T]]T5H5H%H$--Z^ZmZmJmn
q! YYtzz'')**!gl'..t~~>OOPQRYYL Aq!GQc\*Ba#**:rxx|T]]SBll9-44T\\5G5G5JDMMZG$$--b7::bhh3G.GHBR!5!5a!8$:N:Nq:QRB/"))$..$--I\I\]``glgtgt`uA))ByMA-.22,2GB
 		*dmmR8dAFAT]]DNNdmm4SUVU\U\]_U`allnA		*b!''"+6AI3a<0B *11*b$--PMi0044L4IC &,,T^^<MMaPSUUX[[J%<<ZXJ 		*dmmR8dAFAT]]DNNdmm4SUVU\U\]_U`allnA		*b!''"+6A $ahhaggFJ",//*t~~2Mt}}^b^q^q"r
T^^ ;T=P=PRSTJ		-z:Az4>>4==AA y!((a$--HA]Q&&**1773A 		*b)!T3,7A ''T\\(9:BR!5!5a!8$:N:Nq:QRB)11*gr4==Y__aM		*gr43F3FGMMOA		*gr43F3FGMMOA##DNNdmm$CX\XfXf#gA##DNNdmm$CX\XfXf#gA'DOO*CCtVH	*-?x-XXJ *ByM9M](()B.A cpqrtuwxay%z\]&9!Xt&W%z"M1a 		!Q1%A||A2.H 		+a.)A q!Qa23a1dAq!8K6LLN""r"*A y\AIIaAq!,DY,OON""r"*A 	l]1a:%>>CCCJF !99XaArsl%;h%FGL,..q"b!<YGGGc4l+mI.FFKKPQKRF * ##DNN3DDQG4PSSZ`ZfZfouo|o|S}%%fQUm4 
 YY8a@F))K0A0A(1aQRTV;BWY_0`$abK%//15K%o61dC9PPUUZ[U\J *1crc6 2Jq"u4EIF $ii1OT1oq!T30GGN'6'>'>q!Q'J$#''+.Fy.QQE A		*b$..$--HAJA!|a'1a'(		*gr2A $)A(??	4>>Z	ii4(
 !%knnU.C D$$I &{s   <r?r#  c                    t         rJd| j                  j                  j                  j                  v rt               s| j                  ||||      S |t        d      |j                  }|B|j                  d   dkD  r0|j                  d   dkD  r||d d d d d f   z  j                  |      }| j                  |||      S )Ncudaz\`seq_idx` support requires fast path support. Please install `mamba_ssm` and `causal_conv1d`r#   r   )r/  r$  r   rM   rm   r   r`  NotImplementedErrorrW   rk   r^   r  )rL   r   r8  r   r+   r   rW   s          r7   ry   zBambaMixer.forwardS  s     "f0C0C0J0J0O0O&OXpXr,,]L.Zabb%n  ##%.*>*>q*AA*E.J^J^_`JadeJe*^Aq$J-GGKKERM!!-~NNr6   r{   )NN)r,   r-   r.   r/   r$   r3   rC   r0   r|   r
   r4   r`  r  r   ry   r   r   s   @r7   r   r   {  s   \8{ \8s \8B &*.2*.b||b dlb t+	b
 4'bP &*.2	D% dlD% t+	D%N H% &*.2*.O dlO t+	O
 4'O &Or6   r   c                   $     e Zd Z fdZd Z xZS )BambaMLPc                    t         |           || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  |j                        | _        t        j                  | j                  | j                  |j                        | _	        t        j                  | j                  | j                  |j                        | _
        t        |j                     | _        y )Nr   )rB   rC   r<   rZ   r  r   r   mlp_bias	gate_projup_proj	down_projr	   r  act_fnrL   r<   rO   s     r7   rC   zBambaMLP.__init__k  s    !--!'!9!94#3#3T5K5KRXRaRabyy!1!143I3IPVP_P_`4#9#94;K;KRXRaRabV../r6   c                     | j                  | j                  | j                  |            | j                  |      z        }|S rz   )r  r  r  r  )rL   rs   r  s      r7   ry   zBambaMLP.forwardu  s6    NN4;;t~~a/@#ADLLQRO#ST	r6   r   r   s   @r7   r  r  j  s    0r6   r  RMSNormc                   h     e Zd Zddeddf fdZdej                  dej                  fdZd Z xZ	S )	BambaRMSNormr   rQ   Nc                     t         |           t        j                  t	        j
                  |            | _        || _        y)z;
        BambaRMSNorm is equivalent to T5LayerNorm
        Nr   r   s      r7   rC   zBambaRMSNorm.__init__|  s1     	ll5::k#:; #r6   r   c                 "   |j                   }|j                  t        j                        }|j	                  d      j                  dd      }|t        j                  || j                  z         z  }| j                  |j                  |      z  S r   )	rW   r^   r0   r   r   r   r   r   r   )rL   r   r   r   s       r7   ry   zBambaRMSNorm.forward  sy    #))%((7 $$Q',,R,>%Ht?T?T4T(UU{{]--k:::r6   c                 ^    t        | j                  j                         d| j                   S )Nz, eps=)r~   r   rk   r   )rL   s    r7   
extra_reprzBambaRMSNorm.extra_repr  s*    ))*+6$2G2G1HIIr6   r   )
r,   r-   r.   r_   rC   r0   r|   ry   r  r   r   s   @r7   r  r  z  s7    $ $$ $;U\\ ;ell ;Jr6   r  c                   J    e Zd Zddededef fdZ	 	 	 	 	 ddej                  dej                  dz  dej                  dz  d	e
dz  d
edz  deej                  ej                  f   dz  dee   deej                  eej                  ej                  f   dz  f   fdZ xZS )BambaDecoderLayerr<   r   r3  c                 x   t         |           d}|dk(  rt        nd } ||      | _        t	        |j
                  |j                        | _        t	        |j
                  |j                        | _        || _	        |dk(  rt        ||      | _        y |dk(  rt        ||      | _        y t        d|      )Nr#   r  linear_attention)r<   r   full_attentionzInvalid layer_type: )rB   rC   r  feed_forwardr  rZ   r  input_layernormpre_ff_layernorm
block_typer   mambar   	self_attn
ValueError)rL   r<   r   r3  num_expertsffn_layer_classrO   s         r7   rC   zBambaDecoderLayer.__init__  s    &1Q&6(D+F3+F,>,>FDWDWX ,V-?-?VEXEX Y$++#6YGDJ+++FI>DN3J>BCCr6   Nr   r   rt   r   	use_cacher   r   rQ   c           
      2   |}| j                  |      }| j                  dk(  r | j                  d|||d|}d }	n+| j                  dk(  r | j                  d||||||d|\  }}	||z   }|}| j	                  |      }| j                  |      }||z   }|	fS )Nr  )r   r8  r   r  )r   r   rt   r   r  r   r5   )r  r  r  r  r  r  )
rL   r   r   rt   r   r  r   r   residualself_attn_weightss
             r7   ry   zBambaDecoderLayer.forward  s     !,,];??00&DJJ +,- 	M !%__ 00/=t~~ 0+-) /#$70 0,M, !=0 --m<))-8 =0///r6   )r  )NNNFN)r,   r-   r.   r$   r3   rn   rC   r0   r|   r1   r
   r   r~   r   r&   FloatTensorry   r   r   s   @r7   r  r    s    D{ Ds D D( /304(,!&HL(0||(0 t+(0 &&-	(0
 (0 $;(0 #5<<#=>E(0 23(0 
u  %(9(95;L;L(L"MPT"TT	U(0r6   r  c                        e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZeedZ ej"                          fd       Z xZS )BambaPreTrainedModelr<   modelTr  r   )r   
attentionsc           
      j   t         |   |       t        |t              rt	        j
                  |j                         t	        j                  |j                  t        j                  t        j                  d|j                  dz                      t	        j
                  |j                         y y )Nr#   )rB   _init_weightsrl   r   initones_r%  copy_r'  r0   r&  r\   r
  r)  )rL   r   rO   s     r7   r  z"BambaPreTrainedModel._init_weights  sq    f%fj)JJv~~&JJv||UYYu||Av?O?ORS?S/T%UVJJvxx  *r6   )r,   r-   r.   r$   r2   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_is_stateful_can_compile_fullgraphr  r   _can_record_outputsr0   r   r  r   r   s   @r7   r  r    sg    &*#,-#4"5NL!*$
 U]]_! !r6   r  c                        e Zd Zdef fdZeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   defd                     Z xZS )
BambaModelr<   c           	      Z   t         |   |       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                        | _        g }t        |j                        D ],  }|j                  t        |||j                  |                . t        j                  |      | _        |j                   | _        t#        |j                  |j$                        | _        t)        |      | _        d| _        | j/                          y )N)r   r3  r  r<   F)rB   rC   pad_token_idpadding_idx
vocab_sizer   	EmbeddingrZ   embed_tokensrangenum_hidden_layersappendr  layers_block_type
ModuleListrH  r   r  r  final_layernormr:   
rotary_embgradient_checkpointing	post_init)rL   r<   decoder_layersirO   s       r7   rC   zBambaModel.__init__  s     !.. ++LL):):F<N<NPTP`P`av//0 	rA!!"3FaTZTlTlmnTo"pq	rmmN3$*$?$?!+F,>,>FDWDWX.f=&+#r6   N	input_idsr   rt   r   inputs_embedsr  r   rQ   c           
      p   |d u |d uz  rt        d      || j                  |      }|}|r|t        | j                        }|=t	        j
                  |j                  d   |j                        j                  d      }t        |x}	t              s)| j                  ||||d}
t        di |
t        di |
d}	| j                  ||      }t        | j                        D ]1  \  }} ||f|	| j                  j                   |      ||||d	|\  }}3 | j#                  |      }t%        ||
      S )Nz:You must specify exactly one of input_ids or inputs_embedsr  r#   rb  r   )r<   r  r   r   rt   )r  r  )rt   )r   rt   r   r  r   )last_hidden_stater   r5   )r  r  r   r<   r0   r\   rk   rM   r   rl   dictr   r   r  	enumeraterH  r  r  r   )rL   r  r   rt   r   r  r  r   r   causal_mask_mappingmask_kwargsr   r  decoder_layerr   s                  r7   ry   zBambaModel.forward  sn    -t";<YZZ  --i8M%0*$++>O <<(;(;A(>}G[G[\ffghiL?-F ++!."0#2 ,K #5"C{"C$C$Rk$R# #oom,oW )$++ 6 		A}*7+24;;3P3PQR3ST) /#$7+ +'M<		 ,,];&++
 	
r6   )NNNNNN)r,   r-   r.   r$   rC   r    r"   r   r0   r1   r|   r
   r  r   r   r&   r   ry   r   r   s   @r7   r  r    s    { &   .2.204(,26!%7
##d*7
 t+7
 &&-	7

 7
 ((4/7
 $;7
 237
 
!7
    7
r6   r  c                   P    e Zd ZddiZddiZddgdgfiZ fdZee	 	 	 	 	 	 	 	 dd	e	j                  dz  d
e	j                  dz  de	j                  dz  dedz  de	j                  dz  de	j                  dz  dedz  dee	j                  z  defd              Z	 	 	 	 	 	 d fd	Z xZS )BambaForCausalLMzlm_head.weightzmodel.embed_tokens.weightlm_headcolwise_gather_outputr   logitsc                 
   t         |   |       t        |      | _        |j                  | _        t        j                  |j                  |j                  d      | _        |j                  | _	        | j                          y )NFr   )rB   rC   r  r  r  r   r   rZ   r  z_loss_coefficientr  r  s     r7   rC   zBambaForCausalLM.__init__=  sc     '
 ++yy!3!3V5F5FUS"(";"; 	r6   Nr  r   rt   r   r  labelsr  logits_to_keeprQ   c	           
      L    | j                   d
||||||d|	}
|
j                  }t        |t              rt	        | d      n|}| j                  |dd|ddf         }d}| | j                  d
||| j                  j                  d|	}| j                  dkD  r[|j                  d      j                  |j                        j                  d      j                         }|| j                  |z  z   }t        |||
j                   |
j"                  |
j$                  	      S )aJ  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, BambaForCausalLM

        >>> model = BambaForCausalLM.from_pretrained("...")
        >>> tokenizer = AutoTokenizer.from_pretrained("...")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)r  r   rt   r   r  r  N)r  r  r  r   rd   ri   rV   rU   )lossr  r   r   r  r5   )r  r  rl   r3   slicer  loss_functionr<   r  r  	logsumexpr^   rW   r   r   r   r   r   r  )rL   r  r   rt   r   r  r  r  r  r   outputsr   slice_indicesr  r  z_losss                   r7   ry   zBambaForCausalLM.forwardG  s6   H ,64:: ,
)%+',
 ,
  118B>SV8W~ot4]kmA}a,?@A%4%%pVFt{{OeOepiopD&&*))b)1444::4FJJ1MRRTd55>>%#33!//))
 	
r6   c           
      h    | j                   j                  |d<   t        
|   |f||||||d|}	|	S )Nr  )r   r   r  rt   r  is_first_iteration)r<   num_logits_to_keeprB   prepare_inputs_for_generation)rL   r  r   r   r  rt   r  r  r   model_inputsrO   s             r7   r  z.BambaForCausalLM.prepare_inputs_for_generation  sU     $(;;#A#A w<	
+)'%1	
 	
 r6   )NNNNNNNr   )NNNNTF)r,   r-   r.   _tied_weights_keys_tp_plan_pp_planrC   r   r   r0   r1   r|   r
   r  r   r3   r   ry   r  r   r   s   @r7   r  r  7  s&   *,GH23H_-z:;H  .2.204(,26*.!%-.=
##d*=
 t+=
 &&-	=

 =
 ((4/=
   4'=
 $;=
 ell*=
 
 =
  =
D   r6   r  )r  r  r  )r   )r#   )Qcollections.abcr   typingr   r   r0   r    r   r  activationsr	   cache_utilsr
   r   
generationr   integrationsr   integrations.accelerater   integrations.hub_kernelsr   masking_utilsr   r   modeling_layersr   modeling_outputsr   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   r   utils.genericr   r    utils.import_utilsr!   utils.output_capturingr"   configuration_bambar$   
get_loggerr,   r0  r&   Moduler:   r   r|   r3   r   r_   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r  __all__r5   r6   r7   <module>r     s4  4 % &   & ! . ) 7 = 8 P 9 O K F & l l G 9 5 , 
		H	%	 0><299 ><B(	UU\\ 	U# 	U%,, 	U& %II%<<% 
% <<	%
 LL4'% % % '(%4#L@)RYY @)F; ;*VU\\ VS V
((	lO lO^ryy   Y'J299 J (J(:02 :0z !? ! !0 N
% N
 N
b g+_ g gT Er6   