
    ^jh                        d Z ddlZddlmZ ddlZddlmZ ddlmZ ddl	m
Z
 ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZmZ ddlmZ ddlmZ  ej>                  e       Z!dejD                  de#fdZ$d Z%d Z&d Z' G d dej
                  jP                        Z) G d dejP                        Z* G d dejP                        Z+ G d de      Z,e G d d e             Z- ed!"      e G d# d$e                    Z. ed%"      e G d& d'e                    Z/e G d( d)e-             Z0 ed*"       G d+ d,e-e             Z1g d-Z2y).zPyTorch MAMBA2 model.    N)	dataclass)nn   )initialization)ACT2FN)CacheDynamicCache)GenerationMixin)lazy_load_kernel)GradientCheckpointingLayer)PreTrainedModel)ModelOutputauto_docstringis_torchdynamo_compilinglogging)resolve_internal_import   )Mamba2Configinput_tensorpad_sizec                     t        | j                        dk(  r
ddddd|ddfnddd|ddf}t        j                  j                  j                  | |dd      S )z
    Padding x tensor with `pad_size` on the seq_len dim (dim=1)

    Assumes that we only have tensors of either size 4 or 3
       r   constant)modevalue)lenshapetorchr   
functionalpad)r   r   	pad_shapes      u/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/mamba2/modeling_mamba2.pypad_tensor_by_sizer#   (   sf     47|7I7I3Ja3OAq!Q!Q/VWYZ\]_gijlmUnI88""<ST"UU    c                    t        | |      } t        | j                        dk(  r.| j                  | j                  d   d|| j                  d         S | j                  | j                  d   d|| j                  d   | j                  d         S )z
    Padding input_tensor with `pad_size` on the seq_len dim (dim=1) and
    simultaneously splitting it into chunk sequences.

    Assumes that we only have tensors of either size 4 or 3
    r   r      )r#   r   r   reshape)r   r   
chunk_sizes      r"   reshape_into_chunksr*   3   s     &lH=L
<!###L$6$6q$92z<K]K]^_K`aa ##q!2z<3E3Ea3H,J\J\]^J_
 	
r$   c                 "   | j                  d      } | d   j                  g | j                         | } t        j                  t        j                  ||| j
                  t        j                        d      }| j                  | d      } t        j                  | d      }t        j                  t        j                  ||| j
                  t        j                        d      }|j                  | t        j                         }|S )zo
    More stable segment sum calculation. Uses cumulative sums and masking instead of direct subtractions.
    r&   .Ndevicedtype)diagonalr   dim)
sizeexpandr   trilonesr.   boolmasked_fillcumsuminf)r   r)   masktensor_segsums       r"   segment_sumr>   G   s     ""2&J 2<	*11S<3D3D3FS
SL::ejjZ@S@S[`[e[efqstD++TE15LLL26M ::ejjZ@S@S[`[e[efqrsD!--teeiiZ@Mr$   c                     |N|j                   d   dkD  r<|j                   d   dkD  r*| j                  }| |dddddf   z  j                  |      } | S )zm
    Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
    Nr   r   )r   r/   to)hidden_statesattention_maskr/   s      r"   apply_mask_to_padding_statesrC   [   sa    
 !n&:&:1&=&AnFZFZ[\F]`aFa##&1d
)CCGGNr$   c                   (     e Zd Zd fd	ZddZ xZS )MambaRMSNormGatedc                     t         |           t        j                  t	        j
                  |            | _        || _        y Nsuper__init__r   	Parameterr   r7   weightvariance_epsilonselfhidden_sizeeps	__class__s      r"   rJ   zMambaRMSNormGated.__init__h   s/    ll5::k#:; #r$   c                    |j                   }|j                  t        j                        }|?|t        j
                  j                  |j                  t        j                              z  }|j                  d      j                  dd      }|t        j                  || j                  z         z  }| j                  |j                  |      z  S Nr'   r&   T)keepdim)r/   r@   r   float32r   r   silupowmeanrsqrtrM   rL   )rO   rA   gateinput_dtypevariances        r"   forwardzMambaRMSNormGated.forwardm   s    #))%((7)BMM,>,>twwu}}?U,VVM $$Q',,R,>%Ht?T?T4T(UU{{]--k:::r$   gư>rG   __name__
__module____qualname__rJ   r^   __classcell__rR   s   @r"   rE   rE   g   s    $
	;r$   rE   c                   2    e Zd ZdZddededef fdZ ej                         d        Z
	 	 ddej                  d	edz  d
ej                  dz  fdZ	 	 ddej                  d	edz  d
ej                  dz  fdZ	 	 dd	edz  d
ej                  dz  fdZ xZS )Mamba2Mixeru  
    Compute ∆, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    ∆, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    config	layer_idxinitialize_mixer_weightsc           	         t         |           |j                  | _        |j                  | _        |j                  | _        |j                  | _        t        |j                  | j                  z        | _
        t        |j                        | _        || _        |j                  | _        |j                  | _        t         |j                     | _        |j$                  | _        |j&                  | _        |j(                  | _        |j*                  | _        |j,                  | _        |j.                  | _        |j0                  | _        |j2                  | _        |j4                  | _        | j                  d| j(                  z  | j
                  z  z   | _        t9        j:                  | j6                  | j6                  |j                  |j                  | j6                  |j                  dz
        | _        | j                  | j6                  z   | j                  z   }t9        j>                  | j                  ||j@                        | _!        t9        jD                  tG        jH                  | j                              | _%        t9        jD                  tG        jH                  | j                              | _&        tO        | j                  | j$                        | _(        t9        jD                  tG        jH                  | j                              | _)        |r3| jJ                  jT                  jV                  dk7  r| jY                          t9        j>                  | j                  | j                  |j@                        | _-        |j@                  | _         t]        d      }t_        |dd       a0t_        |d	d       a1t]        d
      }te        |d      a3te        |d      a4te        |d      a5tm        tf        th        tj        tb        t`        f      a7tn        stp        js                  d       |jt                  |   | _;        y )Nr'   r   )in_channelsout_channelsbiaskernel_sizegroupspaddingrn   rQ   metazcausal-conv1dcausal_conv1d_updatecausal_conv1d_fnz	mamba-ssmz8ops.triton.selective_state_update.selective_state_update)chained_pathz1ops.triton.ssd_combined.mamba_chunk_scan_combinedz8ops.triton.ssd_combined.mamba_split_conv1d_scan_combineda  The fast path is not available because one of `(selective_state_update, causal_conv1d_fn, causal_conv1d_update)` is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d)<rI   rJ   	num_headsrP   
state_sizessm_state_sizeconv_kernelconv_kernel_sizeintr5   intermediate_sizetime_step_rankri   use_conv_bias
hidden_act
activationr   actlayer_norm_epsilonrms_normn_groupshead_dimr)   time_step_limittime_step_mintime_step_maxtime_step_floorconv_dimr   Conv1dconv1dLinearuse_biasin_projrK   r   emptydt_biasA_logrE   normDr.   typeinit_mamba2_weightsout_projr   getattrru   rv   r   selective_state_updatemamba_chunk_scan_combined mamba_split_conv1d_scan_combinedallis_fast_path_availableloggerwarning_oncelayer_types
layer_type)rO   rh   ri   rj   projection_sizecausal_conv1d	mamba_ssmrR   s          r"   rJ   zMamba2Mixer.__init__   sS   ))!--$// & 2 2!$V]]T5E5E%E!F!&"7"78"#11 ++&++,"(";"; ++%55#11#11%55..T]]1BTEXEX1XXii%%**==&&*
 004==@4>>Qyy
 ||EKK$?@ \\%++dnn"=>
%d&<&<$BYBYZ	ekk$..9:#(;(;(@(@F(J$$&		$"8"8$:J:JQWQ`Q`a )9&}6LdS"=2DdK %[1	!8$^"
 %<$W%
! ,C$^,
(
 "%&)0 $"
 &> !,,Y7r$   c                 t   t        j                  d| j                  dz   | j                  j                  t         j
                        }t        j                  | j                  t        j                  |             t        j                  | j                         t        j                  t        j                  | j                  | j                  j                  t         j
                        t        j                  | j                        t        j                  | j                         z
  z  t        j                  | j                         z         j#                  | j$                        }|t        j                  t        j&                  |              z   }t        j                  | j                  |       y )Nr   r-   )min)r   arangerx   r   r.   rV   initcopy_logones_r   exprandr   mathr   r   clampr   expm1)rO   Adtinv_dts       r"   r   zMamba2Mixer.init_mamba2_weights   s   LLDNNQ.tzz7H7HPUP]P]^

4::uyy|,

466YYJJt~~dll.A.AWxx**+dhht7I7I.JJLhht))*+
 %D((%
)	 	 eiibS!1 122

4<<(r$   NrA   cache_paramsrB   c                 r   t        ||      }| j                  |      }|j                  \  }}}| j                  | j                  z  }|j                  d   d| j
                  z  z
  d| j                  z  | j                  z  z
  | j                  z
  dz  }	|d uxr |j                  | j                        }
|
rL|j                  | j                     j                  d   }|j                  | j                     j                  d   }|
r|dk(  r|j                  d      j                  |	|	| j
                  | j                  | j                  gd      \  }}}}}t        || j                   j"                  j                  d      | j                   j$                  | j&                        }t)        j                  || j
                  ||gd      \  }}}t)        j*                  | j,                  j/                                }|d d d df   d d d d d f   j1                  d| j2                  | j                        j5                  t(        j6                        }|d d d d d f   j1                  dd| j2                        }| j8                  d d d df   j1                  d| j2                        }| j:                  d d d df   j1                  d| j2                        }|j=                  || j                  |j                  d   | j                  z        }|j=                  || j                  |j                  d   | j                  z        }|j=                  || j                  | j2                        }t?        ||||||d |d	
      }|j=                  || j                  | j2                  z        }| jA                  ||      }| jC                  |      d d d df   }|S t)        j*                  | j,                  j/                                }| jD                  d
t/        d      fk(  ri nd| jD                  i}| jF                  r|tI        || j                   j"                  j                  d      | j                   j$                  | j8                  |f| j:                  | jJ                  d | j&                  | j@                  j"                  | j@                  jL                  | jB                  j"                  | jB                  j$                  | j2                  | j                  ddd|}|S |j                  |	|	| j
                  | j                  | j                  gd      \  }}}}}|jO                  dd      }|
rt)        jP                  |gd      }|YtR        jT                  jW                  || jX                  |j                  d   z
  df      }|j[                  || j                         | j&                  dvr5| j]                  | j!                  |      dd |j                  d   f         }nPt_        || j                   j"                  j                  d      | j                   j$                  | j&                        }|
r|d d d d | d f   }|jO                  dd      }t        ||      }t)        j                  || j
                  ||gd      \  }}}ta        |j=                  ||d| j2                        |||j=                  ||| j                  d      |j=                  ||| j                  d      f| jJ                  | j:                  d d d| j8                  d|
rnd d|\  }}|||jc                  || j                         |j=                  ||d      }| jA                  ||      }| jC                  |      }|S )Nr&   r'   r   r   r2   .r/   T)zr   dt_softplusg        r;   dt_limitF)r   r)   seq_idxr   rmsnorm_weightrmsnorm_epsoutproj_weightoutproj_biasheaddimngroupsnorm_before_gatereturn_final_statesri   )rW   swish)xrL   rn   r   )r)   r   r   r   r   r   r   initial_states)2rC   r   r   r   rz   r~   rx   has_previous_stateri   layersconv_statesrecurrent_statessqueezesplitr   ru   r   rL   rn   r   r   r   r   floatr5   r   r@   rV   r   r   viewr   r   r   r   trainingr   r)   rM   	transposecatr   r   r    r|   update_conv_stater   rv   r   update_recurrent_state)rO   rA   r   rB   projected_states
batch_sizeseq_len_groups_time_state_sized_mlpuse_precomputed_states
conv_staterecurrent_stater[   hidden_states_B_Cr   BCr   r   r   hidden_states_reshapedoutdt_limit_kwargsnew_conv_statescan_output	ssm_states                              r"   cuda_kernels_forwardz Mamba2Mixer.cuda_kernels_forward   s?    5]NS<<6 "/!4!4
GQ!%1D1D!D""2&$((()$--$"5"556 nn  ".T!9!ml>]>]^b^l^l>m "%,,T^^<HHKJ*11$..ARRSTUO "gl0@0H0H0K0Q0Qt55t}}dnnU[] 1R 1-Aq$)2
 !5!""**1-  ! #(++!'')?AWX#M1a 4::++-..A!T3,1d
+222t}}dFYFYZ]]didqdq]rAAq$J&&r2t}}=Bll1dC<077DMMJGq$|$++B>Az4==!''!*2MNAz4==!''!*2MNA%2%7%7
DNNTXTaTa%b"2& M *..z4>>DMM;YZM IImT:M --.q$|<CH 
A 4::++-..A$($8$8S%,<O$ObV`bfbvbvUwO }}!56$KK&&..q1KK$$LL ff# ##'99#3#3 $		 : :#'==#7#7!%!3!3 MM MM%*(-#$ &%v 
K 5E4J4JE4#9#94==$..Y_a 5K 511d-r
 %6$?$?1$E!) ).		:?P2QWY(Z%+%']]%6%6)D,A,ADUD[D[\^D_,_ab+c&N !22>T^^2\??*;;(,=N1OPSUrWhWnWnoqWrUrPr1s(t%(8+#{{1199!<![[--#'??	)% *(9!Q	/(J%$5$?$?1$E!$@ARTb$c!&+kk%++-CE[\'#q! *C!&&z7BNFF:wrBFF:wrB*  $ff (, LL $6L?RV* &*&Y$ (\-E 77	T^^7\)..z7BG"iiT: mmK0
r$   c                    |j                   \  }}}|j                  }t        ||      }| j                  |      }|j                   d   d| j                  z  z
  d| j
                  z  | j                  z  z
  | j                  z
  dz  }	|j                  |	|	| j                  | j                  | j                  gd      \  }}}
}}|j                  dd      }|d uxr |j                  | j                        }|r&|j                  | j                     j                  d   }|r|dk(  r|j                  || j                        d| j                    d f   }t#        j$                  || j&                  j(                  j+                  d      z  d      }| j,                  r|| j&                  j.                  z   }| j1                  |      }n|rt#        j2                  |gd      }|Yt4        j6                  j9                  || j                   |j                   d   z
  df      }|j                  || j                         | j1                  | j'                  |      dd |j                   d   f   j                  dd            }|r|d d | d d d f   }t        ||      }t#        j                  || j                  | j
                  | j                  z  | j
                  | j                  z  gd      \  }}}t#        j:                  | j<                  j?                                }|r|dk(  r|j                  | j                     j@                  }|d d dd d f   d d d df   }|j                  dd      jC                  ||j                   d   | jD                        }| jF                  d   jC                  | jF                  j                   d   | jD                        }t"        j4                  j6                  jI                  ||jK                  |j                        z         }t#        jL                  || jN                  d   | jN                  d         }|d	   jC                  | j                  | jD                  | j                        jK                  t"        jP                  
      }t#        j:                  |d   |z        jK                  |      }|jS                  || j
                  d      dd d d f   }|jC                  || j
                  | j                  | j
                  z  |j                   d         jU                         }|jS                  |d|j                   d         }|d   |dd d d f   z  }|jS                  |d| jD                        }||d   z  jK                  |      }|j                  | j                     jV                  d   |z  |z   }|jY                  || j                        }|jS                  || j
                  d      dd d d f   }|jC                  || j
                  | j                  | j
                  z  |j                   d         jU                         }|jS                  |d|j                   d         }|jK                  |j@                  |j                        }|j[                  || j                  z  | jD                  | j                        }|j[                  || j                  z  | j                  d      }t#        j\                  ||      }|j[                  || j                  | jD                        }| j^                  d   jC                  | j^                  j                   d   | jD                        }|||z  z   jK                  |j                        }|jS                  |d      d d d df   }nt4        j6                  jI                  || jF                  z         }t#        jL                  || jN                  d   | jN                  d         }|jS                  ||d| jD                        j?                         }|jS                  ||d| j                        j?                         }|jS                  ||d| j                        j?                         }|ja                  | j                  | j
                  z  d| j                        }|ja                  | j                  | j
                  z  d| j                        }| jb                  || jb                  z  z
  | jb                  z  }| j^                  d   te        ||      z  }||d   z  }|jK                  |j                        |z  }||||fD cg c]  }tg        ||| jb                         c}\  }}}}|ji                  dddd      }t#        jj                  |d      } t#        j:                  tm        |            }!|d d d d d d d d d d d f   |d d d d d d d d d d d f   z  }"|"j%                  d      }#|#d   |!ji                  ddddd      d   z  }$|$j%                  d      }%|%d   |d d d d d f   z  j%                  d      }&t#        j:                  | d d d d d d dd f   | z
        }'||'ji                  dddd      d   z  }(|(dd d d f   |d   z  j%                  d      })|rR|j                  | j                     jV                  d   d d d f   jK                  |)j                  |)j@                        nt#        jn                  |)d d d df         }*t#        j2                  |*|)gd      })t#        j:                  tm        t4        j6                  j9                  | d d d d d d df   d                  }+|+j                  dd      }+|+d	   |)d d d d d df   z  j%                  d      },|,d d d df   |,d d df   }-})t#        j:                  |       }.|dd d d f   |)d d d d d df   z  }/|.ji                  dddd      }0|/j%                  d      |0d   z  }1|&|1z   }|jS                  |d| j                  | jD                        }||z   }|dkD  r|d d d |d d d d f   }|jS                  ||d      }|-||jY                  |-| j                         | jq                  ||
      }2| js                  |2jK                  |            }3|3S c c}w )Nr&   r'   r2   r   r   r   .r,   ).NNr   )r.   r-   )r3   output_sizer   r   r1   )r/   r.   )r   r   ):r   r/   rC   r   r~   r   rz   rx   r   r   r   r   ri   r   r   r   r|   r   sumr   rL   r   r   rn   r   r   r   r   r    r   r   r   r.   r5   r   r   softplusr@   r   r   rV   r(   
contiguousr   r   r   bmmr   repeat_interleaver)   r#   r*   permuter:   r>   
zeros_liker   r   )4rO   rA   r   rB   r   r   r   r/   r   r   r[   r   r   r   r   r   r   r   r   cache_devicer   dAdBdBx
ssm_statesssm_states_reshaped
C_reshapedyr   r   
D_residualtA_cumsumLG_intermediateGM_intermediateMY_diagdecay_statesB_decaystatesprevious_statesdecay_chunk
new_statesr   state_decay_outC_times_statesstate_decay_out_permutedY_offr   contextualized_statess4                                                       r"   torch_forwardzMamba2Mixer.torch_forward  s    "/!4!4
GQ## 5]NS<<6!''+a$2H2H.HH1t}}K\_c_r_rKrrsw  tB  tB  B  GH  H,<,B,Bt55t~~V\^ -C -
)1d%r .77!<!-T!9!ml>]>]^b^l^l>m!%,,T^^<HHKJ "gl&889JVZVdVd8efilp  mB  mB  lB  lC  gC  DK %		dkk0088;;! !!$58H8H$H! $): ;%$)IIz;L.MSU$V!' mm//%(=(=@Q@W@WXZ@[([]^'_ ..{dnn.U $5F)GMiN_NeNefhNiMiHi)j)t)tuvxy)z {%$5a'Ao$F!89JN[#kk##T]]T5H5H%H$--Z^ZmZmJmn
q! YYtzz'')**!gl'..t~~>EEL Aq!GQc\*Ba#**:rxx|T]]SBll9-44T\\5G5G5JDMMZG$$--b7::bhh3G.GHBR!5!5a!8$:N:Nq:QRB/"))$..$--I\I\]``glgtgt`uA))ByMA-.22,2GB
 		*dmmR8dAFAT]]DNNdmm4SUVU\U\]_U`allnA		*b!''"+6AI3a<0B *11*b$--PMi0044L4IC &,,T^^<MMaPSUUX[[J%<<ZSWSaSa<bJ 		*dmmR8dAFAT]]DNNdmm4SUVU\U\]_U`allnA		*b!''"+6A $ahhaggFJ",//*t~~2Mt}}^b^q^q"r
T^^ ;T=P=PRSTJ		-z:Az4>>4==AA y!((a$--HA]Q&&**1773A 		*b)!T3,7A ''T\\(9:BR!5!5a!8$:N:Nq:QRB)11*gr4==Y__aM		*gr43F3FGMMOA		*gr43F3FGMMOA##DNNdmm$CX\XfXf#gA##DNNdmm$CX\XfXf#gA'DOO*CCtVH	*-?x-XXJ *ByM9M](()B.A cpqrtuwxay%z\]&9!Xt&W%z"M1a 		!Q1%A||A2.H 		+a.)A q!Qa23a1dAq!8K6LLN""r"*A y\AIIaAq!,DY,OON""r"*A 	l]1a:%>>CCCJF !99XaArsl%;h%FGL,..q"b!<YGGGc4l+mI.FFKKPQKRF * ##DNN3DDQG4PSSZ`ZfZfouo|o|S}%%fQUm4 
 YY8a@F))K0A0A(1aQRTV;BWY_0`$abK%//15K%o61dC9PPUUZ[U\J *1crc6 2Jq"u4EIF $ii1OT1oq!T30GGN'6'>'>q!Q'J$#''+.Fy.QQE A		*b$..$--HAJA!|a'1a'(		*gr2A $)A33I3Xii4(
 !%knnU.C D$$I &{s   t	c                     t         rId| j                  j                  j                  j                  v rt               s| j                  |||      S | j                  |||      S )Ncuda)r   r   rL   r.   r   r   r   r  )rO   rA   r   rB   kwargss        r"   r^   zMamba2Mixer.forwardh  sT     "f0C0C0J0J0O0O&OXpXr,,]L.YY!!-~NNr$   )TNN)ra   rb   rc   __doc__r   r}   r8   rJ   r   no_gradr   Tensorr   r   r  r^   rd   re   s   @r"   rg   rg   y   s    ]8| ]8 ]8W[ ]8~ U]]_) )$ &*.2	m||m dlm t+	mf &*.2	E%||E% dlE% t+	E%V &*.2		O dl	O t+		Or$   rg   c                   &     e Zd Zd fd	Zd Z xZS )Mamba2RMSNormc                     t         |           t        j                  t	        j
                  |            | _        || _        y)zM
        Mamba2RMSNorm is equivalent to T5LayerNorm and LlamaRMSNorm
        NrH   rN   s      r"   rJ   zMamba2RMSNorm.__init__u  s1     	ll5::k#:; #r$   c                 "   |j                   }|j                  t        j                        }|j	                  d      j                  dd      }|t        j                  || j                  z         z  }| j                  |j                  |      z  S rT   )	r/   r@   r   rV   rX   rY   rZ   rM   rL   )rO   rA   r\   r]   s       r"   r^   zMamba2RMSNorm.forward}  sy    #))%((7 $$Q',,R,>%Ht?T?T4T(UU{{]--k:::r$   r_   r`   re   s   @r"   r  r  t  s    $;r$   r  c                   T     e Zd Z fdZ	 	 ddedz  dej                  dz  fdZ xZS )Mamba2Blockc                     t         |           || _        || _        |j                  | _        t        |j                  |j                        | _        t        ||d      | _
        y )Nrs   F)ri   rj   )rI   rJ   rh   ri   residual_in_fp32r  rP   r   r   rg   mixer)rO   rh   ri   rR   s      r"   rJ   zMamba2Block.__init__  sU    " & 7 7!&"4"4&:S:ST	 9W\]
r$   Nr   rB   c                    |}| j                  |j                  | j                   j                  j                              }| j                  r|j                  t
        j                        }| j                  |||      }||z   }|S )Nr   r   rB   )r   r@   rL   r/   r  r   rV   r  )rO   rA   r   rB   r  residuals         r"   r^   zMamba2Block.forward  su     !		-"2"29I9I9O9O"2"PQ  {{5==1H

=|\j
k =0r$   r  )	ra   rb   rc   rJ   r   r   r  r^   rd   re   s   @r"   r  r    s7    ^ &*.2	 dl t+	r$   r  c                   h     e Zd ZU eed<   dZdgZdZdZdZ	 e
j                          fd       Z xZS )Mamba2PreTrainedModelrh   backboner  Tc                    t         |   |       t        |t              r%|j	                          t        j                  |j                  j                  t        j                  d             |j                  j                  )t        j                  |j                  j                         t        j                  |j                  j                  t        j                  d             | j                  j                  rC|j                  j                  }|t        j                  | j                  j                         z  }yyy)zInitialize the weights.   )aN)rI   _init_weights
isinstancerg   r   r   kaiming_uniform_r   rL   r   sqrtrn   zeros_r   rh   rescale_prenorm_residualnum_hidden_layers)rO   moduleprR   s      r"   r(  z#Mamba2PreTrainedModel._init_weights  s     	f%fk* &&(!!&--"6"6$))A,G}}!!-FMM../!!&//"8"8DIIaLI{{33 OO**TYYt{{<<== 4 +r$   )ra   rb   rc   r   __annotations__base_model_prefix_no_split_modulessupports_gradient_checkpointing_can_compile_fullgraph_is_statefulr   r  r(  rd   re   s   @r"   r#  r#    sB    "&&*#!LU]]_> >r$   r#  z-
    Class for the MAMBA2 model outputs.
    )custom_introc                   |    e Zd ZU dZdZej                  dz  ed<   dZe	dz  ed<   dZ
eej                     dz  ed<   y)Mamba2Outputa4  
    cache_params (`Cache`):
        The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to
        avoid providing the old `input_ids`.

        Includes both the State space model state matrices after the selective scan, and the Convolutional states
    Nlast_hidden_stater   rA   )ra   rb   rc   r  r:  r   FloatTensorr1  r   r   rA   tuple r$   r"   r9  r9    sG     37u((4/6!%L%$,%59M5**+d29r$   r9  zK
    Base class for causal language model (or autoregressive) outputs.
    c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
dz  ed<   dZeej                     dz  ed<   y)Mamba2CausalLMOutputa  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    cache_params (`Cache`):
        The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to
        avoid providing the old `input_ids`.

        Includes both the State space model state matrices after the selective scan, and the Convolutional states
    Nlosslogitsr   rA   )ra   rb   rc   r  r@  r   r;  r1  rA  r   r   rA   r<  r=  r$   r"   r?  r?    s[    
 &*D%

d
")'+FE$+!%L%$,%59M5**+d29r$   r?  c                        e Zd Z fdZd Zd Zd Ze	 	 	 	 	 	 	 ddej                  dz  dej                  dz  de
dz  d	edz  d
edz  dedz  dej                  dz  deez  fd       Z xZS )Mamba2Modelc           	         t         |   |       t        j                  |j                  |j
                        | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        d| _        t        |j
                  |j                        | _        | j!                  | j"                         | j%                          y c c}w )Nr   Frs   )rI   rJ   r   	Embedding
vocab_sizerP   
embeddings
ModuleListranger.  r  r   gradient_checkpointingr  r   norm_f"_register_load_state_dict_pre_hook	load_hook	post_init)rO   rh   idxrR   s      r"   rJ   zMamba2Model.__init__  s     ,,v'8'8&:L:LMmmSXY_YqYqSr$sC[3%G$st&+##F$6$6F<U<UV//? %ts   &Cc                 f    |D ],  }d|v s|j                  |      ||j                  dd      <    y  y )Nz
embedding.zembeddings.)popreplace)rO   
state_dictprefixargsks        r"   rM  zMamba2Model.load_hook  s;     	Aq EO^^TUEV
199\=AB	r$   c                     | j                   S rG   rG  rO   s    r"   get_input_embeddingsz Mamba2Model.get_input_embeddings  s    r$   c                     || _         y rG   rX  rO   new_embeddingss     r"   set_input_embeddingsz Mamba2Model.set_input_embeddings  s	    (r$   N	input_idsinputs_embedsr   	use_cacheoutput_hidden_statesreturn_dictrB   returnc                 ^   ||n| j                   j                  }||n#| j                  s| j                   j                  nd}||n| j                   j                  }|du |duz  rt        d      || j                  |      }| j                  r| j                  r|rd}|r|t        | j                         }|}	|rdnd}
| j                  D ]  } ||	||      }	|s|
|	fz   }
 | j                  |	      }	|r|
|	fz   }
|st        d |	||
fD              S t        |	|r||
      S d|
      S )	a  
        cache_params (`Cache`, *optional*):
            If passed along, the model uses the previous state in all the blocks (which will give the output for the
            `input_ids` provided as if the model add `state_input_ids + input_ids` as context).
        use_cache (`bool`, *optional*):
            If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits.
        NFz:You must specify exactly one of input_ids or inputs_embeds)rh   r=  r   c              3   &   K   | ]	  }||  y wrG   r=  ).0vs     r"   	<genexpr>z&Mamba2Model.forward.<locals>.<genexpr>E  s     fqXYXefs   )r:  r   rA   )rh   rb  r   ra  rc  
ValueErrorrG  rJ  r	   r   rK  r<  r9  )rO   r_  r`  r   ra  rb  rc  rB   r  rA   all_hidden_statesmixer_blocks               r"   r^   zMamba2Model.forward  se   ( %9$D $++JjJj 	 "+!6IZ^ZgZgT[[=R=Rmr	%0%<k$++BYBY-t";<YZZ  OOI6M&&4==YI-'t{{;L%"6BD;; 	IK')-M $$58H$H!	I M2 1]4D Df]LBS$Tfff+)2+
 	
8<+
 	
r$   )NNNNNNN)ra   rb   rc   rJ   rM  rZ  r^  r   r   
LongTensorr   r8   r  r<  r9  r^   rd   re   s   @r"   rC  rC    s    
)  .215%)!%,0#'.2<
##d*<
 ''$.<
 dl	<

 $;<
 #Tk<
 D[<
 t+<
 
	<
 <
r$   rC  z
    The MAMBA2 Model transformer with a language modeling head on top (linear layer with weights not tied to the input
    embeddings).
    c                   j    e Zd ZddiZ fdZd Zd Z	 	 	 	 	 ddedz  dej                  dz  d	e
dz  f fd
Ze	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dedz  dej                  dz  de
dz  de
dz  de
dz  dej                  dz  deej                  z  deez  fd       Z xZS )Mamba2ForCausalLMzlm_head.weightzbackbone.embeddings.weightc                     t         |   |       t        |      | _        t	        j
                  |j                  |j                  d      | _        | j                          y )NFrr   )
rI   rJ   rC  r$  r   r   rP   rF  lm_headrN  )rO   rh   rR   s     r"   rJ   zMamba2ForCausalLM.__init__W  sF     #F+yy!3!3V5F5FUSr$   c                 6    | j                   j                         S rG   )r$  rZ  rY  s    r"   rZ  z&Mamba2ForCausalLM.get_input_embeddings^  s    }}1133r$   c                 8    | j                   j                  |      S rG   )r$  r^  r\  s     r"   r^  z&Mamba2ForCausalLM.set_input_embeddingsa  s    }}11.AAr$   Nr   rB   is_first_iterationc           	      F    t        	|   |f|||||d|}|r|sd |d<   |S )N)r`  ra  r   rB   rt  rB   )rI   prepare_inputs_for_generation)
rO   r_  r`  ra  r   rB   rt  r  model_inputsrR   s
            r"   rv  z/Mamba2ForCausalLM.prepare_inputs_for_generationd  sN     w<
'%)1
 
 /-1L)*r$   r_  r`  labelsrb  rc  ra  logits_to_keeprd  c
           	      &   ||n| j                   j                  }| j                  |||||||      }|d   }t        |	t              rt        |	 d      n|	}| j                  |dd|ddf   j                  | j                  j                  j                              j                         }d}|* | j                  d||| j                   j                  d|
}|s|f|dd z   }||f|z   S |S t        |||j                  |j                        S )aN  
        cache_params (`Cache`, *optional*):
            If passed along, the model uses the previous state in all the blocks (which will give the output for the
            `input_ids` provided as if the model add `state_input_ids + input_ids` as context).
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        use_cache (`bool`, *optional*):
            If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits.
        N)r   r`  rb  rc  ra  rB   r   )rA  rx  rF  r   )r@  rA  r   rA   r=  )rh   rc  r$  r)  r}   slicerq  r@   rL   r/   r   loss_functionrF  r?  r   rA   )rO   r_  r`  r   rx  rb  rc  ra  rB   ry  r  mamba2_outputsrA   slice_indicesrA  r@  outputs                    r"   r^   zMamba2ForCausalLM.forward}  s-   2 &1%<k$++BYBY%'!5#) ' 
 'q)8B>SV8W~ot4]kmA}a,?@CCDLLDWDWD]D]^_eeg%4%%pVFt{{OeOepiopDY!33F)-)9TGf$EvE#'44(66	
 	
r$   )NNNNF)	NNNNNNNNr   )ra   rb   rc   _tied_weights_keysrJ   rZ  r^  r   r   r  r8   rv  r   rm  r;  r}   r<  r?  r^   rd   re   s   @r"   ro  ro  N  sB    +,HI4B %).2*/
 dl t+ !4K2  .226%)*.,0#'!%.2-.6
##d*6
 ((4/6
 dl	6

   4'6
 #Tk6
 D[6
 $;6
 t+6
 ell*6
 
%	%6
 6
r$   ro  )ro  rC  r#  )3r  r   dataclassesr   r   r    r   r   activationsr   cache_utilsr   r	   
generationr
   integrationsr   modeling_layersr   modeling_utilsr   utilsr   r   r   r   utils.import_utilsr   configuration_mamba2r   
get_loggerra   r   r  r}   r#   r*   r>   rC   ModulerE   rg   r  r  r#  r9  r?  rC  ro  __all__r=  r$   r"   <module>r     s     !   & ! . ) , 9 - S S 9 . 
		H	%VU\\ VS V
((	; ;$xO")) xOv;BII ;", 4 ">O "> ">J 
 :; : : 
 :; : :& V
' V
 V
r `
- `
`
F Hr$   