
    ^jq                    ~   d dl Z d dlmZ d dlmZ d dlmZ d dlZd dlmZ ddl	m
Z ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZmZm Z  ddl!m"Z"m#Z# ddl$m%Z%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z,m-Z-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5 ddl6m7Z7  e.jp                  e9      Z: G d dej                  jv                        Z< G d dejv                        Z= G d dejv                        Z>d ej~                  d!e@d"ej~                  fd#ZA	 dKd$ejv                  d%ej~                  d&ej~                  d'ej~                  d(ej~                  dz  d)eBd*eBfd+ZCd, ZD ed-      dLd.       ZE G d/ d0ejv                        ZFd1ej~                  d2e@fd3ZGd4 ZHd5 ZI G d6 d7ejv                        ZJ G d8 d9ejv                        ZK G d: d;ejv                        ZL G d< d=e      ZM G d> d?e      ZNe+ G d@ dAe&             ZOe+ G dB dCeO             ZP G dD dEeOe      ZQ e+dFG       G dH dIeO             ZRg dJZSy)M    N)Callable)cycle)Optional)nn   )initialization)ACT2FN)CacheDynamicCache)GenerationMixin)use_kernel_func_from_hub)force_accelerate_hooks)lazy_load_kernel)create_causal_mask)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast SequenceClassifierOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tupleis_torchdynamo_compilinglogging)maybe_autocastmerge_with_config_defaults)resolve_internal_import)capture_outputs   )Zamba2Configc                   (     e Zd Zd fd	ZddZ xZS )Zamba2RMSNormGatedc                     t         |           t        j                  t	        j
                  |            | _        || _        || _        y N)	super__init__r   	Parametertorchonesweightvariance_epsilon
group_size)selfhidden_sizer0   eps	__class__s       u/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/zamba2/modeling_zamba2.pyr*   zZamba2RMSNormGated.__init__5   s6    ll5::k#:; #$    c                 b   |j                   }|j                  t        j                        }|?|t        j
                  j                  |j                  t        j                              z  }|j                  ^ }}|| j                  z  } |j                  g ||| j                   }|j                  d      j                  dd      }|t        j                  || j                  z         z  } |j                  g ||| j                  z   }| j                  |j                  |      z  S N   T)keepdim)dtypetor,   float32r   
functionalsilushaper0   viewpowmeanrsqrtr/   r.   )	r1   hidden_statesgateinput_dtypeprefix_dimslast_dimgroup_counthidden_states_groupvariances	            r5   forwardzZamba2RMSNormGated.forward;   s   #))%((7)BMM,>,>twwu}}?U,VVM!.!4!4h$//10m00\+\{\DOO\&**1-222t2D1EKK4K`K`@`4aa0+00]+]{T__?\]{{]--k:::r6   gư>r(   )__name__
__module____qualname__r*   rN   __classcell__r4   s   @r5   r&   r&   4   s    %;r6   r&   c                   h     e Zd Zddeddf fdZdej                  dej                  fdZd Z xZ	S )	Zamba2RMSNormr3   returnNc                     t         |           t        j                  t	        j
                  |            | _        || _        y)z<
        Zamba2RMSNorm is equivalent to T5LayerNorm
        N)r)   r*   r   r+   r,   r-   r.   r/   )r1   r2   r3   r4   s      r5   r*   zZamba2RMSNorm.__init__J   s1     	ll5::k#:; #r6   rF   c                 "   |j                   }|j                  t        j                        }|j	                  d      j                  dd      }|t        j                  || j                  z         z  }| j                  |j                  |      z  S r8   )	r<   r=   r,   r>   rC   rD   rE   r/   r.   )r1   rF   rH   rM   s       r5   rN   zZamba2RMSNorm.forwardR   sy    #))%((7 $$Q',,R,>%Ht?T?T4T(UU{{]--k:::r6   c                 ^    t        | j                  j                         d| j                   S )Nz, eps=)tupler.   rA   r/   )r1   s    r5   
extra_reprzZamba2RMSNorm.extra_reprY   s*    ))*+6$2G2G1HIIr6   rO   )
rP   rQ   rR   floatr*   r,   TensorrN   r\   rS   rT   s   @r5   rV   rV   I   s7    $ $$ $;U\\ ;ell ;Jr6   rV   c                        e Zd ZU ej                  ed<   ddef fdZe	 	 	 ddedz  de	d   de
dz  ded	ef   fd
       Z ej                         ed               Z xZS )Zamba2RotaryEmbeddinginv_freqNconfigc                    t         |           |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |      \  }| _
        | j                  d|d       | j                  d|j                         d       y )N	rope_typedefaultra   F)
persistentoriginal_inv_freq)r)   r*   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenrb   rope_parametersrd   compute_default_rope_parametersr   attention_scalingregister_bufferclone)r1   rb   devicerope_init_fnra   r4   s        r5   r*   zZamba2RotaryEmbedding.__init__`   s    "("@"@$*$B$B!44[A!%!E!E>>Y&.t~~>L+7V+L($(ZeD0(..2BuUr6   rp   ztorch.deviceseq_lenrW   ztorch.Tensorc                    | j                   d   }t        | dd      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                        j                  |t        j                        |z  z  z  }||fS )	a  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        
rope_thetahead_dimNg      ?r   r9   r<   rp   r<   )	rk   getattrr2   num_attention_headsr,   arangeint64r=   r]   )rb   rp   rr   basedimattention_factorra   s          r5   rl   z5Zamba2RotaryEmbedding.compute_default_rope_parametersp   s    & %%l3fj$/c63E3EIcIc3c U\\!S!5;;?BB&X]XcXcBdgjjk
 )))r6   c                 N   | j                   d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d      5  |j                         |j                         z  j                  dd      }t        j                  ||fd	      }|j                         | j                  z  }|j                         | j                  z  }	d d d        j	                  |j                   
      	j	                  |j                   
      fS # 1 sw Y   AxY w)Nr   r:   r#   mpscpuF)device_typeenabledr9   r}   rv   )ra   r]   expandrA   r=   rp   
isinstancetypestrr   	transposer,   catcosrm   sinr<   )
r1   xposition_idsinv_freq_expandedposition_ids_expandedr   freqsembr   r   s
             r5   rN   zZamba2RotaryEmbedding.forward   sR    !MM$4-8>>@GGHZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfkUC 	5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C		5 vvAGGv$cff177f&;;;	5 	5s   BFF$r(   NNN)rP   rQ   rR   r,   r^   __annotations__r$   r*   staticmethodr   intr[   r]   rl   no_gradr   rN   rS   rT   s   @r5   r`   r`   ]   s    llV| V  &*+/"*t#*(* t* 
~u$	%	* *: U]]_<  <r6   r`   rF   n_reprW   c                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r#   N)rA   r   reshape)rF   r   batchnum_key_value_headsslenru   s         r5   	repeat_kvr      so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TTr6   modulequerykeyvalueattention_maskscalingdropoutc                    t        || j                        }t        || j                        }	t        j                  ||j	                  dd            |z  }
||
|z   }
t
        j                  j                  |
dt        j                        j                  |j                        }
t
        j                  j                  |
|| j                        }
t        j                  |
|	      }|j	                  dd      j                         }||
fS )Nr9   r   r:   )r}   r<   )ptrainingr#   )r   num_key_value_groupsr,   matmulr   r   r?   softmaxr>   r=   r<   r   r   
contiguous)r   r   r   r   r   r   r   kwargs
key_statesvalue_statesattn_weightsattn_outputs               r5   eager_attention_forwardr      s     3 ; ;<JUF$?$?@L<<z';';Aq'ABWLL!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|\:K''1-88:K$$r6   c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..Nr:   r9   r   )rA   r,   r   )r   x1x2s      r5   rotate_halfr      sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r6   rotary_pos_embc                     |j                  |      }|j                  |      }| |z  t        |       |z  z   }||z  t        |      |z  z   }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer   )qkr   r   unsqueeze_dimq_embedk_embeds          r5   apply_rotary_pos_embr      sY    & --
&C
--
&C3w;q>C/0G3w;q>C/0GGr6   c                   F    e Zd ZdZ	 	 	 ddededz  dedz  dedz  f fdZ	 	 	 ddej                  ded	ej                  dz  d
e	dz  de
ej                  ej                  f   dz  dee   de
ej                  ej                  dz  e
ej                     dz  f   fdZ xZS )Zamba2AttentionaZ  
    Multi-headed attention from 'Attention Is All You Need' paper.

    Adapted from transformers.models.mistral.modeling_mistral.MistralAttention:
    The input dimension here is attention_hidden_size = 2 * hidden_size, and head_dim = attention_hidden_size // num_heads.
    The extra factor of 2 comes from the input being the concatenation of original_hidden_states with the output of the previous (mamba) layer
    (see fig. 2 in https://huggingface.co/papers/2405.16712).
    Additionally, replaced
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) with
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim/2)
    Finally, this attention layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this
    layer is tied, un-tied adapters (formally the same as LoRA but used in the base model) modules are added to the q, k, v projectors to increase
    expressivity with a small memory overhead (see Fig. 2 of https://huggingface.co/papers/2411.15242).
    Nrb   	layer_idxnum_fwd_mem_blocksblock_idc           	         t         |           || _        || _        |j                  | _        |j
                  | _        |j                  |j                  z  | _	        |j                  | _
        | j                  dz  dz  | _        d| _        |j                  | _        t        j                  |j                  |j                  | j                  z  d      | _        t        j                  |j                  |j                  | j                  z  d      | _        t        j                  |j                  |j                  | j                  z  d      | _        t        j                  |j                  | j                  z  |j&                  d      | _        || _        |j,                  | _        || _        |j2                  rt        j4                  g       | _        t        j4                  g       | _        t        j4                  g       | _        t=        | j*                        D ]  }||j>                  z  |k(  r{t        j@                  t        j                  | j                  | j                  jB                  d      t        j                  | j                  jB                  | j                  d            }t        j@                  t        j                  | j                  | j                  jB                  d      t        j                  | j                  jB                  | j                  d            }t        j@                  t        j                  | j                  | j                  jB                  d      t        j                  | j                  jB                  | j                  d            }n<t        jD                         }t        jD                         }t        jD                         }| j6                  jG                  |       | j8                  jG                  |       | j:                  jG                  |       ! tI        | j.                        D 	
ci c]  \  }	}
|
|	
 c}
}	| _%        y c c}
}	w )Nr9   g      TFbias)&r)   r*   rb   r   attention_hidden_sizeattention_head_dimru   ry   r   r   rh   r   	is_causalattention_dropoutr   Linearq_projk_projv_projr2   o_projr   hybrid_layer_idslayer_block_mapr   use_shared_attention_adapter
ModuleListlinear_q_adapter_listlinear_k_adapter_listlinear_v_adapter_listrangenum_mem_blocks
Sequentialadapter_rankIdentityappend	enumerate	layer_dic)r1   rb   r   r   r   ilinear_q_adapterlinear_k_adapterlinear_v_adapterindexr   r4   s              r5   r*   zZamba2Attention.__init__   sJ    	"%+%A%A"11$*$>$>&B\B\$\!'-'E'E$)d2!'!9!9ii < <f>X>X[_[h[h>hotuii < <f>X>X[_[h[h>hotuii < <f>X>X[_[h[h>hotuii : :T]] JFL^L^ejk"4%66 ..)+r):D&)+r):D&)+r):D&4223 Dv,,,8')}}		$"<"<dkk>V>V]bc		$++":":D<V<V]bc($ (*}}		$"<"<dkk>V>V]bc		$++":":D<V<V]bc($ (*}}		$"<"<dkk>V>V]bc		$++":":D<V<V]bc($
 (*{{}$'){{}$'){{}$**112BC**112BC**112BC)D, <ETEYEY;Z[<5%%,[[s   Q6rF   r   past_key_valuesposition_embeddingsr   rW   c                    |j                   d d }g |d| j                  }| j                  |      }	| j                  |      }
| j	                  |      }| j
                  j                  rW| j                  |   }|	 | j                  |   |      z   }	|
 | j                  |   |      z   }
| | j                  |   |      z   }|	j                  |      j                  dd      }	|
j                  |      j                  dd      }
|j                  |      j                  dd      }| j
                  j                  r|\  }}t        |	|
||      \  }	}
||j                  |
||      \  }
}t!        j"                  | j
                  j$                  t&              } || |	|
||f| j(                  sdn| j*                  | j,                  d|\  }} |j.                  g |d j1                         }| j3                  |      }||fS )Nr:   r#   r9           )r   r   )rA   ru   r   r   r   rb   r   r   r   r   r   rB   r   use_mem_roper   updater   get_interface_attn_implementationr   r   r   r   r   r   r   )r1   rF   r   r   r   r   r   input_shapehidden_shapequery_statesr   r   adapter_layer_idxr   r   attention_interfacer   r   s                     r5   rN   zZamba2Attention.forward,  s    $))#2.88b8$--8{{=1[[/
{{=1;;33 $y 9'*W$*D*DEV*WXe*ffL#&Sd&@&@AR&STa&bbJ'*W$*D*DEV*WXe*ffL#((6@@AF__\2<<QB
#((6@@AF;;##*HC';L*VY[^'_$L*&'6'='=j,Xa'b$J(?(M(MKK,,.E)
 %8	%
  $}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((r6   r   )rP   rQ   rR   __doc__r$   r   r*   r,   r^   r
   r[   r   r   rN   rS   rT   s   @r5   r   r      s    $ !%)-#6\6\ :6\  $J	6\
 *6\x /3(,HL1)||1) 1) t+	1)
 1) #5<<#=>E1) +,1) 
u||U\\D0%2E2LL	M1)r6   r   input_tensorpad_sizec                     t        | j                        dk(  r
ddddd|ddfnddd|ddf}t        j                  j                  j                  | |dd      S )z
    Padding x tensor with `pad_size` on the seq_len dim (dim=1)

    Assumes that we only have tensors of either size 4 or 3
       r   constant)moder   )lenrA   r,   r   r?   pad)r   r   	pad_shapes      r5   pad_tensor_by_sizer   c  sf     47|7I7I3Ja3OAq!Q!Q/VWYZ\]_gijlmUnI88""<ST"UUr6   c                    t        | |      } t        | j                        dk(  r.| j                  | j                  d   d|| j                  d         S | j                  | j                  d   d|| j                  d   | j                  d         S )z
    Padding input_tensor with `pad_size` on the seq_len dim (dim=1) and
    simultaneously splitting it into chunk sequences.

    Assumes that we only have tensors of either size 4 or 3
    r   r   r:   r9   )r   r   rA   r   )r   r   
chunk_sizes      r5   reshape_into_chunksr   n  s     &lH=L
<!###L$6$6q$92z<K]K]^_K`aa ##q!2z<3E3Ea3H,J\J\]^J_
 	
r6   c                 "   | j                  d      } | d   j                  g | j                         | } t        j                  t        j                  ||| j
                  t        j                        d      }| j                  | d      } t        j                  | d      }t        j                  t        j                  ||| j
                  t        j                        d      }|j                  | t        j                         }|S )zo
    More stable segment sum calculation. Uses cumulative sums and masking instead of direct subtractions.
    r:   .Nrw   )diagonalr   r   )
sizer   r,   trilr-   rp   boolmasked_fillcumsuminf)r   r   masktensor_segsums       r5   segment_sumr    s     ""2&J 2<	*11S<3D3D3FS
SL::ejjZ@S@S[`[e[efqstD++TE15LLL26M ::ejjZ@S@S[`[e[efqrsD!--teeiiZ@Mr6   c                        e Zd ZdZddededz  f fdZ	 	 ddej                  de	dz  dej                  dz  fd	Z
dde	dz  dej                  dz  fd
Z ed      	 	 dde	dz  dej                  dz  fd       Z xZS )Zamba2MambaMixeru  
    Compute ∆, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    ∆, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    Nrb   r   c           	      &   t         |           || _        |j                  | _        |j                  | _        |j                  | _        t        |j                  | j                  z        | _
        || _        |j                  | _        d| _        t        j                         | _        |j"                  | _        |j$                  | _        |j(                  | _        | j                  j,                  | _        |j0                  | _        |j2                  | _        |j4                  | _        |j6                  | _        | j                  d| j&                  z  | j
                  z  z   | _        t        j:                  | j8                  | j8                  d|j                  | j8                  |j                  dz
        | _        | j                  | j8                  z   | j.                  z   }t        j>                  | j                  ||j@                        | _!        t        jD                  tG        jH                  | j.                              | _%        tG        jL                  d| j.                  dz         }t        jD                  tG        jN                  |            | _(        tS        | j                  | j                  | j&                  z  d      | _*        t        jD                  tG        jH                  | j.                              | _+        t        j>                  | j                  | j                  |j@                        | _,        |jZ                  r|t]        d	      }t_        |d
d       a0t_        |dd       a1t]        d      }te        |d      a3te        |d      a4te        |d      a5tm        tf        th        tj        tb        t`        f      a7nd a0d a1d a3d a4d a5da7t_        |dd      rtn        stp        js                  d       |jt                  |   | _;        y )Nr@   r9   Tr#   )in_channelsout_channelsr   kernel_sizegroupspaddingr   gh㈵>)r0   r3   zcausal-conv1dcausal_conv1d_updatecausal_conv1d_fnz	mamba-ssmz8ops.triton.selective_state_update.selective_state_update)chained_pathz1ops.triton.ssd_combined.mamba_chunk_scan_combinedz8ops.triton.ssd_combined.mamba_split_conv1d_scan_combinedFuse_mamba_kernelsa  The fast path is not available because one of `(selective_state_update, causal_conv1d_fn, causal_conv1d_update)` is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d)<r)   r*   rb   r2   mamba_d_statessm_state_sizemamba_d_convconv_kernel_sizer   mamba_expandintermediate_sizer   use_conv_bias
activationr   SiLUactuse_mem_eff_pathmamba_ngroupsn_groupsmamba_headdimru   n_mamba_heads	num_headsr   time_step_limittime_step_mintime_step_maxconv_dimConv1dconv1dr   add_bias_linearin_projr+   r,   r-   dt_biasrz   logA_logr&   normDout_projr  r   rx   r  r  r!   selective_state_updatemamba_chunk_scan_combined mamba_split_conv1d_scan_combinedallis_fast_path_availableloggerwarning_oncelayer_types
layer_type)r1   rb   r   projection_sizeAcausal_conv1d	mamba_ssmr4   s          r5   r*   zZamba2MambaMixer.__init__  sW   !--$22 & 3 3!$V%8%84;K;K%K!L"#11 779 & 7 7,,,,22 ++%55#11#11..T]]1BTEXEX1XXii++==''!+
 004==@4>>Qyy''
 ||EJJt~~$>? LLDNNQ./\\%))A,/
&""t/E/E/V\`
	 ejj89		$"8"8$:J:JQWQgQgh ##,_=M#*=:PRV#W &}6H$O(5I%<(b&" )@([)% 0G(b0, &)*-4$(&" $( #%)"(,%/3,%*"6.5>T> !,,Y7r6   rF   cache_paramsr   c                    |j                   \  }}}| j                  | j                  z  }d| j                  z  d| j                  z  | j                  z  z   | j                  z   }|d uxr |j                  | j                        }	|	rL|j                  | j                     j                  d   }
|j                  | j                     j                  d   }|	r,|dk(  r&| j                  |j                  d            }|j                   d   |z
  dz  }||| j                  | j                  | j                  g}t        j                  ||d      \  }}}}}t        |
| j                   j"                  j                  d      | j                   j$                  | j&                        }t        j                  || j                  ||gd      \  }}}t        j(                  | j*                  j-                                }|d d d df   d d d d d f   j/                  d| j0                  | j                        j3                  t        j4                        }|d d d d d f   j/                  dd| j0                        }| j6                  d d d df   j/                  d| j0                        }| j8                  d d d df   j/                  d| j0                        }|j;                  || j                  |j                   d   | j                  z        }|j;                  || j                  |j                   d   | j                  z        }|j;                  || j                  | j0                        }t=        ||||||d |d	
      }|j;                  || j                  | j0                  z        }| j?                  ||      }| jA                  |j3                  | j@                  j"                  jB                              d d d df   }|S |Bt        jD                  |dk(        s*|jB                  }||d d d d d f   z  j3                  |      }| j                  |      }t        j(                  | j*                  j-                                }| jF                  i nd
| jF                  i}|t        jD                  |dk(        }nd}| jH                  r| jJ                  r||rtM        || j                   j"                  j                  d      | j                   j$                  | j6                  |f| j8                  | jN                  d | j&                  | j>                  j"                  | j>                  jP                  | j@                  j"                  | j@                  j$                  | j0                  | j                  ddd|\  }}|S t        j                  || j                  | j                  | j                  gd      \  }}}|jS                  dd      }|	rt        jT                  
|gd      }|XtV        jX                  j[                  || j\                  |j                   d   z
  df      }|j_                  || j                         t`        | j&                  dvr5| jc                  | j!                  |      dd |j                   d   f         }nPta        || j                   j"                  j                  d      | j                   j$                  | j&                        }|	r|d d d d | d f   }|jS                  dd      }t        j                  || j                  ||gd      \  }}}|Bt        jD                  |dk(        s*|jB                  }||d d d d d f   z  j3                  |      }te        |j;                  ||d| j0                        |||j;                  ||| j                  d      |j;                  ||| j                  d      f| jN                  | j8                  d d d| j6                  d|	rnd d|\  } }|||jg                  || j                         | j;                  ||d      } | j?                  | |      } | jA                  | j3                  | j@                  j"                  jB                              }|S )Nr9   r   r#   r:   r   .rv   T)zr1  dt_softplusdt_limitF)r5  r   seq_idxr   rmsnorm_weightrmsnorm_epsoutproj_weightoutproj_biasheaddimngroupsnorm_before_gatereturn_final_states)r@   swish)r   r.   r   r   )r   r5  rF  rI  rQ  r1  rG  initial_states)4rA   r%  r  r  r(  has_previous_stater   layersconv_statesrecurrent_statesr0  squeezer,  r,   splitr  r.  r.   r   r   expr3  r]   r   ru   r=   r>   r1  r5  rB   r7  r4  r6  r<   r:  r)  r#  r   r9  r   r/   r   r   r   r?   r   r  update_conv_stater  r"  r8  update_recurrent_state)!r1   rF   rD  r   
batch_sizerr   _groups_time_state_sized_to_removeuse_precomputed_states
conv_staterecurrent_statein_projected_statesd_mlpsplit_projection_dimrG   hidden_states_B_CdtBCrA  r1  r5  hidden_states_reshapedoutr<   projected_statesdt_limit_kwargsinput_not_masked	ssm_state	time_stepnew_conv_statescan_outputs!                                    r5   cuda_kernels_forwardz%Zamba2MambaMixer.cuda_kernels_forward  s    "/!4!4
GQ!%1D1D!D$0001t}}3DtGZGZ3ZZ]a]k]kk!-T!9!ml>]>]^b^l^l>m!%,,T^^<HHKJ*11$..ARRSTUO "gl"&,,}/D/DQ/G"H(..r2[@QFE$)5$2H2H$--Y]YgYg#h 05<OQekm0n-Aq$)2 4!""**1-  ! #(++!'')?AWX#M1a
 4::++-..A!T3,1d
+222t}}dFYFYZ]]didqdq]rAAq$J&&r2t}}=Bll1dC<077DMMJGq$|$++B>Az4==!''!*2MNAz4==!''!*2MNA%2%7%7
DNNTXTaTa%b"2& M *..z4>>DMM;YZM IImT:M -- 0 01E1E1K1K LMaQUWZl[CL 
G )%))Na<O2P%++!.1d
1K!K O OPU V#||M:4::++-..A$($8$8$@bzSWSgSgFhO)#(99^q-@#A #' $$<;OTd!A$KK&&..q1KK$$LL" ff# ##'99#3#3 $		 : :#'==#7#7!%!3!3 MM MM%*(,#"$ &%"Yj 
 6;[[$++T]]DNNK62' %6$?$?1$E!) ).		:?P2QWY(Z%+%']]%6%6)D,A,ADUD[D[\^D_,_ab+c&N !22>4>>R#+tFW/W(,=N1OPSUrWhWnWnoqWrUrPr1s(t%(8+#{{1199!<![[--#'??	)% *(9!Q	/(J%$5$?$?1$E!&+kk%++-CE[\'#q!
 "-eiiRS@S6T)//E%2^Aq$J5O%O$S$STY$ZM)B!&&z7BNFF:wrBFF:wrB*  $ff (, LL $6L?RV* &*&Y  (\-E 77	4>>R)..z7BG"iiT: mmKNN4==3G3G3M3M$NO
r6   c                 4   |j                   \  }}}|j                  }|-|j                  | j                        r| j	                  |      }n1|||d d d d d f   z  j                  |      }| j	                  |      }|j                   d   d| j                  z  z
  d| j                  z  | j                  z  z
  | j                  z
  dz  }	|j                  |	|	| j                  | j                  | j                  gd      \  }}}
}}|j                  dd      }|d uxr |j                  | j                        }|r&|j                  | j                     j                  d   }|r|dk(  r|j                  || j                        d| j                    d f   }t#        j$                  || j&                  j(                  d d dd d f   z  d      }| j*                  r|| j&                  j,                  z  }| j/                  |      j                  |      d d d df   }n|rt#        j0                  |gd      }|jt2        j4                  j7                  || j                   |j                   d   z
  df      }|j                  || j                        d| j                    d f   }| j/                  | j'                  |      dd |j                   d   f   j                  dd            }|r|d d | d d d f   }|*|j                  }||d d d d d f   z  j                  |      }t#        j                  || j                  | j                  | j                  z  | j                  | j                  z  gd      \  }}}t#        j8                  | j:                  j=                                }|ri|dk(  rc|j>                  dk(  r
|d d d df   n|d d dd d f   d d d df   }|j                  dd      jA                  ||j                   d   | jB                        }| jD                  d   jA                  | jD                  j                   d   | jB                        }t"        j2                  j4                  jG                  ||j                  |j                        z         }t#        jH                  || jJ                        }|d   jA                  | j                  | jB                  | j                        j                  t"        jL                  	      }t#        j8                  |d   |z        }|jO                  || j                  d      dd d d f   }|jA                  || j                  | j                  | j                  z  |j                   d         jQ                         }|jO                  |d|j                   d         }|d   |dd d d f   z  }|jO                  |d| jB                        }||d   z  }|j                  | j                     jR                  d   jU                         }||z  |z   }|jW                  || j                        }|jO                  || j                  d      dd d d f   }|jA                  || j                  | j                  | j                  z  |j                   d         jQ                         }|jO                  |d|j                   d         }|j                  |j                        }|jY                  || j                  z  | jB                  | j                        }|jY                  || j                  z  | j                  d      }t#        jZ                  ||      }|jY                  || j                  | jB                        }| j\                  d   jA                  | j\                  j                   d   | jB                        }|||z  z   j                  |j                        }|jO                  |d      d d d df   }nLt2        j4                  jG                  || jD                  z         }t#        jH                  || jJ                        }|jO                  ||d| jB                        j=                         }|jO                  ||d| j                        j=                         }|jO                  ||d| j                        j=                         }|j_                  | j                  | j                  z  d| j                  
      }|j_                  | j                  | j                  z  d| j                  
      }| j`                  || j`                  z  z
  | j`                  z  }| j\                  d   tc        ||      z  }||d   z  }|j                  |j                        |z  }||||fD cg c]  }te        ||| j`                         c}\  }}}}|jg                  dddd      }t#        jh                  |d      }t#        j8                  tk        |            } |d d d d d d d d d d d f   |d d d d d d d d d d d f   z  }!|!j%                  d      }"|"d   | jg                  ddddd      d   z  }#|#j%                  d      }$|$d   |d d d d d f   z  j%                  d      }%t#        j8                  |d d d d d d dd f   |z
        }&||&jg                  dddd      d   z  }'|'jg                  ddddd      d   |jg                  ddddd      dd d d f   z  j%                  d      jg                  ddddd      }(|rR|j                  | j                     jR                  d   d d d f   j                  |(j                  |(jl                        nt#        jn                  |(d d d df         })t#        j0                  |)|(gd      }(t#        j8                  tk        t2        j4                  j7                  |d d d d d d df   d                  }*|(jg                  ddddd      }+|*d   |+d d d d d df   z  j%                  d      },|,jg                  ddddd      }-|-d d d df   |-d d df   }.}(t#        j8                  |      }/|dd d d f   |(d d d d d df   z  }0|/jg                  dddd      }1|0j%                  d      |1d   z  }2|%|2z   }|jO                  |d| j                  | jB                        }||z   }|dkD  r|d d d |d d d d f   }|jO                  ||d      }|.||jW                  |.| j                         | jq                  ||
      }3| js                  |3j                  |            }4|4S c c}w )Nr:   r9   r   r#   r   .r  ).NNrv   )r}   output_sizer   r   )r<   rp   )r#   r   ):rA   r<   rT  r   r0  r=   r  r%  r  r(  rY  r,  r   rU  rV  r[  r  r,   sumr.  r.   r  r   r"  r   r   r?   r   rZ  r3  r]   ndimr   ru   r1  softplusclampr*  r>   r   r   rW  ro   r\  rB   bmmr5  repeat_interleaver   r   r   permuter  r  rp   
zeros_liker4  r6  )5r1   input_statesrD  r   r]  rr   r^  r<   rm  re  rG   rF   rh  use_precomputed_staterb  rV  ri  rj  rA  r1  dAdBdBx
ssm_statesssm_states_reshaped
C_reshapedyr5  r   
D_residualtA_cumsumLG_intermediateGM_intermediateMY_diagdecay_statesB_decay_contractionstatesprevious_statesdecay_chunkstates_permutedresult
new_statesrp  state_decay_outC_times_statesstate_decay_out_permutedY_offrs  contextualized_statess5                                                        r5   torch_forwardzZamba2MambaMixer.torch_forward  s   !-!3!3
GQ""#(G(G(W#||L9) ,~aDj/I IMMeT#||L9!''+a$2H2H.HH1t}}K\_c_r_rKrrtx  uC  uC  C  HI  I(8(>(>t55t~~V\^ )? )
%1dM2 &//15 ,D 8 l\=\=\]a]k]k=l %,,T^^<HHKJ !W\&88WX[^b^s^s]s]tXtuK!IIkDKK4F4Fq!Qw4O&OUWXM!!!1!11 HH]366u=aslKM$ %		:}*E2 N' mm//!**]-@-@-DDaH +<<[$..YZ]`d`u`u_u_vZvw HHT[[%?E]mFYFYZ\F]E]@]%^%h%hijlm%noM$ -a'Ao >)%++!.1d
1K!K O OPU V#kk-$:P:PRVR_R_bfbubuRuw{  xE  xE  HL  H[  H[  x[  :\  bd  eq!YYtzz'')** W\ &(WW\AtSL!r!Q'{1dC<7PBa#**:rxx|T]]SBll9-44T\\5G5G5JDMMZG$$--b7::bhh3G.GHBR!3!34B/"))$..$--I\I\]``glgtgt`uA2i=1,-B
 		*dmmR8dAFAT]]DNNdmm4SUVU\U\]_U`allnA		*b!''"+6AI3a<0B *11*b$--PM}Y//C &,,T^^<MMaPVVXJ#b3.J%<<ZXJ 		*dmmR8dAFAT]]DNNdmm4SUVU\U\]_U`allnA		*b!''"+6A $qww/J",//*t~~2Mt}}^b^q^q"r
T^^ ;T=P=PRSTJ		-z:Az4>>4==AA y!((a$--HA]Q&&**1773A 		*b)!T3,7A ''T\\(9:BR!3!34B)11*gr4==Y__aM		*gD4G4GHNNPA		*gr43F3FGMMOA##DNNdmm$CX\XfXf#gA##DNNdmm$CX\XfXf#gA'DOO*CCtVH	*-?x-XXJ *ByM9M](()B.A cpqrtuwxay%z\]&9!Xt&W%z"M1a 		!Q1%A||A2.H 		+a.)A q!Qa23a1dAq!8K6LLN""r"*A y\AIIaAq!,DY,OON""r"*A 	l]1a:%>>CCAFF !99XaArsl%;h%FGL"#l&:&:1aA&Fy&Q"Q)11!Q1a@K}OdOdefhiklnoqrOstwy}  @A  uA  PB  B  G  G  LM  G  N  V  V  WX  Z[  ]^  `a  cd  eF ) ##DNN3DDQG4PSSZ`ZfZfouo|o|S}%%fQUm4 
 YY8a@F))K0A0A(1aQRTV;BWY_0`$abK$nnQ1a;O!/2_Q4QT_5UUZZ_`ZaF1aA6J *1crc6 2Jq"u4EIF $ii1OT1oq!T30GGN'6'>'>q!Q'J$#''+.Fy.QQE A		*b$..$--HAJA!|a'1a'(		*gr2A$)A33It~~Nii4(
 !%knnU.C D$$K &{s   vr.  c                     t         rId| j                  j                  j                  j                  v rt               s| j                  |||      S | j                  |||      S )Ncuda)r;  r0  r.   rp   r   r   rt  r  )r1   rF   rD  r   r   s        r5   rN   zZamba2MambaMixer.forwardm  sT     "f0C0C0J0J0O0O&OXpXr,,]L.YY!!-~NNr6   r(   NN)rP   rQ   rR   r   r$   r   r*   r,   r^   r
   rt  r  r   rN   rS   rT   s   @r5   r  r    s    d8| d8d
 d8R &*.2	e||e dle t+	eP~% ~%[`[g[gjn[n ~%B H% &*.2	
O dl
O t+	
O &
Or6   r  c                   8     e Zd Zddededz  f fdZddZ xZS )	Zamba2MLPNrb   r   c           	          t         	|           || _        |j                  | _        |j                  | _        || _        || _        t        j                  | j                  d| j                  z  |j                        | _
        t        j                  | j                  | j                  |j                        | _        t        |j                     | _        t        j                  g       | _        t#        | j
                        D ]  }||j$                  z  |k(  rt        j&                  t        j                  | j                  j                  | j                  j(                  d      t        j                  | j                  j(                  d| j                  z  d            }nt        j*                         }| j                   j-                  |        |j.                  }t1        |      D ci c]  \  }}||
 c}}| _        yc c}}w )aQ  
        This MLP layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this layer
        is tied, un-tied adapter modules (formally same as LoRA, but used in the base model) are added to the up and gate projectors to increase expressivity with a small memory overhead.
        r9   r   FN)r)   r*   rb   r2   r  r   r   r   r   r/  gate_up_proj	down_projr	   
hidden_actact_fnr   gate_up_proj_adapter_listr   r   r   r   r   r   r   r   r   )
r1   rb   r   r   r   gate_up_proj_adapterr   r   r   r4   s
            r5   r*   zZamba2MLP.__init__|  s   
 	!--!'!9!9"4 IId&6&6D<R<R8RY_YoYop4#9#94;K;KRXRhRhiV../)+r):&t../ 	HA6(((H4')}}IIdkk55t{{7O7OV[\IIdkk66D<R<R8RY^_($
 (*{{}$**112FG	H !11;D_;UV<5%%,VVs   3H
c                     | j                  |      }| j                  |   }| | j                  |   |      z   }t        j                  |dd      }| j                  |d         |d   z  }| j                  |      }|S )Nr9   r:   r   r   r#   )r  r   r  r,   chunkr  r  )r1   hidden_stater   gate_up_stateoutputs        r5   rN   zZamba2MLP.forward  s    )),7NN9-	%(Q(F(Fy(QR^(__M1"={{=#34}Q7GG-r6   r  r(   )rP   rQ   rR   r$   r   r*   rN   rS   rT   s   @r5   r  r  {  s%    W| WPSVZPZ W<r6   r  c                        e Zd Zddededz  dedz  f fdZ	 	 	 ddej                  dej                  dedej                  dz  d	edz  d
ej                  dz  de
e   deej                     fdZ xZS )Zamba2AttentionDecoderLayerNrb   r   r   c                 @   t         |           || _        t        |j                        }t        |d||      | _        t        |||      | _        t        |j                  |j                        | _        t        |j                  |j                        | _        y )Nr:   )r   r   r   )r   r   r3   )r)   r*   r   r   r   r   	self_attnr  feed_forwardrV   r   rms_norm_epsinput_layernormr2   pre_ff_layernorm)r1   rb   r   r   num_gsr4   s        r5   r*   z$Zamba2AttentionDecoderLayer.__init__  s     V,,-(2RXckl%fRZ[,V-I-IvObObc -f.@.@fFYFY Zr6   rF   original_hidden_statesr   r   r   r   rW   c           	          t        j                  ||gd      }| j                  |      } | j                  d|||||d|\  }}| j	                  |      }| j                  ||      }|S )a  
        Args:
            hidden_states (`torch.FloatTensor`): output of previous Mamba layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output of shape `(batch, seq_len, embed_dim)`.
                This is concatenated with `hidden_states` (which is the output of the previous (mamba) layer). The
                concatenated tensor is then used as input of the pre-attention RMSNorm
                (see fig. 2 in https://huggingface.co/papers/2405.16712).
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        r:   r   )rF   r   r   r   r    )r,   concatenater  r  r  r  )	r1   rF   r  r   r   r   r   r   r^  s	            r5   rN   z#Zamba2AttentionDecoderLayer.forward  s    6 ))=:P*QWYZ,,];)4>> 
')+ 3
 
q --m<))-Cr6   r  r   )rP   rQ   rR   r$   r   r*   r,   r^   r
   
LongTensorr   r   r[   FloatTensorrN   rS   rT   s   @r5   r  r    s    [| [sTz [UX[_U_ [ /3(,7;)||) !&) 	)
 t+) ) #--4) +,) 
u  	!)r6   r  c                   r    e Zd Zdedef fdZ	 	 	 	 	 	 	 	 ddej                  dej                  dz  dedz  dej                  dz  dej                  dz  d	edz  d
e	dz  dej                  dz  dej                  dz  dee   deej                  eej                  ej                  f   dz  f   fdZ xZS )Zamba2MambaDecoderLayerrb   r   c                     t         |           t        ||      | _        t	        |j
                  |j                        | _        || _        y )N)rb   r   r  )	r)   r*   r  mambarV   r2   r  r  r   )r1   rb   r   r4   s      r5   r*   z Zamba2MambaDecoderLayer.__init__  s>    %VyI
,V-?-?VEXEXY"r6   NrF   r  r   causal_maskr   	use_cacher   transformer_hidden_statesr   rW   c
                 t    |}|	||	z   n|}| j                  |      } | j                  d|||d|
}||z   }|S )aX  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
        )rF   rD  r   r  )r  r  )r1   rF   r  r   r   r  r   r  r   r  r   residuals               r5   rN   zZamba2MambaDecoderLayer.forward  sn    0 !
 :S9^M55dq 	 ,,];"

 
'()
 	
 !=0r6   NNNNNFNN)rP   rQ   rR   r$   r   r*   r,   r^   r
   r  r  r   r   r[   r  rN   rS   rT   s   @r5   r  r    s   #| # # 7; $.2+/(,!&049=*||* !&t 3* :	*
 t+* \\D(* * $;* &&-* $)<<$#6* +,* 
u  %(9(95;L;L(L"MPT"TT	U*r6   r  c                       e Zd Zdedej
                  def fdZ	 	 	 	 	 	 	 	 ddej                  dej                  dz  de
dz  d	ej                  dz  d
ej                  dz  dedz  dedz  dej                  dz  dej                  dz  dee   deej"                  eej"                  ej"                  f   dz  f   fdZ xZS )Zamba2HybridLayershared_transformerlinearr  c                 L    t         |           || _        || _        || _        y r(   )r)   r*   r  mamba_decoderr  )r1   r  r  r  r4   s       r5   r*   zZamba2HybridLayer.__init__  s'     	""4r6   NrF   r  r   r   r  r   r  r   r   r   rW   c
           
           | j                   |f||||||	d|
}| j                  |      } | j                  |f|||||d|
}|S )ap  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output that will be concatenated with
            hidden activations to form the input of the shared transformer layer.
            layer_idx (`int`): layer number.
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_values (`Cache`, *optional*): cached past key and value projection states
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        )r  r   r   r   r   r   )r  r   r   r  r   )r  r  r  )r1   rF   r  r   r   r  r   r  r   r   r   r  s               r5   rN   zZamba2HybridLayer.forward  s    < %<D$;$;	%
#9&+ 3%	%
 	%
! %)KK0I$J!***
&?)+ 3
 
 r6   r  )rP   rQ   rR   r  r   r   r  r*   r,   r^   r   r
   r  r  r   r   r[   r  rN   rS   rT   s   @r5   r  r    s"   5"=5GIyy5Yp5 7; $.2+/(,!&7;044||4 !&t 34 :	4
 t+4 \\D(4 4 $;4 #--44 &&-4 +,4 
u  %(9(95;L;L(L"MPT"TT	U4r6   r  c                        e Zd ZU eed<   dZdZddgZdgZdZ	dZ
dZdZeedZ ej"                          fd       Z xZS )	Zamba2PreTrainedModelrb   modelTr  r  r   )rF   
attentionsc                    t         |   |       t        |t              rt	        j
                  t	        j                  | j                  j                        t        j                  | j                  j                        t        j                  | j                  j                        z
  z  t        j                  | j                  j                        z         j                  | j                  j                        }|t	        j                  t	        j                  |              z   }t!        j"                  |j$                  |       t	        j&                  d|j(                  dz         }t!        j"                  |j*                  t	        j                  |             t!        j,                  |j.                         y y )N)minr#   )r)   _init_weightsr   r  r,   rZ  randrb   r'  mathr2  r+  r*  rz  time_step_floorexpm1initcopy_r1  rz   r(  r3  ones_r5  )r1   r   rh  inv_dtrA  r4   s        r5   r  z#Zamba2PreTrainedModel._init_weights_  s(   f%f./

4;;44588DKK556$++B[B[9\\^((4;;4456 e33e4	  %))U[["%5$566FJJv~~v.Q 0 01 45AJJv||UYYq\2JJvxx  0r6   )rP   rQ   rR   r$   r   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_flex_attn_supports_sdpa_is_statefulr  r   _can_record_outputsr,   r   r  rS   rT   s   @r5   r  r  O  sj    &*#,.GH#4"5NL0%
 U]]_! !r6   r  c                       e Zd ZdZdef fdZeee	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  dedz  d	e	j                  dz  d
edz  dee   deez  fd                     Zd Z xZS )Zamba2Modelzh
    Model consisting of *config.num_hidden_layers* layers.

    Args:
        config: Zamba2Config
    rb   c                 L   t         |   |       || _        |j                  | _        |j
                  | _        t        j                  |j
                  |j                  | j                        | _	        |j                  | _
        | j                         | _        |j                  | _        t        |j                  |j                        | _        |j"                  r1|j$                  rt&        j)                  d       t+        |      | _        d| _        | j1                          y )Nr  ze`use_long_context` set to `True`: using rescaled `rope_theta` and extended `max_position_embeddings`.F)r)   r*   rb   pad_token_idpadding_idx
vocab_sizer   	Embeddingr2   embed_tokenslayers_block_type
get_layersrU  r   rV   r  final_layernormr   use_long_contextr<  r=  r`   
rotary_embgradient_checkpointing	post_initr1   rb   r4   s     r5   r*   zZamba2Model.__init__z  s     !.. ++LL):):F<N<NPTP`P`a!'!9!9oo'$*$?$?!,V-?-?VEXEXY&&##{ 4F;DO&+# 	r6   N	input_idsr   r   r   inputs_embedsr  r   rW   c           	         |d u |d uz  rt        d      || j                  |      }|}t        j                  |      }	|r|t	        | j
                        }|V||j                         nd}
t        j                  |j                  d   |j                        |
z   }|j                  d      }t        | j
                  ||||      }| j
                  j                  r| j                  ||      }nd }t        | j                        D ]  \  }} |||	|||f||||d|} | j!                  |      }t#        ||r|	      S d 	      S )
NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either one)rb   r   r#   rp   )rb   r  r   r   r   )r   )r   r  r   r   )last_hidden_stater   )
ValueErrorr  r,   ro   r   rb   get_seq_lengthrz   rA   rp   r   r   r   r  r   rU  r  r   )r1   r  r   r   r   r  r  r   rF   r  past_seen_tokensr  r   r   layers                  r5   rN   zZamba2Model.forward  s    -t";<s    --i8M%!&]!; 0*$++>OCRC^==?de <<(;(;A(>}G[G[\_ooL'11!4L(;;')+%
 ;;##"&//-l/"["& )$++ 6 	Iu!& !0#$7) M	 ,,];&+/8O
 	
>B
 	
r6   c                     g }i | _         d| _        g }t        | j                        D ]O  \  }}t	        | j
                  |      }|dk(  rd| d}t        |t              r"t        |      | j
                  j                  k\  rDt        |t              rt        |      }t        |      }| j                   j                  ||i       n|j                  |       || j
                  j                  z  }t        | j
                  |      }	t        j                   | j
                  j"                  | j
                  j"                  d      }
|j                  t%        |	|
|             ?|j                  |       R t        j&                  |      S )	Nr   )r   hybridzlayers.z.shared_transformer)r   Fr   )_tied_weights_keysfirst_transformer_layer_idr   r  r  rb   r   listr   r   r   nextr   r   r  r   r   r2   r  r   )r1   rU  unique_hybrid_blockslayer_idr?  mamba_layerprefix_patterntarget_patternr   
attn_blocklinear_layers              r5   r  zZamba2Model.get_layers  sQ   "$*+'!$-d.D.D$E 	+ Hj1$++RKX%#*8*4G!H ##7>/0DKK4N4NN!"6=/45I/J,%)*>%?N++22NN3ST )//?#dkk&@&@@8xX
!yy)@)@$++BYBY`ef/
L+VWk*3	+4 }}V$$r6   )NNNNNN)rP   rQ   rR   r   r$   r*   r    r"   r   r,   r  r^   r
   r  r  r   r   r[   r   rN   r  rS   rT   s   @r5   r  r  q  s    | .   .2.204(,26!%@
##d*@
 t+@
 &&-	@

 @
 ((4/@
 $;@
 +,@
 
(	(@
    @
D %r6   r  c                   N    e Zd ZddiZdef fdZee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  d	edz  d
ej                  dz  dej                  dz  dedz  deej                  z  dee   deez  fd              Z	 	 	 	 	 	 d fd	Z xZS )Zamba2ForCausalLMzlm_head.weightzmodel.embed_tokens.weightrb   c                     t         |   |       t        |      | _        |j                  | _        t        j                  |j                  |j                  d      | _        | j                          y NFr   )
r)   r*   r  r  r  r   r   r2   lm_headr  r  s     r5   r*   zZamba2ForCausalLM.__init__  sU      (
 ++yy!3!3V5F5FUS 	r6   Nr  r   r   r   r  labelsr  logits_to_keepr   rW   c	           
      b    | j                   d||||||d|	}
|
j                  }t        |t              rt	        | d      n|}| j                  |dd|ddf         }d}| | j                  ||| j                  fi |	}t        |||
j                  |
j                  |
j                        S )al  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Zamba2ForCausalLM

        >>> model = Zamba2ForCausalLM.from_pretrained("Zyphra/Zamba2-7B-v1")
        >>> tokenizer = AutoTokenizer.from_pretrained("Zyphra/Zamba2-7B-v1")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)r  r   r   r   r  r  Nlosslogitsr   rF   r  r  )r  r  r   r   slicer  loss_functionr  r   r   rF   r  )r1   r  r   r   r   r  r  r  r  r   outputsrF   slice_indicesr  r  s                  r5   rN   zZamba2ForCausalLM.forward  s    H ,64:: ,
)%+',
 ,
  118B>SV8W~ot4]kmA}a,?@A%4%% 	D &#33!//))
 	
r6   c           
      h    | j                   j                  |d<   t        
|   |f||||||d|}	|	S )Nr  )r   r   r  r   r  is_first_iteration)rb   num_logits_to_keepr)   prepare_inputs_for_generation)r1   r  r   r   r  r   r  r  r   model_inputsr4   s             r5   r!  z/Zamba2ForCausalLM.prepare_inputs_for_generationJ  sU     $(;;#A#A w<	
+)'%1	
 	
 r6   NNNNNNNr   )NNNNTF)rP   rQ   rR   r  r$   r*   r   r   r,   r  r^   r
   r  r  r   r   r   r[   r   rN   r!  rS   rT   s   @r5   r  r    s0   *,GH|   .2.204(,26*.!%-.@
##d*@
 t+@
 &&-	@

 @
 ((4/@
   4'@
 $;@
 ell*@
 +,@
 
'	'@
  @
J   r6   r  a  
    The Zamba2 Model with a sequence classification head on top (linear layer).

    [`Zamba2ForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-2) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    )custom_introc                   .    e Zd Zdef fdZee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de
dz  dej                  dz  d	ej                  dz  d
edz  deej                  z  dee   deez  fd              Z xZS )Zamba2ForSequenceClassificationrb   c                     t         |   |       |j                  | _        t        |      | _        t        j                  |j                  | j                  d      | _        | j                          y r  )
r)   r*   
num_labelsr  r  r   r   r2   scorer  r  s     r5   r*   z(Zamba2ForSequenceClassification.__init__s  sS      ++ (
YYv114??O
 	r6   Nr  r   r   r   r  r  r  r  r   rW   c	           	          | j                   |f|||||d|	}
|
d   }| j                  |      }||j                  d   }n|j                  d   }| j                  j                  |dk7  rt        d      | j                  j                  d}n||| j                  j                  k7  j                  |j                  t        j                        }t        j                  |j                  d   |j                  t        j                        }||z  j                  d      }n.d}t        j                  | j                  j                   d       |t        j                  ||j                  	      |f   }d}|! | j                   d|||| j                  d
|	}t#        |||
j$                  |
j&                  |
j(                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        )r   r   r   r  r  r   Nr#   z=Cannot handle batch sizes > 1 if no padding token is defined.r:   rw   z will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r  )r  r  pooled_logitsrb   r  r  )r  r)  rA   rb   r  r  r=   rp   r,   int32rz   argmaxr<  r=  r4   rP   r  r   r   rF   r  )r1   r  r   r   r   r  r  r  r  r   transformer_outputsrF   r  r]  last_non_pad_tokennon_pad_masktoken_indicesr+  r  s                      r5   rN   z'Zamba2ForSequenceClassification.forward|  s   ( 8Btzz8
)%+'8
 8
 ,A.M* "+J&,,Q/J;;##+
a\]];;##+!#"%)A)AAEEfmmUZU`U`aL!LL)<V]]Z_ZeZefM"/,">!F!Fr!J!#>>**+ ,Z Z
 u||Jv}}MOaab%4%% $V=Y]YdYdhnD 0 /??-;;*55
 	
r6   r#  )rP   rQ   rR   r$   r*   r   r   r,   r  r^   r
   r  r  r   r   r   r[   r   rN   rS   rT   s   @r5   r&  r&  d  s   |   .2.204(,26*.!%-.@
##d*@
 t+@
 &&-	@

 @
 ((4/@
   4'@
 $;@
 ell*@
 +,@
 
1	1@
  @
r6   r&  )r  r&  r  r  )r   )r#   )Tr  collections.abcr   	itertoolsr   typingr   r,   r    r   r  activationsr	   cache_utilsr
   r   
generationr   integrationsr   integrations.accelerater   integrations.hub_kernelsr   masking_utilsr   modeling_layersr   modeling_outputsr   r   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   r   utils.genericr   r    utils.import_utilsr!   utils.output_capturingr"   configuration_zamba2r$   
get_loggerrP   r<  Moduler&   rV   r`   r^   r   r   r]   r   r   r   r   r   r   r  r  r  r  r  r  r  r  r  r&  __all__r  r6   r5   <module>rJ     sT  *  $     & ! . ) 4 = 8 / 9 q q K F & l l G 9 5 . 
		H	%; ;*JBII J(><BII ><B	UU\\ 	U# 	U%,, 	U& %II%<<% 
% <<	%
 LL4'% % %2( *+ ,2y)bii y)~VU\\ VS V
((bOryy bOJ'		 'T3")) 3l18 1h=2 =@ !O ! !B D%' D% D%Pg- gT L
&; L
L
^ kr6   