Ë
    ÛÓ^jÕ¦  ã                   ó¬  — d dl mZ d dlZd dlmZ ddlmZ ddlmZ ddl	m
Z
mZ ddlmZ dd	lmZmZmZmZmZ dd
lmZ ddlmZmZ ddlmZmZ ddlmZmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1  e(jd                  e3«      Z4 ed«       G d„ dejj                  «      «       Z6d„ Z7 ed«      dDd„«       Z8dejr                  de:dejr                  fd„Z;	 dEd ejj                  d!ejr                  d"ejr                  d#ejr                  d$ejr                  dz  d%e<d&e<d'e#e%   fd(„Z= ee8«       G d)„ d*ejj                  «      «       Z> G d+„ d,ejj                  «      Z? G d-„ d.ejj                  «      Z@e G d/„ d0ejj                  «      «       ZA G d1„ d2ejj                  «      ZB G d3„ d4e«      ZC G d5„ d6e«      ZD G d7„ d8e!«      ZEeCeDd9œZFe& G d:„ d;eE«      «       ZG	 	 	 dFd<ejr                  eHejr                     z  dz  d=e:dz  d$ejr                  dz  dejr                  e:z  fd>„ZIe& G d?„ d@eEe«      «       ZJ G dA„ dBeeE«      ZKg dC¢ZLy)Gé    )ÚCallableN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úlazy_load_kernelÚuse_experts_implementationÚuse_kernel_forward_from_hubÚuse_kernel_func_from_hubÚuse_kernelized_func)Úforce_accelerate_hooks)Úcreate_causal_maskÚcreate_recurrent_attention_mask)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)Úresolve_internal_import)ÚOutputRecorderÚcapture_outputsé   )ÚJambaConfigÚRMSNormc                   óh   ‡ — e Zd Zddeddfˆ fd„Zdej                  dej                  fd„Zd„ Zˆ xZ	S )	ÚJambaRMSNormÚepsÚreturnNc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)z;
        JambaRMSNorm is equivalent to T5LayerNorm
        N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer'   Ú	__class__s      €ús/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/jamba/modeling_jamba.pyr+   zJambaRMSNorm.__init__;   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕó    Úhidden_statesc                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S )Né   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtor-   Úfloat32ÚpowÚmeanÚrsqrtr0   r/   )r1   r6   Úinput_dtypeÚvariances       r4   ÚforwardzJambaRMSNorm.forwardC   sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r5   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)Útupler/   Úshaper0   )r1   s    r4   Ú
extra_reprzJambaRMSNorm.extra_reprJ   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr5   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Úfloatr+   r-   ÚTensorrC   rG   Ú__classcell__©r3   s   @r4   r&   r&   9   s7   ø„ ñ$¨ð $¸$õ $ð; U§\¡\ð ;°e·l±ló ;öJr5   r&   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Nr9   r8   ©Údim)rF   r-   Úcat)ÚxÚx1Úx2s      r4   Úrotate_halfrV   N   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r5   Úrotary_pos_embc                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezerV   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds          r4   Úapply_rotary_pos_embra   U   sY   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐr5   r6   Ún_repr(   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r"   N)rF   ÚexpandÚreshape)r6   rb   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         r4   Ú	repeat_kvrj   o   so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr5   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t        || j                  «      }t        || j                  «      }	t        j                  ||j	                  dd«      «      |z  }
|�|
|z   }
t
        j                  j                  |
dt        j                  ¬«      j                  |j                  «      }
t
        j                  j                  |
|| j                  ¬«      }
t        j                  |
|	«      }|j	                  dd«      j                  «       }||
fS )Nr8   r   r9   ©rQ   r;   )ÚpÚtrainingr"   )rj   Únum_key_value_groupsr-   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr=   r<   r;   rq   rv   Ú
contiguous)rk   rl   rm   rn   ro   rp   rq   rr   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r4   Úeager_attention_forwardr�   {   sà   € ô ˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r5   c                   óÀ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 ddej                  dej                  dz  de	dz  d	e
e   d
eej                  ej                  dz  f   f
d„Zˆ xZS )ÚJambaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperÚconfigÚ	layer_idxc                 ó  •— t         ‰| �  «        || _        || _        t	        |d|j
                  |j                  z  «      | _        |j                  |j                  z  | _	        | j                  dz  | _
        |j                  | _        d| _        t        j                  |j
                  |j                  | j                  z  d¬«      | _        t        j                  |j
                  |j                  | j                  z  d¬«      | _        t        j                  |j
                  |j                  | j                  z  d¬«      | _        t        j                  |j                  | j                  z  |j
                  d¬«      | _        y )Nri   g      à¿TF©Úbias)r*   r+   r„   r…   Úgetattrr2   Únum_attention_headsri   rg   rw   rp   Úattention_dropoutÚ	is_causalr   ÚLinearÚq_projÚk_projÚv_projÚo_proj)r1   r„   r…   r3   s      €r4   r+   zJambaAttention.__init__˜   s,  ø€ Ü‰ÑÔØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆŒÜ—i‘i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^ÐejÔkˆ�r5   Nr6   ro   Úpast_key_valuesrr   r(   c                 óÞ  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	|� |j                  ||	| j                  «      \  }}	t        j                  | j                  j                  t        «      }
 |
| |||	|f| j                  sdn| j                  | j                   dœ|¤Ž\  }} |j"                  g |¢d‘­Ž j%                  «       }| j'                  |«      }||fS )Nr9   r"   r8   ç        )rq   rp   )rF   ri   rŽ   Úviewry   r�   r�   Úupdater…   r   Úget_interfacer„   Ú_attn_implementationr�   rv   r‹   rp   re   r|   r‘   )r1   r6   ro   r’   rr   Úinput_shapeÚhidden_shapeÚquery_statesr}   r~   Úattention_interfacer€   r   s                r4   rC   zJambaAttention.forward¦   sq  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r5   ©NN)rH   rI   rJ   Ú__doc__r#   Úintr+   r-   rL   r   r   r   rE   rC   rM   rN   s   @r4   rƒ   rƒ   ”   s‡   ø„ áGðl˜{ð l°sõ lð" /3Ø(,ñ	")à—|‘|ð")ð Ÿ™ tÑ+ð")ð  ™ð	")ð
 Ð+Ñ,ð")ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷")r5   rƒ   c                   óô   ‡ — e Zd ZdZdefˆ fd„Z	 	 ddej                  dedz  dej                  dz  fd„Z
ddedz  dej                  dz  fd	„Z ed
«      	 	 ddedz  dej                  dz  fd„«       Zˆ xZS )ÚJambaMambaMixeruƒ  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    r„   c           	      óÔ  •— t         ‰| �  «        || _        || _        |j                  | _        |j
                  | _        |j                  | _        |j                  |j                  z  | _
        |j                  | _        |j                  | _        |j                  | _        t#        j$                  | j                  | j                  | j                  | j                  | j                  | j                  dz
  ¬«      | _        |j(                  | _        t,        |j(                     | _        t#        j0                  | j                  | j                  dz  | j                   ¬«      | _        t#        j0                  | j                  | j                  | j                  dz  z   d¬«      | _        t#        j0                  | j                  | j                  d¬«      | _        t9        j:                  d| j                  dz   «      d d d …f   }|j=                  | j                  d«      j?                  «       }t#        j@                  t9        jB                  |«      «      | _"        t#        j@                  t9        jF                  | j                  «      «      | _$        t#        j0                  | j                  | j                  | j                   ¬«      | _%        tM        | j                  |jN                  ¬«      | _(        tM        | j                  |jN                  ¬«      | _)        tM        | j                  |jN                  ¬«      | _*        tW        d	«      }tY        |d
d «      a-tY        |dd «      a.tW        d«      }t_        |d¬«      a0tY        |dd «      a1tY        |dd «      a2tg        t`        tb        t\        tZ        td        f«      a4th        stj        jm                  d«       |jn                  |   | _8        y )Nr"   )Úin_channelsÚout_channelsrˆ   Úkernel_sizeÚgroupsÚpaddingr8   r‡   FTr9   ©r'   zcausal-conv1dÚcausal_conv1d_updateÚcausal_conv1d_fnz	mamba-ssmz8ops.triton.selective_state_update.selective_state_update)Úchained_pathÚselective_scan_fnÚmamba_inner_fna  The fast path is not available because on of `(selective_state_update, selective_scan_fn, causal_conv1d_fn, causal_conv1d_update, mamba_inner_fn)` is None. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d.)9r*   r+   r„   r…   r2   Úmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizeÚmamba_expandÚintermediate_sizeÚmamba_dt_rankÚtime_step_rankÚmamba_conv_biasÚuse_conv_biasÚmamba_proj_biasÚuse_biasr   ÚConv1dÚconv1dÚ
hidden_actÚ
activationr   Úactr�   Úin_projÚx_projÚdt_projr-   Úarangerd   r|   r,   ÚlogÚA_logr.   ÚDÚout_projr&   Úrms_norm_epsÚdt_layernormÚb_layernormÚc_layernormr   r‰   r©   rª   r   Úselective_state_updater¬   r­   ÚallÚis_fast_path_availableÚloggerÚwarning_onceÚlayer_typesÚ
layer_type)r1   r„   r…   ÚAÚcausal_conv1dÚ	mamba_ssmr3   s         €r4   r+   zJambaMambaMixer.__init__Ó   s  ø€ Ü‰ÑÔØˆŒØ"ˆŒØ!×-Ñ-ˆÔØ$×2Ñ2ˆÔØ &× 3Ñ 3ˆÔØ!'×!4Ñ!4°v×7IÑ7IÑ!IˆÔØ$×2Ñ2ˆÔØ#×3Ñ3ˆÔØ×.Ñ.ˆŒÜ—i‘iØ×.Ñ.Ø×/Ñ/Ø×#Ñ#Ø×-Ñ-Ø×)Ñ)Ø×)Ñ)¨AÑ-ô
ˆŒð !×+Ñ+ˆŒÜ˜&×+Ñ+Ñ,ˆŒô —y‘y ×!1Ñ!1°4×3IÑ3IÈAÑ3MÐTX×TaÑTaÔbˆŒä—i‘i × 6Ñ 6¸×8KÑ8KÈd×NaÑNaÐdeÑNeÑ8eÐlqÔrˆŒä—y‘y ×!4Ñ!4°d×6LÑ6LÐSWÔXˆŒô �L‰L˜˜D×/Ñ/°!Ñ3Ó4°Tº1°WÑ=ˆØ�H‰H�T×+Ñ+¨RÓ0×;Ñ;Ó=ˆä—\‘\¤%§)¡)¨A£,Ó/ˆŒ
Ü—‘œeŸj™j¨×)?Ñ)?Ó@ÓAˆŒÜŸ	™	 $×"8Ñ"8¸$×:JÑ:JÐQU×Q^ÑQ^Ô_ˆŒä(¨×)<Ñ)<À&×BUÑBUÔVˆÔÜ'¨×(;Ñ(;À×ATÑATÔUˆÔÜ'¨×(;Ñ(;À×ATÑATÔUˆÔô )¨Ó9ˆÜ& }Ð6LÈdÓSÐÜ" =Ð2DÀdÓKÐô % [Ó1ˆ	Ü!8ØÐ$^ô"
Ðô $ IÐ/BÀDÓIÐÜ  Ð,<¸dÓCˆô "%Ü#Ô%6Ô8HÔJ^Ô`nÐoó"
Ðõ &Ü×ÑðRôð
 !×,Ñ,¨YÑ7ˆ�r5   Nr6   Úcache_paramsro   c                 ó¢	  — |j                   \  }}}|d uxr" |j                  | j                  «      xr |dk(  }| j                  |«      j	                  dd«      }|j                  dd¬«      \  }}	|�||j                  d«      z  }| j                  j                  j                  | j                  j                  j                  d«      | j                  j                  j                  d«      «      }
|rrt        |j                  d«      |j                  | j                     j                  d   |
| j                  j                  | j                   «      }|j                  d«      }n‡|�Xt"        j$                  j'                  || j(                  |j                   d   z
  df«      }|j+                  || j                  «       t-        ||
| j                  j                  | j                   ¬«      }|�||j                  d«      z  }| j/                  |j	                  dd«      «      }t1        j2                  || j4                  | j6                  | j6                  gd¬«      \  }}}| j9                  |«      }| j;                  |«      }| j=                  |«      }| j>                  j                  j@                  }t1        jB                  «       5  t1        jD                  | j>                  j                  j@                  «      | j>                  j                  _         d d d «       | j?                  |«      j	                  dd«      }t1        jB                  «       5  || j>                  j                  _         d d d «       t1        jF                  | jH                  jK                  «       «       }|�|jK                  «       nd }|rjtM        |j                  | j                     jN                  d   |d   |d   ||d d …df   |d d …df   | jP                  |	d   |d¬	«
      j                  d«      }nptS        ||||j	                  dd«      |j	                  dd«      | jP                  jK                  «       |	|dd¬
«
      \  }}|�|�|jU                  || j                  «       | jW                  |j	                  dd«      «      }|S # 1 sw Y   �Œ¤xY w# 1 sw Y   �ŒXxY w)Nr"   r8   rP   r   r9   )r½   ).r   T)Údt_softplus)Údelta_softplusÚreturn_last_state),rF   Úhas_previous_stater…   r¿   ry   ÚchunkrY   r»   r/   r•   Úsizer©   ÚsqueezeÚlayersÚconv_statesrˆ   r½   r   rz   Úpadr±   Úupdate_conv_staterª   rÀ   r-   Úsplitrµ   r¯   rÈ   rÉ   rÊ   rÁ   ÚdataÚno_gradÚ
zeros_likeÚexprÄ   rK   rË   Úrecurrent_statesrÅ   r¬   Úupdate_recurrent_staterÆ   )r1   r6   rÕ   ro   Ú
batch_sizeÚseq_lenÚ_Úuse_precomputed_statesÚprojected_statesÚgateÚconv_weightsrß   Ússm_parametersÚ	time_stepÚBÚCÚtime_proj_biasÚdiscrete_time_steprÒ   Úscan_outputsÚ	ssm_stateÚcontextualized_statess                         r4   Úcuda_kernels_forwardz$JambaMambaMixer.cuda_kernels_forward  sB  € ð "/×!4Ñ!4Ñˆ
�G˜Qà Ð$Òi¨×)HÑ)HÈÏÉÓ)XÒiÐ]dÐhiÑ]ið 	ð  Ÿ<™<¨Ó6×@Ñ@ÀÀAÓFÐð /×4Ñ4°Q¸AÐ4Ó>Ñˆ�tàÐ%Ø)¨N×,DÑ,DÀQÓ,GÑGˆMð —{‘{×)Ñ)×.Ñ.¨t¯{©{×/AÑ/A×/FÑ/FÀqÓ/IÈ4Ï;É;×K]ÑK]×KbÑKbÐcdÓKeÓfˆÙ!Ü0Ø×%Ñ% bÓ)Ø×#Ñ# D§N¡NÑ3×?Ñ?ÀÑBØØ—‘× Ñ Ø—‘óˆMð *×3Ñ3°BÓ7‰MàÐ'Ü Ÿm™m×/Ñ/°À×@UÑ@UÐXe×XkÑXkÐlnÑXoÑ@oÐqrÐ?sÓt�Ø×.Ñ.¨{¸D¿N¹NÔKÜ,¨]¸LÈ$Ï+É+×JZÑJZÐgk×gvÑgvÔwˆMàÐ%Ø)¨N×,DÑ,DÀQÓ,GÑGˆMð Ÿ™ ]×%<Ñ%<¸QÀÓ%BÓCˆÜŸ+™+Ø˜T×0Ñ0°$×2EÑ2EÀt×GZÑGZÐ[Ðacô
‰ˆ	�1�að ×%Ñ% iÓ0ˆ	Ø×Ñ˜QÓˆØ×Ñ˜QÓˆð Ÿ™×*Ñ*×/Ñ/ˆÜ�]‰]‹_ñ 	NÜ%*×%5Ñ%5°d·l±l×6GÑ6G×6LÑ6LÓ%MˆD�L‰L×ÑÔ"÷	Nà!Ÿ\™\¨)Ó4×>Ñ>¸qÀ!ÓDÐÜ�]‰]‹_ñ 	4Ø%3ˆD�L‰L×ÑÔ"÷	4ô �Y‰Y�t—z‘z×'Ñ'Ó)Ó*Ð*ˆà3AÐ3M˜×-Ñ-Ô/ÐSWˆÙ!Ü1Ø×#Ñ# D§N¡NÑ3×DÑDÀQÑGØ˜fÑ%Ø" 6Ñ*ØØ’!�Q�$‘Ø’!�Q�$‘Ø—‘Ø�V‘ØØ ô÷ ‰i˜‹mñ ô '8ØØ"ØØ—‘˜A˜qÓ!Ø—‘˜A˜qÓ!Ø—‘—‘“ØØØ#Ø"&ô'Ñ#ˆL˜)ð Ð$¨Ð)AØ×3Ñ3°I¸t¿~¹~ÔNð !%§¡¨l×.DÑ.DÀQÈÓ.JÓ KÐà$Ð$÷S	Nñ 	Nú÷	4ñ 	4ús   ËAR7ÍSÒ7SÓSc           	      ób
  — |j                   \  }}}|j                  }| j                  |«      j                  dd«      }|j	                  dd¬«      \  }	}
|�|	|j                  d«      z  }	|�P|j                  | j                  «      r5|j                  | j                     j                  d   j                  «       }n9t        j                  || j                  | j                  f|	j                  |¬«      }|��m|j                  | j                  «      rÀ|dk(  r»|j!                  |	| j                  «      d| j"                   d …f   }t        j$                  || j&                  j(                  d d …dd d …f   z  d¬«      }	| j*                  r|	| j&                  j,                  z  }	| j/                  |	«      j1                  |«      j                  d«      }	n¹t2        j4                  j7                  |	| j"                  |	j                   d   z
  df«      }|j!                  || j                  «      d| j"                   d …f   }| j/                  | j'                  |	«      dd |…f   «      }	n'| j/                  | j'                  |	«      dd |…f   «      }	|�|	|j                  d«      z  }	| j9                  |	j                  dd«      «      }t        j:                  || j<                  | j                  | j                  gd¬«      \  }}}| j?                  |«      }| jA                  |«      }| jC                  |«      }| jE                  |«      }t2        j4                  jG                  |«      j                  dd«      }t        jH                  | jJ                  jM                  «       «       }t        jH                  |d d d …d d d …f   |d d …d d …d d …d f   z  «      }|d d …d d …d d …d f   |d d …d d d …d d …f   jM                  «       z  }||	d d …d d …d d …d f   jM                  «       z  }g }tO        |«      D ]}  }|d d …d d …|d d …f   |z  |d d …d d …|d d …f   z   }t        jP                  |j1                  |«      |d d …|d d …f   j                  d«      «      }|jS                  |d d …d d …df   «       Œ t        jT                  |d¬«      }||	| jV                  d d d …d f   z  z   }|| j/                  |
«      z  }|�|jY                  || j                  «       | j[                  |j                  dd«      «      }|S )Nr"   r8   rP   r   )Údevicer;   .r9   ).rF   r;   r¿   ry   rÛ   rY   rÚ   r…   rÞ   rç   Úcloner-   Úzerosr³   r¯   rû   rá   r±   Úsumr»   r/   r·   rˆ   r¾   r<   r   rz   rà   rÀ   râ   rµ   rÈ   rÉ   rÊ   rÁ   Úsoftplusræ   rÄ   rK   Úrangerx   ÚappendÚstackrÅ   rè   rÆ   )r1   Úinput_statesrÕ   ro   ré   rê   rë   r;   rí   r6   rî   r÷   Ú
conv_staterð   rñ   rò   ró   rõ   rÒ   Ú
discrete_AÚ
discrete_BÚdeltaB_urö   ÚiÚscan_outputrø   s                             r4   Úslow_forwardzJambaMambaMixer.slow_forward~  s¿  € Ø!-×!3Ñ!3Ñˆ
�G˜QØ×"Ñ"ˆàŸ<™<¨Ó5×?Ñ?ÀÀ1ÓEÐØ.×4Ñ4°Q¸AÐ4Ó>Ñˆ�tàÐ%Ø)¨N×,DÑ,DÀQÓ,GÑGˆMàÐ#¨×(GÑ(GÈÏÉÔ(Wà$×+Ñ+¨D¯N©NÑ;×LÑLÈQÑO×UÑUÓW‰IäŸ™Ø˜T×3Ñ3°T×5HÑ5HÐIØ$×+Ñ+°5ôˆIð Ñ#Ø×.Ñ.¨t¯~©~Ô>À7ÈaÂ<Ø)×;Ñ;¸MÈ4Ï>É>ÓZÐ[^Ðae×avÑavÐ`vÑ`wÐ[wÑx�
Ü %§	¡	¨*°t·{±{×7IÑ7IÊ!ÈQÒPQÈ'Ñ7RÑ*RÐXZÔ [�Ø×%Ò%Ø! T§[¡[×%5Ñ%5Ñ5�MØ $§¡¨Ó 7× :Ñ :¸5Ó A× KÑ KÈBÓ O‘äŸ]™]×.Ñ.Ø!Ø×*Ñ*¨]×-@Ñ-@ÀÑ-DÑDÀaÐHó�
ð *×;Ñ;¸JÈÏÉÓWÐX[Ð^b×^sÑ^sÐ]sÑ]tÐXtÑu�
Ø $§¡¨¯©°]Ó)CÀCÈÈ'ÈÀMÑ)RÓ S‘à ŸH™H T§[¡[°Ó%?ÀÀXÀgÀXÀÑ%NÓOˆMàÐ%Ø)¨N×,DÑ,DÀQÓ,GÑGˆMð Ÿ™ ]×%<Ñ%<¸QÀÓ%BÓCˆÜŸ+™+Ø˜T×0Ñ0°$×2EÑ2EÀt×GZÑGZÐ[Ðacô
‰ˆ	�1�að ×%Ñ% iÓ0ˆ	Ø×Ñ˜QÓˆØ×Ñ˜QÓˆà!Ÿ\™\¨)Ó4ÐÜŸ]™]×3Ñ3Ð4FÓG×QÑQÐRSÐUVÓWÐô �Y‰Y�t—z‘z×'Ñ'Ó)Ó*Ð*ˆÜ—Y‘Y˜q ¢q¨$²Ð!1Ñ2Ð5GÊÊ1ÊaÐQUÈÑ5VÑVÓWˆ
Ø'ªª1ªa°¨Ñ6¸º1¸dÂAÂq¸=Ñ9I×9OÑ9OÓ9QÑQˆ
Ø ªa²²A°t¨mÑ <× BÑ BÓ DÑDˆàˆÜ�w“ò 	6ˆAØ"¢1¢a¨ªA :Ñ.°Ñ:¸XÂaÊÈAÊqÀjÑ=QÑQˆIÜŸ,™, y§|¡|°EÓ':¸AºaÀÂA¸g¹J×<PÑ<PÐQSÓ<TÓUˆKØ×Ñ ªAªq°!¨GÑ 4Õ5ð	6ô —k‘k ,°BÔ7ˆØ! ]°T·V±V¸DÂ!ÀT¸MÑ5JÑ%JÑKˆØ" T§X¡X¨d£^Ñ3ˆàÐ#Ø×/Ñ/°	¸4¿>¹>ÔJð !%§¡¨k×.CÑ.CÀAÀqÓ.IÓ JÐØ$Ð$r5   r»   c                 óV  — | j                   j                  rXt        r,d| j                  j                  j
                  j                  vr&t        j                  d«       d| j                   _        | j                   j                  r| j                  |||«      S | j                  |||«      S )NÚcudazÔFast Mamba kernels are not available. Make sure that they are installed and that the mamba module is on a CUDA device. Turning off the fast path `config.use_mamba_kernels=False` and falling back to the slow path.F)r„   Úuse_mamba_kernelsrÍ   rÀ   r/   rû   ÚtyperÎ   rÏ   rù   r
  )r1   r6   rÕ   ro   s       r4   rC   zJambaMambaMixer.forwardË  sŠ   € ð �;‰;×(Ò(Ý&¨&¸¿¹×8JÑ8J×8QÑ8Q×8VÑ8VÑ*Vä×ÑðVôð
 -2ˆD�K‰KÔ)à�;‰;×(Ò(Ø×,Ñ,¨]¸LÈ.ÓYÐYØ× Ñ  °¸nÓMÐMr5   r�   )rH   rI   rJ   rž   r#   r+   r-   rL   r   Ú
LongTensorrù   r
  r   rC   rM   rN   s   @r4   r¡   r¡   Ë   sÈ   ø„ ñðC8˜{õ C8ðP &*Ø26ñ	c%à—|‘|ðc%ð ˜d‘lðc%ð ×(Ñ(¨4Ñ/ó	c%ñLJ%°u¸t±|ð J%Ð\a×\lÑ\lÐosÑ\só J%ñZ ˜HÓ%ð &*Ø26ñ	Nð ˜d‘lðNð ×(Ñ(¨4Ñ/ò	Nó &ôNr5   r¡   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚJambaMLPc                 óÊ  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _	        t        |j                     | _        y ©NFr‡   )r*   r+   r„   r2   r³   r   r�   Ú	gate_projÚup_projÚ	down_projr   r¼   Úact_fn©r1   r„   r3   s     €r4   r+   zJambaMLP.__init__â  s¦   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Ñ/ˆ�r5   c                 óˆ   — | j                  | j                  | j                  |«      «      | j                  |«      z  «      }|S )N)r  r  r  r  )r1   rS   r  s      r4   rC   zJambaMLP.forwardì  s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÑ#SÓTˆ	ØÐr5   )rH   rI   rJ   r+   rC   rM   rN   s   @r4   r  r  á  s   ø„ ô0ör5   r  c                   ó�   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  dej                  dej                  fd„Zˆ xZ	S )	ÚJambaExpertsz2Collection of expert weights stored as 3D tensors.r„   c                 óò  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        t        j                  t        j                  | j                  d| j                  z  | j
                  «      «      | _        t        j                  t        j                  | j                  | j
                  | j                  «      «      | _        t        |j                     | _        y )Nr8   )r*   r+   Únum_local_expertsÚnum_expertsr2   Ú
hidden_dimr³   Úintermediate_dimr   r,   r-   ÚemptyÚgate_up_projr  r   r¼   r  r  s     €r4   r+   zJambaExperts.__init__õ  s­   ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ ×,Ñ,ˆŒØ &× 8Ñ 8ˆÔÜŸL™L¬¯©°T×5EÑ5EÀqÈ4×K`ÑK`ÑG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸ™¤e§k¡k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Ñ/ˆ�r5   r6   Útop_k_indexÚtop_k_weightsr(   c                 óf  — t        j                  |«      }t        j                  «       5  t         j                  j                  j                  || j                  ¬«      }|j                  ddd«      }t        j                  |j                  d¬«      d«      j                  «       }d d d «       D ]ð  }|d   }|| j                  k(  rŒt        j                  |   «      \  }}	||	   }
t        j                  j                  |
| j                  |   «      j                  dd¬«      \  }}| j                  |«      |z  }t        j                  j                  || j                   |   «      }|||	|d f   z  }|j#                  d|	|j%                  |j&                  «      «       Œò |S # 1 sw Y   �ŒxY w)N)Únum_classesr8   r"   r   )r9   éþÿÿÿrP   r9   )r-   rå   rä   r   rz   Úone_hotr  ÚpermuteÚgreaterrþ   ÚnonzeroÚwhereÚlinearr"  rÛ   r  r  Ú
index_add_r<   r;   )r1   r6   r#  r$  Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_staterî   ÚupÚcurrent_hidden_statess                 r4   rC   zJambaExperts.forwardþ  s™  € ô $×.Ñ.¨}Ó=ÐÜ�]‰]‹_ñ 	SÜŸ(™(×-Ñ-×5Ñ5°kÈt×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸ™ {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷	Sð
 %ò 
	nˆJØ# A™ˆJØ˜T×-Ñ-Ò-ØÜ#(§;¡;¨{¸:Ñ/FÓ#GÑ ˆI�yØ)¨)Ñ4ˆMÜ—}‘}×+Ñ+¨M¸4×;LÑ;LÈZÑ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆD�"Ø$(§K¡K°Ó$5¸Ñ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9NÐPT×P^ÑP^Ð_iÑPjÓ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÑ<eÑ$eÐ!Ø×*Ñ*¨1¨iÐ9N×9QÑ9QÐRe×RkÑRkÓ9lÕmð
	nð #Ð"÷#	Sñ 	Sús   ªA=F&Æ&F0)
rH   rI   rJ   rž   r#   r+   r-   rL   rC   rM   rN   s   @r4   r  r  ñ  sM   ø„ á<ð0˜{õ 0ð#à—|‘|ð#ð —\‘\ð#ð —|‘|ð	#ð
 
�‰÷#r5   r  c                   óf   ‡ — e Zd ZdZdefˆ fd„Zd„ Zdej                  dej                  fd„Z	ˆ xZ
S )ÚJambaSparseMoeBlockaÈ  
    This implementation is
    strictly equivalent to standard MoE with full capacity (no
    dropped tokens). It's faster since it formulates MoE operations
    in terms of block-sparse operations to accommodate imbalanced
    assignments of tokens to experts, whereas standard MoE either
    (1) drop tokens at the cost of reduced performance or (2) set
    capacity factor to number of experts and thus waste computation
    and memory on padding.
    r„   c                 ó,  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        |j                  | _        t        j                  | j                  | j                  d¬«      | _        t        |«      | _        y r  )r*   r+   r2   r  r³   Úffn_dimr  Únum_experts_per_tokÚtop_kr   r�   Úrouterr  Úexpertsr  s     €r4   r+   zJambaSparseMoeBlock.__init__%  sm   ø€ Ü‰ÑÔØ ×,Ñ,ˆŒØ×/Ñ/ˆŒØ!×-Ñ-ˆÔØ×/Ñ/ˆŒ
ä—i‘i §¡°×1AÑ1AÈÔNˆŒÜ# FÓ+ˆ�r5   c                 óú   — t         j                  j                  j                  |dt         j                  ¬«      }t        j
                  || j                  d¬«      \  }}||j                  |j                  «      fS )Nr9   rt   rP   )	r-   r   rz   r{   rK   Útopkr=  r<   r;   )r1   r6   Úrouter_logitsÚrouting_weightsr$  r#  s         r4   Úroute_tokens_to_expertsz+JambaSparseMoeBlock.route_tokens_to_experts/  sb   € ÜŸ(™(×-Ñ-×5Ñ5°mÈÔSX×S^ÑS^Ð5Ó_ˆÜ%*§Z¡Z°ÀÇÁÐQSÔ%TÑ"ˆ�{Ø˜M×,Ñ,¨]×-@Ñ-@ÓAÐAÐAr5   r6   r(   c                 óâ   — |j                   \  }}}|j                  d|«      }| j                  |«      }| j                  ||«      \  }}| j	                  |||«      }|j                  |||«      }|S )Nr9   )rF   r•   r>  rD  r?  re   )r1   r6   ré   Úsequence_lengthr  rB  r#  r$  s           r4   rC   zJambaSparseMoeBlock.forward4  sx   € Ø2?×2EÑ2EÑ/ˆ
�O ZØ%×*Ñ*¨2¨zÓ:ˆØŸ™ MÓ2ˆØ%)×%AÑ%AÀ-ÐQ^Ó%_Ñ"ˆ�]ØŸ™ ]°KÀÓOˆØ%×-Ñ-¨j¸/È:ÓVˆØÐr5   )rH   rI   rJ   rž   r#   r+   rD  r-   rL   rC   rM   rN   s   @r4   r9  r9    s5   ø„ ñ	ð,˜{õ ,òBð
 U§\¡\ð °e·l±l÷ r5   r9  c                   óÄ   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
ee   dej                  fd„Zˆ xZS )ÚJambaAttentionDecoderLayerr„   r…   c                 óR  •— t         ‰| �  «        |j                  r|j                  |   nd}t        ||«      | _        |dkD  rt
        nt        } ||«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        y )Nr"   r¨   )r*   r+   Úlayers_num_expertsrƒ   Ú	self_attnr9  r  Úfeed_forwardr&   r2   rÇ   Úinput_layernormÚpre_ff_layernorm©r1   r„   r…   r  Úffn_layer_classr3   s        €r4   r+   z#JambaAttentionDecoderLayer.__init__?  sˆ   ø€ Ü‰ÑÔØ>D×>WÒ>W�f×/Ñ/°	Ò:Ð]^ˆÜ'¨°	Ó:ˆŒà1<¸q²Õ-ÄhˆÙ+¨FÓ3ˆÔÜ+¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ ,¨V×-?Ñ-?ÀV×EXÑEXÔ YˆÕr5   Nr6   ro   Úposition_idsr’   Ú	use_cacherr   r(   c           	      ó¾   — |}| j                  |«      } | j                  d|||||dœ|¤Ž\  }}||z   }|}| j                  |«      }| j                  |«      }||z   }|S )N)r6   ro   rQ  r’   rR  © )rM  rK  rN  rL  )	r1   r6   ro   rQ  r’   rR  rr   Úresidualrë   s	            r4   rC   z"JambaAttentionDecoderLayer.forwardI  s�   € ð !ˆØ×,Ñ,¨]Ó;ˆØ)˜4Ÿ>™>ð 
Ø'Ø)Ø%Ø+Øñ
ð ñ
Ñˆ�qð ! =Ñ0ˆØ ˆØ×-Ñ-¨mÓ<ˆØ×)Ñ)¨-Ó8ˆØ  =Ñ0ˆØÐr5   )NNNF)rH   rI   rJ   r#   rŸ   r+   r-   rL   r  r   Úboolr   r   ÚFloatTensorrC   rM   rN   s   @r4   rH  rH  >  sœ   ø„ ðZ˜{ð Z°sõ Zð /3Ø04Ø(,Ø!&ñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð Ð+Ñ,ðð 
×	Ñ	÷r5   rH  c                   ó¸   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
e   d
ej                  fd„Zˆ xZS )ÚJambaMambaDecoderLayerr„   r…   c                 óT  •— t         ‰| �  «        |j                  r|j                  |   nd}t        ||¬«      | _        |dkD  rt
        nt        } ||«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        y )Nr"   )r„   r…   r¨   )r*   r+   rJ  r¡   Úmambar9  r  rL  r&   r2   rÇ   rM  rN  rO  s        €r4   r+   zJambaMambaDecoderLayer.__init__e  sˆ   ø€ Ü‰ÑÔØ>D×>WÒ>W�f×/Ñ/°	Ò:Ð]^ˆÜ$¨F¸iÔHˆŒ
Ø1<¸q²Õ-ÄhˆÙ+¨FÓ3ˆÔÜ+¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔÜ ,¨V×-?Ñ-?ÀV×EXÑEXÔ YˆÕr5   Nr6   ro   rQ  r’   rr   r(   c                 ó°   — |}| j                  |«      }| j                  |||¬«      }||z   }|}| j                  |«      }| j                  |«      }||z   }|S )N)r6   rÕ   ro   )rM  r[  rN  rL  )r1   r6   ro   rQ  r’   rr   rU  s          r4   rC   zJambaMambaDecoderLayer.forwardn  sv   € ð !ˆØ×,Ñ,¨]Ó;ˆØŸ
™
Ø'Ø(Ø)ð #ó 
ˆð
 ! =Ñ0ˆØ ˆØ×-Ñ-¨mÓ<ˆØ×)Ñ)¨-Ó8ˆØ  =Ñ0ˆØÐr5   )NNN)rH   rI   rJ   r#   rŸ   r+   r-   rL   r  r   r   r   rW  rC   rM   rN   s   @r4   rY  rY  d  s‹   ø„ ðZ˜{ð Z°sõ Zð /3Ø04Ø(,ñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð Ð+Ñ,ðð 
×	Ñ	÷r5   rY  c                   ó¬   ‡ — e Zd ZU eed<   dZdZddgZdgZdZ	dZ
dZdZeege eej$                  d¬«      d	œZ ej*                  «       ˆ fd
„«       Zˆ xZS )ÚJambaPreTrainedModelr„   ÚmodelTrH  rY  r’   r>  )Ú
layer_name)r6   Ú
attentionsrB  c                 óŒ  •— t         ‰| �  |«       t        |t        «      r§t	        j
                  d|j                  dz   «      d d d …f   }|j                  |j                  d«      j                  «       }t        j                  |j                  t	        j                  |«      «       t        j                  |j                  «       y t        |t         «      rmt        j"                  |j$                  d| j&                  j(                  ¬«       t        j"                  |j*                  d| j&                  j(                  ¬«       y y )Nr"   r9   r”   )r?   Ústd)r*   Ú_init_weightsÚ
isinstancer¡   r-   rÂ   r¯   rd   r³   r|   ÚinitÚcopy_rÄ   rÃ   Úones_rÅ   r  Únormal_r"  r„   Úinitializer_ranger  )r1   rk   rÒ   r3   s      €r4   rd  z"JambaPreTrainedModel._init_weights•  sÖ   ø€ ä‰Ñ˜fÔ%Ü�fœoÔ.Ü—‘˜Q × 5Ñ 5¸Ñ 9Ó:¸4Â¸7ÑCˆAØ—‘˜×1Ñ1°2Ó6×AÑAÓCˆAÜ�J‰J�v—|‘|¤U§Y¡Y¨q£\Ô2Ü�J‰J�v—x‘xÕ Ü˜¤Ô-Ü�L‰L˜×,Ñ,°3¸D¿K¹K×<YÑ<YÕZÜ�L‰L˜×)Ñ)°¸¿¹×9VÑ9VÖWð .r5   )rH   rI   rJ   r#   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_is_statefulÚ_can_compile_fullgraphrH  rY  rƒ   r    r   r�   Ú_can_record_outputsr-   rä   rd  rM   rN   s   @r4   r^  r^  …  s€   ø… ØÓØÐØ&*Ð#Ø5Ð7OÐPÐØ#4Ð"5ÐØÐØ€NØ€LØ!Ðà4Ð6LÐMØ$Ù'¨¯	©	¸hÔGñÐð €U‡]�]ƒ_ó	Xó ô	Xr5   r^  )Ú	attentionr[  c                   óò   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   defd„«       «       «       Zˆ xZS )Ú
JambaModelr„   c                 ó   •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        g }t        |j                  «      D ]1  }t        |j                  |      }|j                   |||¬«      «       Œ3 t        j                  |«      | _        t!        |j                  |j"                  ¬«      | _        d| _        | j)                  «        y )N)r…   r¨   F)r*   r+   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr2   Úembed_tokensr   Únum_hidden_layersÚALL_DECODER_LAYER_TYPESÚlayers_block_typer  Ú
ModuleListrÞ   r&   rÇ   Úfinal_layernormÚgradient_checkpointingÚ	post_init)r1   r„   Údecoder_layersr  Úlayer_classr3   s        €r4   r+   zJambaModel.__init__§  sÝ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔØˆÜ�v×/Ñ/Ó0ò 	DˆAÜ1°&×2JÑ2JÈ1Ñ2MÑNˆKØ×!Ñ!¡+¨fÀÔ"BÕCð	Dô —m‘m NÓ3ˆŒä+¨F×,>Ñ,>ÀF×DWÑDWÔXˆÔà&+ˆÔ#à�‰Õr5   NÚ	input_idsro   rQ  r’   Úinputs_embedsrR  rr   r(   c           	      ót  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      s)| j                  ||||dœ}
t        d
i |
¤Žt        d
i |
¤Ždœ}	|}t        | j                  «      D ]-  \  }} ||f|	| j                  j                   |      |||dœ|¤Ž}Œ/ | j#                  |«      }t%        ||¬	«      S )Nz:You must specify exactly one of input_ids or inputs_embeds)r„   r   r"   )rû   )r„   rˆ  ro   r’   rQ  )Úfull_attentionÚlinear_attention)ro   rQ  r’   rR  )Úlast_hidden_stater’   rT  )Ú
ValueErrorr}  r	   r„   Úget_seq_lengthr-   rÂ   rF   rû   rY   re  Údictr   r   Ú	enumeraterÞ   rÐ   r‚  r   )r1   r‡  ro   rQ  r’   rˆ  rR  rr   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr6   r  Údecoder_layers                 r4   rC   zJambaModel.forward¹  ss  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-ÄÔFð Ÿ+™+Ø!.Ø"0Ø#2Ø ,ñˆKô #5Ñ"C°{Ñ"CÜ$CÑ$RÀkÑ$Rñ#Ðð &ˆÜ )¨$¯+©+Ó 6ò 	ÑˆAˆ}Ù)Øðà2°4·;±;×3JÑ3JÈ1Ñ3MÑNØ)Ø /Ø#ñð ñ‰Mð	ð ×,Ñ,¨]Ó;ˆä%Ø+Ø+ô
ð 	
r5   )NNNNNN)rH   rI   rJ   r#   r+   r   r!   r   r-   r  rL   r   rW  rV  r   r   r   rC   rM   rN   s   @r4   rw  rw  ¥  sÍ   ø„ ð˜{õ ð$  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ6
à×#Ñ# dÑ*ð6
ð Ÿ™ tÑ+ð6
ð ×&Ñ&¨Ñ-ð	6
ð
  ™ð6
ð ×(Ñ(¨4Ñ/ð6
ð ˜$‘;ð6
ð Ð+Ñ,ð6
ð 
 ò6
ó ó ó  ô6
r5   rw  Úgate_logitsr  c                 ó¾  — | �t        | t        «      syt        | t        «      rC| d   j                  }t        j                  | D �cg c]  }|j                  |«      ‘Œ c}d¬«      }t        j                  j                  j                  d¬«      }t        j                  ||d¬«      \  }}	t        j                  j                  j                  |	|«      }
|€>t        j                  |
j                  «       d¬«      }t        j                  |d¬«      }�n|j                  \  }}|j                  d   ||z  z  }|ddd…dd…ddf   j                  |||||f«      j                  d||«      j                  «      }t        j                   |
j                  «       |z  d¬«      t        j                   |d¬«      z  }|ddd…dd…df   j                  ||||f«      j                  d|«      j                  |«      }t        j                   ||z  d¬«      t        j                   |d¬«      z  }t        j                   ||j#                  d«      z  «      }||z  S c c}w )aÄ  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   rP   r9   )re  rE   rû   r-   rR   r<   r   rz   r{   rA  r(  r?   rK   rF   rd   re   rþ   rY   )r•  r  r=  ro   Úcompute_deviceÚ
layer_gateÚconcatenated_gate_logitsrC  rë   Úselected_expertsr0  Útokens_per_expertÚrouter_prob_per_expertré   rF  r~  Úexpert_attention_maskÚ router_per_expert_attention_maskÚoverall_losss                      r4   Úload_balancing_loss_funcr   õ  s9  € ð: Ð¤*¨[¼%Ô"@Øä�+œuÔ%Ø$ Q™×.Ñ.ˆÜ#(§9¡9Ð^iÖ-jÐPZ¨j¯m©m¸NÕ.KÒ-jÐpqÔ#rÐ ä—h‘h×)Ñ)×1Ñ1Ð2JÐPRÐ1ÓS€OäŸ*™* _°eÀÔDÑ€AÐä—(‘(×%Ñ%×-Ñ-Ð.>ÀÓL€KàÐä!ŸJ™J {×'8Ñ'8Ó':ÀÔBÐô "'§¡¨OÀÔ!CÒà&4×&:Ñ&:Ñ#ˆ
�OØ4×:Ñ:¸1Ñ=À*ÈÑB^Ñ_Ðð ˜4¢¢A t¨TÐ1Ñ2ß‰VÐ&¨
°OÀUÈKÐXÓYß‰W�R˜ Ó,ß‰R�Óð	 	ô "ŸI™I k×&7Ñ&7Ó&9Ð<QÑ&QÐWXÔYÔ\a×\eÑ\eØ! qô]
ñ 
Ðð ˜4¢¢A tÐ+Ñ,ß‰VÐ&¨
°OÀ[ÐQÓRß‰W�R˜Ó%ß‰R�Óð	 	)ô "'§¡¨?Ð=]Ñ+]ÐcdÔ!eÔhm×hqÑhqØ,°!ôi
ñ "
Ðô —9‘9Ð.Ð1G×1QÑ1QÐRSÓ1TÑTÓU€LØ˜+Ñ%Ð%ùò[ .ks   ÁIc                   óT  ‡ — e Zd ZddiZddiZddgdgfiZdefˆ fd„Zee		 	 	 	 	 	 	 	 	 dd
e
j                  d	z  de
j                  d	z  de
j                  d	z  ded	z  de
j                  d	z  de
j                  d	z  ded	z  ded	z  dee
j                  z  dee   defd„«       «       Zˆ xZS )ÚJambaForCausalLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr6   Úlogitsr„   c                 óN  •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  |j                  d¬«      | _        |j                  | _	        |j                  | _
        |j                  | _        | j                  «        y r  )r*   r+   rw  r_  r{  r   r�   r2   r£  Úrouter_aux_loss_coefr  r<  r„  r  s     €r4   r+   zJambaForCausalLM.__init__M  s   ø€ Ü‰Ñ˜Ô Ü Ó'ˆŒ
Ø ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ$*×$?Ñ$?ˆÔ!Ø!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ ð 	�‰Õr5   Nr‡  ro   rQ  r’   rˆ  ÚlabelsrR  Úoutput_router_logitsÚlogits_to_keeprr   r(   c
                 ój  — |�|n| j                   j                  } | j                  d|||||||dœ|
¤Ž}|j                  }t	        |	t
        «      rt        |	 d«      n|	}| j                  |dd…|dd…f   «      }d}|� | j                  ||| j                  fi |
¤Ž}d}|rYt        |j                  | j                  | j                  |«      }|�+|| j                  |j                  |j                   «      z  z  }t#        ||||j$                  |j&                  |j(                  |j                  ¬«      S )aj  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, JambaForCausalLM

        >>> model = JambaForCausalLM.from_pretrained("ai21labs/Jamba-v0.1")
        >>> tokenizer = AutoTokenizer.from_pretrained("ai21labs/Jamba-v0.1")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r‡  ro   rQ  r’   rˆ  rR  r©  )ÚlossÚaux_lossr¥  r’   r6   ra  rB  rT  )r„   r©  r_  rŒ  re  rŸ   Úslicer£  Úloss_functionr{  r   rB  r  r<  r§  r<   rû   r   r’   r6   ra  )r1   r‡  ro   rQ  r’   rˆ  r¨  rR  r©  rª  rr   Úoutputsr6   Úslice_indicesr¥  r¬  r­  s                    r4   rC   zJambaForCausalLM.forwardY  sU  € ðN %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð
 +5¨$¯*©*ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ñ	+
ð ñ	+
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ% f¨f°d·o±oÑPÈÑPˆDàˆÙÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ð!Ø˜×1Ñ1°H·K±KÀÇÁÓ4LÑLÑL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r5   )	NNNNNNNNr   )rH   rI   rJ   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr#   r+   r   r   r-   r  rL   r   rW  rV  rŸ   r   r   r   rC   rM   rN   s   @r4   r¢  r¢  G  s<  ø„ à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€Hð
˜{õ 
ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ñP
à×#Ñ# dÑ*ðP
ð Ÿ™ tÑ+ðP
ð ×&Ñ&¨Ñ-ð	P
ð
  ™ðP
ð ×(Ñ(¨4Ñ/ðP
ð × Ñ  4Ñ'ðP
ð ˜$‘;ðP
ð # T™kðP
ð ˜eŸl™lÑ*ðP
ð Ð+Ñ,ðP
ð 
#òP
ó ó ôP
r5   r¢  c                   ó   — e Zd Zy)ÚJambaForSequenceClassificationN)rH   rI   rJ   rT  r5   r4   r¶  r¶  ®  s   „ Ør5   r¶  )r¢  r¶  rw  r^  )r"   )r”   )Nr8   N)MÚcollections.abcr   r-   r   Ú r   rf  Úactivationsr   Úcache_utilsr   r	   Ú
generationr
   Úintegrationsr   r   r   r   r   Úintegrations.accelerater   Úmasking_utilsr   r   Úmodeling_layersr   r   Úmodeling_outputsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úutils.import_utilsr   Úutils.output_capturingr    r!   Úconfiguration_jambar#   Ú
get_loggerrH   rÎ   ÚModuler&   rV   ra   rL   rŸ   rj   rK   r�   rƒ   r¡   r  r  r9  rH  rY  r^  r  rw  rE   r   r¢  r¶  Ú__all__rT  r5   r4   ú<module>rË     s·  ðõ2 %ã Ý å &Ý !ß .Ý )÷õ õ >ß Pß [ß Qß FÝ &ß RÓ RÝ 7Ý 9ß EÝ ,ð 
ˆ×	Ñ	˜HÓ	%€ñ ˜YÓ'ôJ�2—9‘9ó Jó (ðJò((ñ Ð*Ó+òó ,ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(ó%ñ2 Ð)Ó*ô3)�R—Y‘Yó 3)ó +ð3)ôlSN�b—i‘iô SNôlˆr�y‰yô ð  ô$#�2—9‘9ó $#ó ð$#ôN"˜"Ÿ)™)ô "ôJ#Ð!;ô #ôLÐ7ô ôBX˜?ô Xð: )CÐMcÑdÐ ð ôL
Ð%ó L
ó ðL
ðb #Ø
Ø*.ñ	O&Ø—‘  e§l¡lÑ 3Ñ3°dÑ:ðO&à�t‘ðO&ð —L‘L 4Ñ'ð	O&ð
 ‡\�\�CÑóO&ðd ôc
Ð+¨_ó c
ó ðc
ôL	Ð%EÐG[ô 	ò g�r5   