Ë
    ÛÓ^j¦ô  ã                   óž  — d dl mZ d dlmZmZ d dlZd dlmc mZ	 d dlmZ ddl
mZ ddlmZ ddlmZmZ dd	lmZ dd
lmZmZmZmZ ddlmZ ddlmZ ddlmZmZ ddl m!Z! ddl"m#Z#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0m1Z1m2Z2m3Z3 ddl4m5Z5m6Z6 ddl7m8Z8 ddl9m:Z: ddl;m<Z<  e3jz                  e>«      Z?d„ Z@ ed«      dRd„«       ZAdej„                  deCdej„                  fd„ZD	 dSd ejŠ                  d!ej„                  d"ej„                  d#ej„                  d$ej„                  dz  d%eFd&eFd'e-e/   fd(„ZG eeA«       G d)„ d*ejŠ                  «      «       ZHd+ej„                  d,eCfd-„ZId.„ ZJd/„ ZKd0„ ZL G d1„ d2ejŠ                  «      ZM G d3„ d4ej                  jŠ                  «      ZN G d5„ d6ejŠ                  «      ZO G d7„ d8ejŠ                  «      ZP G d9„ d:ejŠ                  «      ZQe G d;„ d<ejŠ                  «      «       ZR G d=„ d>ejŠ                  «      ZS G d?„ d@edA¬B«      ZT edC«       G dD„ dEejŠ                  «      «       ZU G dF„ dGe!«      ZVe0 G dH„ dIe+«      «       ZWe0 G dJ„ dKeW«      «       ZX	 	 	 dTdLej„                  eYej„                     z  dz  dMeCdz  d$ej„                  dz  dej„                  eCz  fdN„ZZe0 G dO„ dPeWe«      «       Z[g dQ¢Z\y)Ué    )ÚCallable)ÚOptionalÚ	TypedDictN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hubÚuse_kernel_func_from_hubÚuse_kernelized_func)Úforce_accelerate_hooks)Úlazy_load_kernel)Úcreate_causal_maskÚcreate_recurrent_attention_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_torchdynamo_compilingÚlogging)Úmaybe_autocastÚmerge_with_config_defaults)Úresolve_internal_import)Úcapture_outputsé   )ÚGraniteMoeHybridConfigc                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Néÿÿÿÿé   ©Údim)ÚshapeÚtorchÚcat)ÚxÚx1Úx2s      ú‰/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/granitemoehybrid/modeling_granitemoehybrid.pyÚrotate_halfr5   8   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'ó    Úrotary_pos_embc                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer5   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds          r4   Úapply_rotary_pos_embrA   ?   sY   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐr6   Úhidden_statesÚn_repÚreturnc                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r'   N)r.   ÚexpandÚreshape)rB   rC   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         r4   Ú	repeat_kvrL   Y   so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr6   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t        || j                  «      }t        || j                  «      }	t        j                  ||j	                  dd«      «      |z  }
|�|
|z   }
t
        j                  j                  |
dt        j                  ¬«      j                  |j                  «      }
t
        j                  j                  |
|| j                  ¬«      }
t        j                  |
|	«      }|j	                  dd«      j                  «       }||
fS )Nr+   r   r*   )r-   Údtype)ÚpÚtrainingr'   )rL   Únum_key_value_groupsr/   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxÚfloat32ÚtorV   rS   rX   Ú
contiguous)rM   rN   rO   rP   rQ   rR   rS   rT   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r4   Úeager_attention_forwardre   e   sà   € ô ˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r6   c                   óö   ‡ — e Zd ZdZdedefˆ fd„Z	 	 ddej                  dej                  dz  de	dz  d	e
ej                  ej                  f   dz  d
ee   de
ej                  ej                  f   fd„Zˆ xZS )ÚGraniteMoeHybridAttentionuæ   Hybrid variant that handles ``position_embeddings is None`` â€” granitemoe-hybrid configs can
    opt out of RoPE via ``position_embedding_type=None``, in which case the model passes ``None``
    instead of a ``(cos, sin)`` tuple.ÚconfigÚ	layer_idxc                 ó^  •— t         ‰| �  «        || _        || _        t	        |d|j
                  |j                  z  «      | _        |j                  |j                  z  | _	        |j                  | _        |j                  | _        d| _        t        j                  |j
                  |j                  | j                  z  |j                   ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                   ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                   ¬«      | _        t        j                  |j                  | j                  z  |j
                  |j                   ¬«      | _        y )NrK   T©Úbias)ÚsuperÚ__init__rh   ri   ÚgetattrÚhidden_sizeÚnum_attention_headsrK   rI   rY   Úattention_multiplierrR   Úattention_dropoutÚ	is_causalr   ÚLinearÚattention_biasÚq_projÚk_projÚv_projÚo_proj©Úselfrh   ri   Ú	__class__s      €r4   rn   z"GraniteMoeHybridAttention.__init__„   sJ  ø€ Ü‰ÑÔØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø×2Ñ2ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆ�r6   NrB   rQ   Úpast_key_valuesÚposition_embeddingsrT   rD   c                 ó  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
|�|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t        «      } || ||	|
|f| j                  sdn| j                   | j"                  dœ|¤Ž\  }} |j$                  g |¢d‘­Ž j'                  «       }| j)                  |«      }||fS )Nr*   r'   r+   ç        )rS   rR   )r.   rK   rw   Úviewr[   rx   ry   rA   Úupdateri   r   Úget_interfacerh   Ú_attn_implementationre   rX   rs   rR   rG   r`   rz   )r|   rB   rQ   r~   r   rT   Úinput_shapeÚhidden_shapeÚquery_statesra   rb   r<   r=   Úattention_interfacerd   rc   s                   r4   Úforwardz!GraniteMoeHybridAttention.forward›   s—  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆàÐ*Ø*‰HˆC�Ü';¸LÈ*ÐVYÐ[^Ó'_Ñ$ˆL˜*àÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r6   ©NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r(   Úintrn   r/   ÚTensorr
   Útupler   r   rŠ   Ú__classcell__©r}   s   @r4   rg   rg   ~   s£   ø„ ñ*ð
Ð5ð 
À#õ 
ð6 )-ØHLñ%)à—|‘|ð%)ð Ÿ™ tÑ+ð%)ð  ™ð	%)ð
 # 5§<¡<°·±Ð#=Ñ>ÀÑEð%)ð Ð+Ñ,ð%)ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷%)r6   rg   Úinput_tensorÚpad_sizec                 ó°   — t        | j                  «      dk(  r
ddddd|ddfnddd|ddf}t        j                  j                  j                  | |dd¬«      S )z‚
    Padding x tensor with `pad_size` on the seq_len dim (dim=1)

    Assumes that we only have tensors of either size 4 or 3
    é   r   Úconstant)ÚmoderP   )Úlenr.   r/   r   r\   Úpad)r•   r–   Ú	pad_shapes      r4   Úpad_tensor_by_sizerž   Æ   sf   € ô 47°|×7IÑ7IÓ3JÈaÒ3O��A�q˜!˜Q ¨!¨QÑ/ÐVWÐYZÐ\]Ð_gÐijÐlmÐUn€Iä�8‰8×Ñ×"Ñ" <°ÀÐSTÐ"ÓUÐUr6   c                 ó  — t        | |«      } t        | j                  «      dk(  r.| j                  | j                  d   d|| j                  d   «      S | j                  | j                  d   d|| j                  d   | j                  d   «      S )zÀ
    Padding input_tensor with `pad_size` on the seq_len dim (dim=1) and
    simultaneously splitting it into chunk sequences.

    Assumes that we only have tensors of either size 4 or 3
    r   r   r*   r+   )rž   r›   r.   rG   )r•   r–   Ú
chunk_sizes      r4   Úreshape_into_chunksr¡   Ñ   s“   € ô & l°HÓ=€Lä
ˆ<×ÑÓ !Ò#à×#Ñ# L×$6Ñ$6°qÑ$9¸2¸zÈ<×K]ÑK]Ð^_ÑK`ÓaÐað ×#Ñ#Ø×Ñ˜qÑ! 2 z°<×3EÑ3EÀaÑ3HÈ,×J\ÑJ\Ð]^ÑJ_ó
ð 	
r6   c                 ó"  — | j                  d«      } | d   j                  g | j                  «       ¢|‘­Ž } t        j                  t        j                  ||| j
                  t        j                  ¬«      d¬«      }| j                  | d«      } t        j                  | d¬«      }t        j                  t        j                  ||| j
                  t        j                  ¬«      d¬«      }|j                  | t        j                   «      }|S )zo
    More stable segment sum calculation. Uses cumulative sums and masking instead of direct subtractions.
    r*   ©.N©ÚdevicerV   )Údiagonalr   éþÿÿÿr,   )
ÚsizerF   r/   ÚtrilÚonesr¥   ÚboolÚmasked_fillÚcumsumÚinf)r•   r    ÚmaskÚtensor_segsums       r4   Úsegment_sumr±   å   sÞ   € ð ×"Ñ" 2Ó&€Jð 2�< 	Ñ*×1Ñ1ÐS°<×3DÑ3DÓ3FÐSÈ
ÒS€Lä�:‰:”e—j‘j ¨ZÀ×@SÑ@SÔ[`×[eÑ[eÔfÐqsÔt€DØ×+Ñ+¨T¨E°1Ó5€Lä—L‘L °2Ô6€Mô �:‰:”e—j‘j ¨ZÀ×@SÑ@SÔ[`×[eÑ[eÔfÐqrÔs€DØ!×-Ñ-¨t¨e´e·i±i°ZÓ@€MØÐr6   c                 ó¦   — |�N|j                   d   dkD  r<|j                   d   dkD  r*| j                  }| |dd…dd…df   z  j                  |«      } | S )zm
    Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
    Nr'   r   )r.   rV   r_   )rB   rQ   rV   s      r4   Úapply_mask_to_padding_statesr³   ù   sa   € ð
 Ð! n×&:Ñ&:¸1Ñ&=ÀÒ&AÀn×FZÑFZÐ[\ÑF]Ð`aÒFaØ×#Ñ#ˆØ&¨ºº1¸d¸
Ñ)CÑC×GÑGÈÓNˆàÐr6   c            
       ó<  ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej                  de	dz  dej                  dz  d	ej                  dz  fd
„Z	 	 dde	dz  dej                  dz  fd„Z ed«      	 	 	 dde	dz  dej                  dz  d	ej                  dz  fd„«       Zˆ xZS )ÚGraniteMoeHybridMambaLayeruP  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)

    The are a few differences between this and Mamba2Mixer:
    - The variable use_precomputed_states is slightly different due to the hybrid cache structure
    - There's a few non-obvious bugs fixed with batching in the slow path that exist in main
    - Some extra variables that our layer doesn't need have been removed
    - We ported most of the refactors in https://github.com/huggingface/transformers/pull/35154, which is (as of Dec 18, 2024) unmerged
    rh   ri   c           	      ó  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        t        |j                  | j                  z  «      | _        || _        |j                  | _        |j                  | _        t"        |j                     | _        |j&                  | _        |j*                  | _        |j.                  | _        |j2                  | _        |j6                  | _        |j:                  | _        |j<                  | _        |j>                  | _        | j                  d| j0                  z  | j                  z  z   | _         tC        jD                  | j@                  | j@                  |j                  | j                  | j@                  | j                  dz
  ¬«      | _#        | j                  | j@                  z   | j                  z   }tC        jH                  | j                  || j(                  ¬«      | _%        tC        jL                  tO        jP                  | j                  «      «      | _)        tO        jT                  d| j                  dz   «      }tC        jL                  tO        jV                  |«      «      | _,        t[        | j                  | j,                  ¬«      | _.        tC        jL                  tO        jP                  | j                  «      «      | _/        tC        jH                  | j                  | j                  | j(                  ¬«      | _0        tc        d«      }te        |dd «      a3te        |dd «      a4tc        d	«      }tk        |d
¬«      a6tk        |d¬«      a7tk        |d¬«      a8ts        tl        tn        tp        th        tf        f«      a:tt        stv        jy                  d«       ntv        jy                  d«       |jz                  |   | _>        y )Nr+   r'   )Úin_channelsÚout_channelsrl   Úkernel_sizeÚgroupsÚpaddingrk   ©Úepszcausal-conv1dÚcausal_conv1d_updateÚcausal_conv1d_fnz	mamba-ssmz8ops.triton.selective_state_update.selective_state_update)Úchained_pathz1ops.triton.ssd_combined.mamba_chunk_scan_combinedz8ops.triton.ssd_combined.mamba_split_conv1d_scan_combineda  The fast path is not available because one of `(selective_state_update, causal_conv1d_fn, causal_conv1d_update)` is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1dzOThe fast path for GraniteMoeHybrid will be used when running the model on a GPU)?rm   rn   Úmamba_n_headsÚ	num_headsrp   Úmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizer�   Úmamba_expandÚintermediate_sizeri   Úmamba_conv_biasÚuse_conv_biasÚ
hidden_actÚ
activationr	   ÚactÚmamba_proj_biasÚuse_biasÚrms_norm_epsÚlayer_norm_epsilonÚmamba_n_groupsÚn_groupsÚmamba_d_headrK   Úmamba_chunk_sizer    Útime_step_limitÚtime_step_minÚtime_step_maxÚconv_dimr   ÚConv1dÚconv1dru   Úin_projÚ	Parameterr/   rª   Údt_biasÚarangeÚlogÚA_logÚGraniteMoeHybridRMSNormGatedÚnormÚDÚout_projr   ro   r¾   r¿   r%   Úselective_state_updateÚmamba_chunk_scan_combinedÚ mamba_split_conv1d_scan_combinedÚallÚis_fast_path_availableÚloggerÚwarning_onceÚlayer_typesÚ
layer_type)r|   rh   ri   Úprojection_sizeÚAÚcausal_conv1dÚ	mamba_ssmr}   s          €r4   rn   z#GraniteMoeHybridMambaLayer.__init__  s,  ø€ Ü‰ÑÔØ×-Ñ-ˆŒØ!×-Ñ-ˆÔØ$×2Ñ2ˆÔØ &× 3Ñ 3ˆÔÜ!$ V×%8Ñ%8¸4×;KÑ;KÑ%KÓ!LˆÔØ"ˆŒØ#×3Ñ3ˆÔØ ×+Ñ+ˆŒÜ˜&×+Ñ+Ñ,ˆŒØ×.Ñ.ˆŒà"(×"5Ñ"5ˆÔà×-Ñ-ˆŒØ×+Ñ+ˆŒØ ×1Ñ1ˆŒà%×5Ñ5ˆÔØ#×1Ñ1ˆÔØ#×1Ñ1ˆÔà×.Ñ.°°T·]±]Ñ1BÀT×EXÑEXÑ1XÑXˆŒÜ—i‘iØŸ™ØŸ™Ø×'Ñ'Ø×-Ñ-Ø—=‘=Ø×)Ñ)¨AÑ-ô
ˆŒð ×0Ñ0°4·=±=Ñ@À4Ç>Á>ÑQˆÜ—y‘yØ×ÑØØ—‘ô
ˆŒô —|‘|¤E§J¡J¨t¯~©~Ó$>Ó?ˆŒô �L‰L˜˜DŸN™N¨QÑ.Ó/ˆÜ—\‘\¤%§)¡)¨A£,Ó/ˆŒ
Ü0°×1GÑ1GÈT×MdÑMdÔeˆŒ	Ü—‘œeŸj™j¨¯©Ó8Ó9ˆŒäŸ	™	 $×"8Ñ"8¸$×:JÑ:JÐQU×Q^ÑQ^Ô_ˆŒô )¨Ó9ˆÜ& }Ð6LÈdÓSÐÜ" =Ð2DÀdÓKÐô % [Ó1ˆ	Ü!8ØÐ$^ô"
Ðô %<ØÐ$Wô%
Ð!ô ,CØÐ$^ô,
Ð(ô
 "%ä&Ü)Ü0Ü Ü$ðó"
Ðõ &Ü×Ñð>õô ×ÑÐ qÔrà ×,Ñ,¨YÑ7ˆ�r6   NrB   Úcache_paramsrQ   Úseq_idxc                 óÈ  — t        ||«      }| j                  |«      }|j                  \  }}}| j                  | j                  z  }	|d uxr |j                  | j                  «      }
|
rL|j                  | j                     j                  d   }|j                  | j                     j                  d   }|
�rÎ|dk(  �rÈ|j                  d«      j                  | j                  | j                  | j                  gd¬«      \  }}}t        || j                   j"                  j                  d«      | j                   j$                  | j&                  «      }t)        j                  || j                  |	|	gd¬«      \  }}}t)        j*                  | j,                  j/                  «       «       }|d d …d df   d d …d d …d f   j1                  d| j2                  | j                  «      j5                  t(        j6                  ¬«      }|d d …d d …d f   j1                  dd| j2                  «      }| j8                  d d …d df   j1                  d| j2                  «      }| j:                  d d …d df   j1                  d| j2                  «      }|j=                  || j                  |j                  d   | j                  z  «      }|j=                  || j                  |j                  d   | j                  z  «      }|j=                  || j                  | j2                  «      }t?        ||||||d |d¬«
      }|j=                  || j                  | j2                  z  «      }| jA                  ||«      }| jC                  |«      d d …d df   }|S t)        j*                  | j,                  j/                  «       «       }| jD                  d	t/        d
«      fk(  ri nd| jD                  i}| jF                  rä|€âtI        || j                   j"                  j                  d«      | j                   j$                  | j8                  |f| j:                  | jJ                  || j&                  | j@                  j"                  | j@                  jL                  | jB                  j"                  | jB                  j$                  | j2                  | j                  dddœ|¤Ž}|S |j                  | j                  | j                  | j                  gd¬«      \  }}}|jO                  dd«      }|
rt)        jP                  |gd¬«      }|�XtR        jT                  jW                  || jX                  |j                  d   z
  df«      }|j[                  || j                  «       | j&                  dvr5| j]                  | j!                  |«      dd |j                  d   …f   «      }nQt_        || j                   j"                  j                  d«      | j                   j$                  | j&                  |¬«      }|
r|d d …d d …| d …f   }|jO                  dd«      }t        ||«      }t)        j                  || j                  |	|	gd¬«      \  }}}ta        |j=                  ||d| j2                  «      |||j=                  ||| j                  d«      |j=                  ||| j                  d«      f| jJ                  | j:                  d |d| j8                  d|
rnd dœ|¤Ž\  }}|�|�|jc                  || j                  «      }|j=                  ||d«      }| jA                  ||«      }| jC                  |«      }|S )Nr   r'   r*   r,   .©rV   T)ÚzrÞ   Údt_softplusr�   r®   Údt_limitF)rä   r    rô   rÌ   Úrmsnorm_weightÚrmsnorm_epsÚoutproj_weightÚoutproj_biasÚheaddimÚngroupsÚnorm_before_gateÚreturn_final_statesr+   )ÚsiluÚswish)r1   Úweightrl   rÌ   rô   )r    rä   r÷   rô   r  rÞ   rø   Úinitial_states)2r³   rÜ   r.   rÓ   rÄ   Úhas_previous_stateri   ÚlayersÚconv_statesÚrecurrent_statesÚsqueezeÚsplitrÈ   rÙ   rÂ   r¾   rÛ   r  rl   rÌ   r/   Úexprá   ÚfloatrF   rK   r_   r^   rÞ   rä   r‚   ræ   rã   rå   rÖ   rX   rè   r    Úvariance_epsilonr[   r0   r   r\   rœ   rÆ   Úupdate_conv_staterÍ   r¿   rç   Úupdate_recurrent_state)r|   rB   ró   rQ   rô   Úprojected_statesÚ
batch_sizeÚseq_lenÚ_Úgroups_time_state_sizeÚuse_precomputed_statesÚ
conv_stateÚrecurrent_stateÚgateÚhidden_states_B_CÚdtÚBÚCrð   rÞ   rä   Úhidden_states_reshapedÚoutÚdt_limit_kwargsr  Úscan_outputÚ	ssm_states                              r4   Úcuda_kernels_forwardz/GraniteMoeHybridMambaLayer.cuda_kernels_forwardr  sÐ  € ô 5°]ÀNÓSˆØŸ<™<¨Ó6Ðð "/×!4Ñ!4Ñˆ
�G˜QØ!%§¡°×1DÑ1DÑ!DÐà!-°TÐ!9Ò!m¸l×>]Ñ>]Ð^b×^lÑ^lÓ>mÐÙ!Ø%×,Ñ,¨T¯^©^Ñ<×HÑHÈÑKˆJØ*×1Ñ1°$·.±.ÑA×RÑRÐSTÑUˆOò " g°£lØ*:×*BÑ*BÀ1Ó*E×*KÑ*KØ×'Ñ'¨¯©¸¿¹ÐGÈRð +Ló +Ñ'ˆDÐ# Rô
 !5Ø!ØØ—‘×"Ñ"×*Ñ*¨1Ó-Ø—‘× Ñ Ø—‘ó!Ðô #(§+¡+Ø!Ø×'Ñ'Ð)?ÐAWÐXØô#ÑˆM˜1˜aô —‘˜4Ÿ:™:×+Ñ+Ó-Ó.Ð.ˆAØ’!�T˜3�,‘¢¢1 d 
Ñ+×2Ñ2°2°t·}±}Àd×FYÑFYÓZ×]Ñ]Ôdi×dqÑdqÐ]ÓrˆAØ’A’q˜$�J‘×&Ñ& r¨2¨t¯}©}Ó=ˆBØ—l‘l¢1 d¨C <Ñ0×7Ñ7¸¸D¿M¹MÓJˆGØ—‘’q˜$ �|Ñ$×+Ñ+¨B°·±Ó>ˆAØ—‘�z 4§=¡=°!·'±'¸!±*ÀÇÁÑ2MÓNˆAØ—‘�z 4§=¡=°!·'±'¸!±*ÀÇÁÑ2MÓNˆAØ%2×%7Ñ%7¸
ÀDÇNÁNÐTX×TaÑTaÓ%bÐ"Ü2ØØ&ØØØØØØØØ ôˆMð *×.Ñ.¨z¸4¿>¹>ÈDÏMÉMÑ;YÓZˆMØ ŸI™I m°TÓ:ˆMð —-‘- Ó.ªq°$¸¨|Ñ<ˆCð@ ˆ
ô{ —‘˜4Ÿ:™:×+Ñ+Ó-Ó.Ð.ˆAØ$(×$8Ñ$8¸SÄ%ÈÃ,Ð<OÒ$O™bÐV`Ðbf×bvÑbvÐUwˆOð �}Š} Ð!5Ü6Ø$Ø—K‘K×&Ñ&×.Ñ.¨qÓ1Ø—K‘K×$Ñ$Ø—L‘LØðð —f‘fØ#Ÿ™Ø#Ø#Ÿ™Ø#'§9¡9×#3Ñ#3Ø $§	¡	× :Ñ :Ø#'§=¡=×#7Ñ#7Ø!%§¡×!3Ñ!3Ø ŸM™MØ ŸM™MØ%*Ø(-ñ#ð$ &ñ%�ðp ˆ
ðE /?×.DÑ.DØ×+Ñ+¨T¯]©]¸D¿N¹NÐKÐQSð /Eó /Ñ+�Ð'¨ð
 %6×$?Ñ$?ÀÀ1Ó$EÐ!Ù)ô ).¯	©	°:Ð?PÐ2QÐWYÔ(ZÐ%ØÐ+Ü"$§-¡-×"3Ñ"3Ø)Ø×.Ñ.Ð1B×1HÑ1HÈÑ1LÑLÈaÐPó#�Kð !×2Ñ2°;ÀÇÁÔOà—?‘?Ð*;Ñ;Ø(,¯©°·±Ð=NÓ1OÐPSÐUrÐWh×WnÑWnÐoqÑWrÐUrÐPrÑ1sÓ(tÑ%ä(8Ø+Ø#Ÿ{™{×1Ñ1×9Ñ9¸!Ó<Ø!Ÿ[™[×-Ñ-Ø#'§?¡?Ø 'ô)Ð%ñ *Ø(9º!ºQÀÀÁ	¸/Ñ(JÐ%Ø$5×$?Ñ$?ÀÀ1Ó$EÐ!ä$@ÐARÐTbÓ$cÐ!Ü&+§k¡kØ%Ø×+Ñ+Ð-CÐE[Ð\Øô'Ñ#�˜q !ô *CØ!×&Ñ& z°7¸BÀÇÁÓNØØØ—F‘F˜: w°·±¸rÓBØ—F‘F˜: w°·±¸rÓBð*ð  $Ÿ™Ø—f‘fØØ#Ø(,Ø ŸL™LØ $Ù6L¡?ÐRVñ*ð &ñ*Ñ&�˜Yð$ Ð(¨\Ð-EØ ,× CÑ CÀIÈtÏ~É~Ó ^�Ià)×.Ñ.¨z¸7ÀBÓG�à"Ÿi™i¨°TÓ:�ð —m‘m KÓ0�Øˆ
r6   c                 óˆ  — |j                   \  }}}|j                  }t        ||«      }| j                  |«      }|j	                  | j
                  | j                  | j                  gd¬«      \  }	}
}|
j                  dd«      }
|d uxr |j                  | j                  «      }|r&|j                  | j                     j                  d   }|r§|dk(  r¢|j                  |
| j                  «      d| j                   d …f   }t        j                   || j"                  j$                  j'                  d«      z  d¬«      }
| j(                  r|
| j"                  j*                  z   }
| j-                  |
«      }
nÇ|rt        j.                  |
gd¬«      }
|�Xt0        j2                  j5                  |
| j                  |
j                   d   z
  df«      }|j                  || j                  «       | j-                  | j#                  |
«      dd |
j                   d   …f   «      }
|r
|
d| d …f   }
|
j                  dd«      }
t        |
|«      }
t        j                  |
| j
                  | j6                  | j8                  z  | j6                  | j8                  z  gd¬«      \  }}}t        j:                  | j<                  j?                  «       «       }|�r­|dk(  �r§|j                  | j                     j@                  d   jB                  }|d d …dd d …f   d d …d df   }|j                  dd«      jE                  ||j                   d   | jF                  «      }| jH                  d   jE                  | jH                  j                   d   | jF                  «      }t        j0                  j2                  jK                  ||jM                  |j                  «      z   «      }t        jN                  || jP                  d   | jP                  d   «      }|d   jE                  | j                  | jF                  | j8                  «      jM                  t        jR                  ¬	«      }t        j:                  |d   |z  «      jM                  |¬
«      }|jU                  || j6                  d«      dd d d …f   }|jE                  || j6                  | j                  | j6                  z  |j                   d   «      jW                  «       }|jU                  |d|j                   d   «      }|d   |dd d d …f   z  }|jU                  |d| jF                  «      }||d   z  jM                  |¬
«      }|j                  | j                     j@                  d   |z  |z   }|jY                  || j                  «      }|jU                  || j6                  d«      dd d d …f   }|jE                  || j6                  | j                  | j6                  z  |j                   d   «      jW                  «       }|jU                  |d|j                   d   «      }|jM                  |jB                  |j                  ¬«      }|j[                  || j                  z  | jF                  | j8                  «      }|j[                  || j                  z  | j8                  d«      }t        j\                  ||«      }|j[                  || j                  | jF                  «      }| j^                  d   jE                  | j^                  j                   d   | jF                  «      }|||z  z   jM                  |j                  «      }|jU                  |d«      d d …d df   }�nt0        j2                  jK                  || jH                  z   «      }t        jN                  || jP                  d   | jP                  d   «      }|jU                  ||d| jF                  «      j?                  «       }|jU                  ||d| j8                  «      j?                  «       }|jU                  ||d| j8                  «      j?                  «       }|ja                  | j                  | j6                  z  d| j                  ¬«      }|ja                  | j                  | j6                  z  d| j                  ¬«      }| jb                  || jb                  z  z
  | jb                  z  }| j^                  d   te        ||«      z  }||d   z  }|jM                  |j                  «      |z  }||||fD �cg c]  }tg        ||| jb                  «      ‘Œ c}\  }}}}|ji                  dddd«      }t        jj                  |d¬«      } t        j:                  tm        |«      «      }!|d d …d d …d d …d d d …d d …f   |d d …d d …d d d …d d …d d …f   z  }"|"j!                  d¬«      }#|#d   |!ji                  ddddd«      d   z  }$|$j!                  d¬«      }%|%d   |d d …d d …d f   z  j!                  d¬«      }&t        j:                  | d d …d d …d d …dd …f   | z
  «      }'||'ji                  dddd«      d   z  }(|(dd d d …f   |d   z  j!                  d¬«      })|rR|j                  | j                     j@                  d   d d …d f   jM                  |)j                  |)jB                  ¬«      nt        jn                  |)d d …d d…f   «      }*t        j.                  |*|)gd¬«      })t        j:                  tm        t0        j2                  j5                  | d d …d d …d d …df   d«      «      «      }+|+j                  dd«      }+|+d   |)d d …d d …d df   z  j!                  d¬«      },|,d d …d d…f   |,d d …df   }-})t        j:                  | «      }.|dd d d …f   |)d d …d d …d df   z  }/|.ji                  dddd«      }0|/j!                  d«      |0d   z  }1|&|1z   }|jU                  |d| j                  | jF                  «      }||z   }|dkD  r|d d …d |…d d …d d …f   }|jU                  ||d«      }|-�|�|jY                  |-| j                  «      }-| jq                  ||	«      }2| js                  |2jM                  |«      «      }3|3S c c}w )Nr*   r,   r'   r+   r   .r£   ).NNrö   ©r¥   r¤   )r-   Úoutput_sizer   r˜   r§   )rV   r¥   )r'   r   ):r.   rV   r³   rÜ   r  rÈ   rÙ   rÂ   r[   r  ri   r  r  r  rÆ   r/   ÚsumrÛ   r  r
  rÊ   rl   rÍ   r0   r   r\   rœ   rÓ   rÄ   r  rá   r  r	  r¥   rF   rK   rÞ   Úsoftplusr_   ÚclamprÖ   r^   rG   r`   r  r‚   Úbmmrä   Úrepeat_interleaver    rž   r¡   Úpermuter­   r±   Ú
zeros_likerã   rå   )4r|   Úinput_statesró   rQ   r  r  r  rV   r  r  r  r  r  r  r  rB   r  r  rð   Úcache_devicerÞ   ÚdAÚdBÚdBxÚ
ssm_statesÚssm_states_reshapedÚ
C_reshapedÚyrä   r–   Ú
D_residualÚtÚA_cumsumÚLÚG_intermediateÚGÚM_intermediateÚMÚY_diagÚdecay_statesÚB_decayÚstatesÚprevious_statesÚdecay_chunkÚ
new_statesr"  Ústate_decay_outÚC_times_statesÚstate_decay_out_permutedÚY_offr!  Úcontextualized_statess4                                                       r4   Útorch_forwardz(GraniteMoeHybridMambaLayer.torch_forward  sš  € ð ".×!3Ñ!3Ñˆ
�G˜QØ×"Ñ"ˆô 4°LÀ.ÓQˆØŸ<™<¨Ó5ÐØ&6×&<Ñ&<Ø×'Ñ'¨¯©¸¿¹ÐGÈRð '=ó '
Ñ#ˆÐ ð .×7Ñ7¸¸!Ó<Ðà!-°TÐ!9Ò!m¸l×>]Ñ>]Ð^b×^lÑ^lÓ>mÐÙ!Ø%×,Ñ,¨T¯^©^Ñ<×HÑHÈÑKˆJñ " g°¢lØ&×8Ñ8Ð9JÈDÏNÉNÓ[Ð\_Ðbf×bwÑbwÐawÑaxÐ\xÑyˆKä %§	¡	Ø˜dŸk™k×0Ñ0×8Ñ8¸Ó;Ñ;Àô!Ðð ×!Ò!Ø$5¸¿¹×8HÑ8HÑ$HÐ!Ø $§¡Ð):Ó ;Ñá%Ü$)§I¡I¨zÐ;LÐ.MÐSUÔ$VÐ!ØÐ'Ü Ÿm™m×/Ñ/Ø%¨×(=Ñ(=Ð@Q×@WÑ@WÐXZÑ@[Ñ([Ð]^Ð'_ó�ð ×.Ñ.¨{¸D¿N¹NÔKà $§¡¨¯©Ð5FÓ)GÈÐMjÐO`×OfÑOfÐgiÑOjÐMjÐHjÑ)kÓ lÐÙ%Ø$5°c¸G¸8¹9°nÑ$EÐ!Ø 1× ;Ñ ;¸A¸qÓ AÐä8Ð9JÈNÓ[ÐÜ#Ÿk™kØØ×#Ñ# T§]¡]°T×5HÑ5HÑ%HÈ$Ï-É-ÐZ^×ZmÑZmÑJmÐnØô
Ñˆ�q˜!ô �Y‰Y�t—z‘z×'Ñ'Ó)Ó*Ð*ˆÚ! g°£là'×.Ñ.¨t¯~©~Ñ>×OÑOÐPQÑR×YÑYˆLð ’A�qš!�G‘šQ  c˜\Ñ*ˆBØ—‘˜a Ó#×*Ñ*¨:°r·x±xÀ±|ÀTÇ]Á]ÓSˆBà—l‘l 9Ñ-×4Ñ4°T·\±\×5GÑ5GÈÑ5JÈDÏMÉMÓZˆGä—‘×$Ñ$×-Ñ-¨b°7·:±:¸b¿h¹hÓ3GÑ.GÓHˆBÜ—‘˜R ×!5Ñ!5°aÑ!8¸$×:NÑ:NÈqÑ:QÓRˆBØ�/Ñ"×)Ñ)¨$¯.©.¸$¿-¹-È×I\ÑI\Ó]×`Ñ`Ôgl×gtÑgtÐ`ÓuˆAä—)‘)˜B˜y™M¨AÑ-Ó.×2Ñ2¸,Ð2ÓGˆBð
 —	‘	˜* d§m¡m°RÓ8¸¸dÂA¸ÑFˆAØ—‘˜ T§]¡]°D·N±NÀdÇmÁmÑ4SÐUV×U\ÑU\Ð]_ÑU`Óa×lÑlÓnˆAØ—	‘	˜* b¨!¯'©'°"©+Ó6ˆAà�I‘  3¨ªa <¡Ñ0ˆBð *×1Ñ1°*¸bÀ$Ç-Á-ÓPˆMØ˜ iÑ0Ñ0×4Ñ4¸LÐ4ÓIˆCð &×,Ñ,¨T¯^©^Ñ<×MÑMÈaÑPÐSUÑUÐX[Ñ[ˆJØ%×<Ñ<¸ZÈÏÉÓXˆJð —	‘	˜* d§m¡m°RÓ8¸¸dÂA¸ÑFˆAØ—‘˜ T§]¡]°D·N±NÀdÇmÁmÑ4SÐUV×U\ÑU\Ð]_ÑU`Óa×lÑlÓnˆAØ—	‘	˜* b¨!¯'©'°"©+Ó6ˆAð $Ÿ™¨a¯h©h¸a¿g¹g˜ÓFˆJà",§/¡/°*¸t¿~¹~Ñ2MÈtÏ}É}Ð^b×^qÑ^qÓ"rÐØŸ™ 
¨T¯^©^Ñ ;¸T×=PÑ=PÐRSÓTˆJÜ—	‘	Ð-¨zÓ:ˆAØ—‘�z 4§>¡>°4·=±=ÓAˆAð —‘�yÑ!×(Ñ(¨¯©¯©°a©¸$¿-¹-ÓHˆAØ�] QÑ&Ñ&×*Ñ*¨1¯7©7Ó3ˆAð —	‘	˜* bÓ)ª!¨T°3¨,Ñ7ŠAô —‘×'Ñ'¨¨T¯\©\Ñ(9Ó:ˆBÜ—‘˜R ×!5Ñ!5°aÑ!8¸$×:NÑ:NÈqÑ:QÓRˆBØ)×1Ñ1°*¸gÀrÈ4Ï=É=ÓY×_Ñ_ÓaˆMØ—	‘	˜* g¨r°4×3FÑ3FÓG×MÑMÓOˆAØ—	‘	˜* g¨r°4×3FÑ3FÓG×MÑMÓOˆAØ×#Ñ# D§N¡N°d·m±mÑ$CÈÐX\×XfÑXfÐ#ÓgˆAØ×#Ñ# D§N¡N°d·m±mÑ$CÈÐX\×XfÑXfÐ#ÓgˆAØŸ™¨'°D·O±OÑ*CÑCÀtÇÁÑVˆHàŸ™ 	Ñ*Ô-?ÀÈxÓ-XÑXˆJð *¨B¨y©MÑ9ˆMØ—‘�]×(Ñ(Ó)¨BÑ.ˆAð cpÐqrÐtuÐwxÐayÖ%zÐ\]Ô&9¸!¸XÀtÇÁÕ&WÒ%zÑ"ˆM˜1˜a ð —	‘	˜!˜Q  1Ó%ˆAÜ—|‘| A¨2Ô.ˆHô —	‘	œ+ a›.Ó)ˆAð šq¢!¢Q¨ªa²Ð2Ñ3°aºº1¸dÂAÂqÊ!Ð8KÑ6LÑLˆNØ×"Ñ" rÐ"Ó*ˆAð ˜y™\¨A¯I©I°a¸¸A¸qÀ!Ó,DÀYÑ,OÑOˆNØ×"Ñ" rÐ"Ó*ˆAð ˜	‘l ]²1²a¸°:Ñ%>Ñ>×CÑCÈÐCÓJˆFô !Ÿ9™9 Xªa²²A°r±s¨lÑ%;¸hÑ%FÓGˆLØ˜,×.Ñ.¨q°"°b¸!Ó<¸YÑGÑGˆGØ˜c 4ª˜lÑ+¨m¸IÑ.FÑF×KÑKÐPQÐKÓRˆFñ *ð ×#Ñ# D§N¡NÑ3×DÑDÀQÑGÊÈ4ÈÑP×SÑSÐZ`×ZfÑZfÐou×o|Ño|ÐSÔ}ä×%Ñ% fªQ°°°¨U¡mÓ4ð ô
 —Y‘Y °Ð8¸aÔ@ˆFÜŸ)™)¤K´·±×0AÑ0AÀ(Ê1ÊaÒQRÐTVÈ;ÑBWÐY_Ó0`Ó$aÓbˆKØ%×/Ñ/°°1Ó5ˆKØ% oÑ6¸ÂÂ1ÀdÈCÀÑ9PÑP×UÑUÐZ[ÐUÓ\ˆJØ *ª1¨c¨r¨c¨6Ñ 2°JºqÀ"¸uÑ4E�IˆFô $Ÿi™i¨Ó1ˆOØ  Tª1 ™o°²qº!¸TÀ3°Ñ0GÑGˆNØ'6×'>Ñ'>¸qÀ!ÀQÈÓ'JÐ$Ø#×'Ñ'¨Ó+Ð.FÀyÑ.QÑQˆEð ˜‘ˆAà—	‘	˜* b¨$¯.©.¸$¿-¹-ÓHˆAà�J‘ˆAà˜!Š|Ø’a˜˜'˜¢1¢aÐ'Ñ(�Ø—	‘	˜* g¨rÓ2ˆAð Ð$¨Ð)AØ(×?Ñ?À	È4Ï>É>ÓZ�	à—i‘i  4Ó(ˆð
 !%§¡¨k¯n©n¸UÓ.CÓ DÐØ$Ð$ùòI &{s   ä<r?rÛ   c                 ó‚  — t         rJd| j                  j                  j                  j                  v rt        «       s| j                  ||||«      S |�t        d«      ‚|j                  }|�B|j                  d   dkD  r0|j                  d   dkD  r||d d …d d …d f   z  j                  |«      }| j                  |||«      S )NÚcudaz\`seq_idx` support requires fast path support. Please install `mamba_ssm` and `causal_conv1d`r'   r   )rê   rÜ   r  r¥   Útyper!   r#  ÚNotImplementedErrorrV   r.   r_   rK  )r|   rB   ró   rQ   rô   rT   rV   s          r4   rŠ   z"GraniteMoeHybridMambaLayer.forwardÞ  sÆ   € õ " f°·±×0CÑ0C×0JÑ0J×0OÑ0OÑ&OÔXpÔXrØ×,Ñ,¨]¸LÈ.ÐZaÓbÐbØÐÜ%Ønóð ð ×#Ñ#ˆØÐ%¨.×*>Ñ*>¸qÑ*AÀAÒ*EÈ.×J^ÑJ^Ð_`ÑJaÐdeÒJeà*¨^ºAºqÀ$¸JÑ-GÑG×KÑKÈEÓRˆMà×!Ñ! -°¸~ÓNÐNr6   ©NNNr‹   )rŒ   r�   rŽ   r�   r(   r�   rn   r/   r‘   r
   Ú	IntTensorr#  rK  r   rŠ   r“   r”   s   @r4   rµ   rµ     s  ø„ ñð\8Ð5ð \8À#õ \8ðB &*Ø.2Ø*.ñbà—|‘|ðbð ˜d‘lðbð Ÿ™ tÑ+ð	bð
 —‘ 4Ñ'óbðP &*Ø.2ñ	D%ð ˜d‘lðD%ð Ÿ™ tÑ+ó	D%ñN ˜HÓ%ð &*Ø.2Ø*.ñOð ˜d‘lðOð Ÿ™ tÑ+ð	Oð
 —‘ 4Ñ'òOó &ôOr6   rµ   c                   ó(   ‡ — e Zd Zdˆ fd„	Zdd„Zˆ xZS )râ   c                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y ©N©rm   rn   r   rÝ   r/   rª   r  r  ©r|   rp   r½   r}   s      €r4   rn   z%GraniteMoeHybridRMSNormGated.__init__ö  s/   ø€ Ü‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕr6   c                 ó¤  — |j                   }|j                  t        j                  «      }|�?|t        j
                  j                  |j                  t        j                  «      «      z  }|j                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S ©Nr+   r*   T)Úkeepdim)rV   r_   r/   r^   r   r\   r  ÚpowÚmeanÚrsqrtr  r  )r|   rB   r  Úinput_dtypeÚvariances        r4   rŠ   z$GraniteMoeHybridRMSNormGated.forwardû  s¥   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆàÐØ)¬B¯M©M×,>Ñ,>¸t¿w¹wÄuÇ}Á}Ó?UÓ,VÑVˆMØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆà�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r6   ©g�íµ ÷Æ°>rT  )rŒ   r�   rŽ   rn   rŠ   r“   r”   s   @r4   râ   râ   õ  s   ø„ õ$÷
	;r6   râ   c                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚGraniteMoeHybridMLPz~
    MLP layer for shared experts

    Args:
        config:
            Configuration object with model hyperparameters.
    rh   c                 ó`  •— t         ‰| �  «        |j                  | _        |j                  | _        t
        |j                     | _        t        j                  | j                  | j                  dz  d¬«      | _
        t        j                  | j                  | j                  d¬«      | _        y )Nr+   Frk   )rm   rn   rp   Ú
input_sizeÚshared_intermediate_sizer	   rË   rÌ   r   ru   Úinput_linearÚoutput_linear©r|   rh   r}   s     €r4   rn   zGraniteMoeHybridMLP.__init__  s   ø€ Ü‰ÑÔà ×,Ñ,ˆŒØ!×:Ñ:ˆÔÜ  ×!2Ñ!2Ñ3ˆŒÜŸI™I d§o¡o°t×7GÑ7GÈ!Ñ7KÐRWÔXˆÔÜŸY™Y t×'7Ñ'7¸¿¹ÈuÔUˆÕr6   rB   rD   c                 ó¤   — | j                  |«      }|j                  dd¬«      }| j                  |d   «      |d   z  }| j                  |«      }|S )Nr+   r*   r,   r   r'   )re  ÚchunkrÌ   rf  )r|   rB   Úchunked_hidden_statess      r4   rŠ   zGraniteMoeHybridMLP.forward  s^   € Ø×)Ñ)¨-Ó8ˆØ -× 3Ñ 3°A¸2Ð 3Ó >ÐØŸ™Ð(=¸aÑ(@ÓAÐDYÐZ[ÑD\Ñ\ˆØ×*Ñ*¨=Ó9ˆØÐr6   ©
rŒ   r�   rŽ   r�   r(   rn   r/   r‘   rŠ   r“   r”   s   @r4   ra  ra    s2   ø„ ñðVÐ5õ Vð U§\¡\ð °e·l±l÷ r6   ra  c                   óÄ   ‡ — e Zd ZU ej                  ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d   de
dz  ded	ef   fd
„«       Z ej                  «       ed„ «       «       Zˆ xZS )ÚGraniteMoeHybridRotaryEmbeddingÚinv_freqNrh   c                 ó®  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |«      \  }| _
        | j                  d|d¬«       | j                  d|j                  «       d¬«       y )NÚ	rope_typeÚdefaultrn  F)Ú
persistentÚoriginal_inv_freq)rm   rn   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrh   Úrope_parametersrp  Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)r|   rh   r¥   Úrope_init_fnrn  r}   s        €r4   rn   z(GraniteMoeHybridRotaryEmbedding.__init__$  s¶   ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àˆŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÒ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆ�$Ô(à×Ñ˜Z¨¸eÐÔDØ×ÑÐ0°(·.±.Ó2BÈuÐÕUr6   r¥   ztorch.devicer  rD   ztorch.Tensorc                 ó  — | j                   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetarK   Ng      ð?r   r+   rö   r¤   )	rw  ro   rp   rq   r/   rß   Úint64r_   r  )rh   r¥   r  Úbaser-   Úattention_factorrn  s          r4   rx  z?GraniteMoeHybridRotaryEmbedding.compute_default_rope_parameters4  s�   € ð& ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r6   c                 óN  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  ||fd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j	                  |j                   ¬
«      	j	                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   r*   r'   ÚmpsÚcpuF)Údevice_typeÚenabledr+   r,   rö   )rn  r  rF   r.   r_   r¥   Ú
isinstancerN  Ústrr#   r[   r/   r0   r<   ry  r=   rV   )
r|   r1   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedr…  ÚfreqsÚembr<   r=   s
             r4   rŠ   z'GraniteMoeHybridRotaryEmbedding.forwardR  sR  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   ÃBFÆF$rT  rP  )rŒ   r�   rŽ   r/   r‘   Ú__annotations__r(   rn   Ústaticmethodr   r�   r’   r  rx  Úno_gradr   rŠ   r“   r”   s   @r4   rm  rm  !  s™   ø… Ø�l‰lÓñVÐ5õ Vð  à04Ø+/Ø"ñ*Ø&¨Ñ-ð*à˜Ñ(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ñ	%ò	*ó ð*ð: €U‡]�]ƒ_Øñ<ó ó ô<r6   rm  c                   ó”   ‡ — e Zd ZdZdefˆ fd„Zdej                  deej                  ej                  ej                  f   fd„Z	ˆ xZ
S )ÚGraniteMoeHybridTopKRoutera¡  Top-k gating that returns the routing decisions without grouping tokens by expert.

    Returns ``(top_k_index, top_k_weights, router_logits)``; the grouping/scattering used to live
    here (via ``expert_size.tolist()``, which broke fullgraph compile) and now happens inside the
    experts forward via ``use_experts_implementation`` so the default ``grouped_mm`` / ``batched_mm``
    paths can compile cleanly.
    rh   c                 óê   •— t         ‰| �  «        |j                  | _        |j                  | _        t        j                  t        j                  | j                  |j                  «      «      | _        y rT  )rm   rn   Únum_local_expertsÚnum_expertsÚnum_experts_per_tokÚtop_kr   rÝ   r/   Úemptyrp   r  rg  s     €r4   rn   z#GraniteMoeHybridTopKRouter.__init__k  sP   ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ×/Ñ/ˆŒ
Ü—l‘l¤5§;¡;¨t×/?Ñ/?À×ASÑASÓ#TÓUˆ�r6   rB   rD   c                 óô   — t        j                  || j                  «      j                  «       }|j	                  | j
                  d¬«      \  }}t        j                  |d¬«      j                  |«      }|||fS )Nr*   r,   )	ÚFÚlinearr  r  Útopkr—  r/   r]   Útype_as)r|   rB   Úrouter_logitsÚtop_k_logitsÚtop_k_indexÚtop_k_weightss         r4   rŠ   z"GraniteMoeHybridTopKRouter.forwardq  se   € ÜŸ™ °·±Ó<×BÑBÓDˆØ$1×$6Ñ$6°t·z±zÀrÐ$6Ó$JÑ!ˆ�kÜŸ™ l¸Ô;×CÑCÀMÓRˆØ˜M¨=Ð8Ð8r6   )rŒ   r�   rŽ   r�   r(   rn   r/   r‘   r’   rŠ   r“   r”   s   @r4   r’  r’  b  sI   ø„ ñðVÐ5õ Vð9 U§\¡\ð 9°e¸E¿L¹LÈ%Ï,É,ÐX]×XdÑXdÐ<dÑ6e÷ 9r6   r’  c                   ó�   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  dej                  dej                  fd„Zˆ xZ	S )	ÚGraniteMoeHybridExpertsz2Collection of expert weights stored as 3D tensors.rh   c                 óò  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        t        j                  t        j                  | j                  d| j                  z  | j
                  «      «      | _        t        j                  t        j                  | j                  | j
                  | j                  «      «      | _        t        |j                     | _        y )Nr+   )rm   rn   r”  r•  rp   Ú
hidden_dimrÈ   Úintermediate_dimr   rÝ   r/   r˜  Úgate_up_projÚ	down_projr	   rË   Úact_fnrg  s     €r4   rn   z GraniteMoeHybridExperts.__init__|  s­   ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ ×,Ñ,ˆŒØ &× 8Ñ 8ˆÔÜŸL™L¬¯©°T×5EÑ5EÀqÈ4×K`ÑK`ÑG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸ™¤e§k¡k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Ñ/ˆ�r6   rB   r   r¡  rD   c                 óf  — t        j                  |«      }t        j                  «       5  t         j                  j                  j                  || j                  ¬«      }|j                  ddd«      }t        j                  |j                  d¬«      d«      j                  «       }d d d «       D ]ð  }|d   }|| j                  k(  rŒt        j                  |   «      \  }}	||	   }
t        j                  j                  |
| j                  |   «      j                  dd¬«      \  }}| j                  |«      |z  }t        j                  j                  || j                   |   «      }|||	|d f   z  }|j#                  d|	|j%                  |j&                  «      «       Œò |S # 1 sw Y   �ŒxY w)N)Únum_classesr+   r'   r   )r*   r§   r,   r*   )r/   r-  r�  r   r\   Úone_hotr•  r,  Úgreaterr'  ÚnonzeroÚwherer›  r§  ri  r©  r¨  Ú
index_add_r_   rV   )r|   rB   r   r¡  Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stater  ÚupÚcurrent_hidden_statess                 r4   rŠ   zGraniteMoeHybridExperts.forward…  s™  € ô $×.Ñ.¨}Ó=ÐÜ�]‰]‹_ñ 	SÜŸ(™(×-Ñ-×5Ñ5°kÈt×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸ™ {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷	Sð
 %ò 
	nˆJØ# A™ˆJØ˜T×-Ñ-Ò-ØÜ#(§;¡;¨{¸:Ñ/FÓ#GÑ ˆI�yØ)¨)Ñ4ˆMÜ—}‘}×+Ñ+¨M¸4×;LÑ;LÈZÑ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆD�"Ø$(§K¡K°Ó$5¸Ñ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9NÐPT×P^ÑP^Ð_iÑPjÓ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÑ<eÑ$eÐ!Ø×*Ñ*¨1¨iÐ9N×9QÑ9QÐRe×RkÑRkÓ9lÕmð
	nð #Ð"÷#	Sñ 	Sús   ªA=F&Æ&F0rk  r”   s   @r4   r£  r£  x  sN   ø„ á<ð0Ð5õ 0ð#à—|‘|ð#ð —\‘\ð#ð —|‘|ð	#ð
 
�‰÷#r6   r£  c                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚGraniteMoeHybridMoEzISparsely-gated mixture-of-experts block: router decides, experts compute.rh   c                 ó„   •— t         ‰| �  «        |j                  | _        t	        |«      | _        t        |«      | _        y rT  )rm   rn   rp   rc  r’  Úrouterr£  Úexpertsrg  s     €r4   rn   zGraniteMoeHybridMoE.__init__£  s3   ø€ Ü‰ÑÔØ ×,Ñ,ˆŒÜ0°Ó8ˆŒÜ.¨vÓ6ˆ�r6   Úlayer_inputrD   c                 óØ   — |j                  «       \  }}}|j                  d|«      }| j                  |«      \  }}}| j                  |||«      }	|	j	                  ||| j
                  «      S )Nr*   )r¨   rG   r½  r¾  r‚   rc  )
r|   r¿  ÚbszÚlengthÚemb_sizerB   r   r¡  r  Úlayer_outputs
             r4   rŠ   zGraniteMoeHybridMoE.forward©  si   € Ø +× 0Ñ 0Ó 2ÑˆˆV�XØ#×+Ñ+¨B°Ó9ˆØ(,¯©°MÓ(BÑ%ˆ�] AØ—|‘| M°;ÀÓNˆØ× Ñ   f¨d¯o©oÓ>Ð>r6   rk  r”   s   @r4   r»  r»     s.   ø„ ÙSð7Ð5õ 7ð? 5§<¡<ð ?°E·L±L÷ ?r6   r»  c                   ó€   — e Zd ZU dZej
                  ed<   ej
                  ed<   eed<   eed<   ej                  ed<   y)ÚGraniteFlashAttentionKwargsaT  
    Keyword arguments for advanced Flash Attention, causal-conv1d, and mamba_ssm kernel usage.
    Use cases include padding-free training and fewer `torch.compile` graph breaks.

    cu_seq_lens_q (`torch.LongTensor`):
        Gets cumulative sequence length for query state.
    cu_seq_lens_k (`torch.LongTensor`):
        Gets cumulative sequence length for key state.
    max_length_q (`int`):
        Maximum sequence length for query state.
    max_length_k (`int`):
        Maximum sequence length for key state.
    seq_idx (`torch.IntTensor):
        Index of each packed sequence.
    Úcu_seq_lens_qÚcu_seq_lens_kÚmax_length_qÚmax_length_krô   N)	rŒ   r�   rŽ   r�   r/   Ú
LongTensorrŽ  r�   rQ  © r6   r4   rÆ  rÆ  ±  s7   … ñð  ×#Ñ#Ó#Ø×#Ñ#Ó#ØÓØÓØ�_‰_Ôr6   rÆ  F)ÚtotalÚRMSNormc                   óh   ‡ — e Zd Zddeddfˆ fd„Zdej                  dej                  fd„Zd„ Zˆ xZ	S )	ÚGraniteMoeHybridRMSNormr½   rD   Nc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)zF
        GraniteMoeHybridRMSNorm is equivalent to T5LayerNorm
        NrU  rV  s      €r4   rn   z GraniteMoeHybridRMSNorm.__init__Ë  s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕr6   rB   c                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S rX  )	rV   r_   r/   r^   rZ  r[  r\  r  r  )r|   rB   r]  r^  s       r4   rŠ   zGraniteMoeHybridRMSNorm.forwardÓ  sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r6   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)r’   r  r.   r  )r|   s    r4   Ú
extra_reprz"GraniteMoeHybridRMSNorm.extra_reprÚ  s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr6   r_  )
rŒ   r�   rŽ   r  rn   r/   r‘   rŠ   rÔ  r“   r”   s   @r4   rÐ  rÐ  É  s7   ø„ ñ$¨ð $¸$õ $ð; U§\¡\ð ;°e·l±ló ;öJr6   rÐ  c                   ó.  ‡ — e Zd Zdedefˆ fd„Ze	 	 	 	 ddej                  dej                  dz  de	dz  de
dz  d	eej                  ej                  f   dz  d
ee   deej                  eej                  ej                  f   dz  f   fd„«       Zˆ xZS )ÚGraniteMoeHybridDecoderLayerrh   ri   c                 ó8  •— t         ‰| �  «        |j                  | _        d | _        t	        |j                  |j
                  ¬«      | _        t	        |j                  |j
                  ¬«      | _        |j                  dkD  rt        |«      nd | _
        |j                  | _        t        |«      | _        d | _        |j                  |   dk(  rt!        ||«      | _        nt#        ||«      | _        |j                  |   | _        t'        |dd«      dkD  | _        y )Nr¼   r   Úlinear_attentionr”  )rm   rn   rp   Ú	self_attnrÐ  rÐ   Úinput_layernormÚpost_attention_layernormr”  r»  Úblock_sparse_moeÚresidual_multiplierra  Ú
shared_mlpÚmambaÚlayers_block_typerµ   rg   Ú
block_typero   Úhas_expertsr{   s      €r4   rn   z%GraniteMoeHybridDecoderLayer.__init__ß  sù   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔàˆŒÜ6°v×7IÑ7IÈv×ObÑObÔcˆÔÜ(?À×@RÑ@RÐX^×XkÑXkÔ(lˆÔ%ð @F×?WÑ?WÐZ[Ò?[Ô 3°FÔ ;ÐaeˆÔØ#)×#=Ñ#=ˆÔ Ü-¨fÓ5ˆŒØˆŒ
à×#Ñ# IÑ.Ð2DÒDÜ3°F¸IÓFˆD�Jä6°v¸yÓIˆDŒNØ ×2Ñ2°9Ñ=ˆŒô # 6Ð+>ÀÓBÀQÑFˆÕr6   NrB   rQ   r~   Ú	use_cacher   rT   rD   c           	      óœ  — |}| j                  |«      }| j                  � | j                  d|||dœ|¤Ž}n | j                  d|||||dœ|¤Ž\  }}||| j                  z  z   }|}| j	                  |«      }| j
                  r&| j                  |«      }	|	| j                  |«      z   }n| j                  |«      }||| j                  z  z   }|S )N)rB   ró   rQ   )rB   rQ   r~   rã  r   rÌ  )rÚ  rß  rÙ  rÝ  rÛ  râ  rÜ  rÞ  )
r|   rB   rQ   r~   rã  r   rT   Úresidualr  Úmoe_hidden_statess
             r4   rŠ   z$GraniteMoeHybridDecoderLayer.forwardö  s  € ð !ˆØ×,Ñ,¨]Ó;ˆà�:‰:Ð!Ø&˜DŸJ™Jð Ø+Ø,Ø-ñð ñ	‰Mð  .˜tŸ~™~ð  Ø+Ø-Ø /Ø#Ø$7ñ ð ñ ÑˆM˜1ð ! =°4×3KÑ3KÑ#KÑKˆØ ˆØ×5Ñ5°mÓDˆà×ÒØ $× 5Ñ 5°mÓ DÐØ-°·±ÀÓ0NÑN‰Mà ŸO™O¨MÓ:ˆMà  =°4×3KÑ3KÑ#KÑKˆØÐr6   )NNFN)rŒ   r�   rŽ   r(   r�   rn   r   r/   r‘   r
   r«   r’   r   rÆ  ÚFloatTensorrŠ   r“   r”   s   @r4   rÖ  rÖ  Þ  sÞ   ø„ ðGÐ5ð GÀ#õ Gð. ð /3Ø(,Ø!&ØHLñ(à—|‘|ð(ð Ÿ™ tÑ+ð(ð  ™ð	(ð
 ˜$‘;ð(ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð(ð Ð4Ñ5ð(ð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	Uò(ó ô(r6   rÖ  c                   óˆ   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZeedœZdZ ej&                  «       ˆ fd„«       Zˆ xZS )ÚGraniteMoeHybridPreTrainedModelrh   ÚmodelTrÖ  r~   )rB   Ú
attentionsc           
      óP  •— t         ‰| �  |«       t        |t        «      rmt	        j
                  |j                  d| j                  j                  ¬«       t	        j
                  |j                  d| j                  j                  ¬«       nFt        |t        «      r6t	        j
                  |j                  d| j                  j                  ¬«       t        |t        «      r“t	        j                  |j                  «       t	        j                  |j                   t#        j$                  t#        j&                  d|j(                  dz   «      «      «       t	        j                  |j*                  «       y t        |t,        «      r t	        j                  |j                  «       y y )Nr�   )r[  Ústdr'   )rm   Ú_init_weightsr‡  r£  ÚinitÚnormal_r§  rh   Úinitializer_ranger¨  r’  r  rµ   Úones_rÞ   Úcopy_rá   r/   rà   rß   rÂ   rä   râ   )r|   rM   r}   s     €r4   rî  z-GraniteMoeHybridPreTrainedModel._init_weights4  s  ø€ ä‰Ñ˜fÔ%Ü�fÔ5Ô6Ü�L‰L˜×,Ñ,°3¸D¿K¹K×<YÑ<YÕZÜ�L‰L˜×)Ñ)°¸¿¹×9VÑ9VÖWÜ˜Ô :Ô;Ü�L‰L˜Ÿ™¨S°d·k±k×6SÑ6SÕTÜ�fÔ8Ô9Ü�J‰J�v—~‘~Ô&Ü�J‰J�v—|‘|¤U§Y¡Y¬u¯|©|¸A¸v×?OÑ?OÐRSÑ?SÓ/TÓ%UÔVÜ�J‰J�v—x‘xÕ Ü˜Ô <Ô=Ü�J‰J�v—}‘}Õ%ð >r6   )rŒ   r�   rŽ   r(   rŽ  Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrÖ  rg   Ú_can_record_outputsÚ_is_statefulr/   r�  rî  r“   r”   s   @r4   ré  ré  "  ss   ø… à"Ó"ØÐØ&*Ð#Ø7Ð8ÐØ#4Ð"5ÐØÐØ€NØÐØ!ÐØ"&Ðà5Ø/ñÐð €Là€U‡]�]ƒ_ó&ó ô&r6   ré  c                   óø   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   deez  fd„«       «       «       Zˆ xZS )ÚGraniteMoeHybridModelrh   c           	      óN  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                  ¬«      | _        |j"                  dk(  rt%        |«      nd | _        d| _        |j*                  | _        | j-                  «        y c c}w )Nr¼   ÚropeF)rm   rn   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingrp   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrÖ  r  rÐ  rÐ   rã   Úposition_embedding_typerm  Ú
rotary_embÚgradient_checkpointingÚembedding_multiplierÚ	post_initr{   s      €r4   rn   zGraniteMoeHybridModel.__init__F  sç   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m‘mÜNSÐTZ×TlÑTlÓNmÖnÀÔ)¨&°)Õ<Ònó
ˆŒô ,¨F×,>Ñ,>ÀF×DWÑDWÔXˆŒ	ØEK×EcÑEcÐgmÒEmÔ9¸&ÔAÐswˆŒØ&+ˆÔ#Ø$*×$?Ñ$?ˆÔ!ð 	�‰Õùò os   ÂD"NÚ	input_idsrQ   r‰  r~   Úinputs_embedsrã  rT   rD   c           	      óÐ  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|| j                  z  }|r|€t        | j                  ¬«      }|€V|�|j                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      s(| j                  |||dœ}
t        d
i |
¤Žt        d
i |
¤Ždœ}	|}d }| j                  �| j                  ||«      }t!        | j"                  «      D ]-  \  }} ||f|	| j                  j$                  |      |||dœ|¤Ž}Œ/ | j'                  |«      }t)        ||¬	«      S )Nz:You must specify exactly one of input_ids or inputs_embeds)rh   r   r'   r%  )rh   r  rQ   r~   )Úfull_attentionrØ  )rQ   r~   rã  r   )Úlast_hidden_stater~   rÌ  )Ú
ValueErrorr  r  r   rh   Úget_seq_lengthr/   rß   r.   r¥   r9   r‡  Údictr   r   r  Ú	enumerater  rà  rã   r   )r|   r  rQ   r‰  r~   r  rã  rT   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsrB   r   ÚiÚdecoder_layers                  r4   rŠ   zGraniteMoeHybridModel.forwardW  s   € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMà%¨×(AÑ(AÑAˆá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-ÄÔFð Ÿ+™+Ø!.Ø"0Ø#2ñ	ˆKô #5Ñ"C°{Ñ"CÜ$CÑ$RÀkÑ$Rñ#Ðð &ˆØ"ÐØ�?‰?Ð&Ø"&§/¡/°-ÀÓ"NÐä )¨$¯+©+Ó 6ò 	ÑˆAˆ}Ù)Øðà2°4·;±;×3PÑ3PÐQRÑ3SÑTØ /Ø#Ø$7ñð ñ‰Mð	ð Ÿ	™	 -Ó0ˆä%Ø+Ø+ô
ð 	
r6   )NNNNNN)rŒ   r�   rŽ   r(   rn   r   r$   r&   r/   rË  r‘   r
   rç  r«   r   rÆ  r’   r   rŠ   r“   r”   s   @r4   r   r   D  sÔ   ø„ ðÐ5õ ð" ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ<
à×#Ñ# dÑ*ð<
ð Ÿ™ tÑ+ð<
ð ×&Ñ&¨Ñ-ð	<
ð
  ™ð<
ð ×(Ñ(¨4Ñ/ð<
ð ˜$‘;ð<
ð Ð4Ñ5ð<
ð 
Ð(Ñ	(ò<
ó ó  ó ô<
r6   r   Úgate_logitsr•  c                 ó¾  — | �t        | t        «      syt        | t        «      rC| d   j                  }t        j                  | D �cg c]  }|j                  |«      ‘Œ c}d¬«      }t        j                  j                  j                  d¬«      }t        j                  ||d¬«      \  }}	t        j                  j                  j                  |	|«      }
|€>t        j                  |
j                  «       d¬«      }t        j                  |d¬«      }�n|j                  \  }}|j                  d   ||z  z  }|ddd…dd…ddf   j                  |||||f«      j                  d||«      j                  «      }t        j                   |
j                  «       |z  d¬«      t        j                   |d¬«      z  }|ddd…dd…df   j                  ||||f«      j                  d|«      j                  |«      }t        j                   ||z  d¬«      t        j                   |d¬«      z  }t        j                   ||j#                  d«      z  «      }||z  S c c}w )aÄ  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   r,   r*   )r‡  r’   r¥   r/   r0   r_   r   r\   r]   rœ  r¬  r[  r  r.   rF   rG   r'  r9   )r  r•  r—  rQ   Úcompute_deviceÚ
layer_gateÚconcatenated_gate_logitsÚrouting_weightsr  Úselected_expertsr²  Útokens_per_expertÚrouter_prob_per_expertr  Úsequence_lengthr
  Úexpert_attention_maskÚ router_per_expert_attention_maskÚoverall_losss                      r4   Úload_balancing_loss_funcr+  ™  s9  € ð: Ð¤*¨[¼%Ô"@Øä�+œuÔ%Ø$ Q™×.Ñ.ˆÜ#(§9¡9Ð^iÖ-jÐPZ¨j¯m©m¸NÕ.KÒ-jÐpqÔ#rÐ ä—h‘h×)Ñ)×1Ñ1Ð2JÐPRÐ1ÓS€OäŸ*™* _°eÀÔDÑ€AÐä—(‘(×%Ñ%×-Ñ-Ð.>ÀÓL€KàÐä!ŸJ™J {×'8Ñ'8Ó':ÀÔBÐô "'§¡¨OÀÔ!CÒà&4×&:Ñ&:Ñ#ˆ
�OØ4×:Ñ:¸1Ñ=À*ÈÑB^Ñ_Ðð ˜4¢¢A t¨TÐ1Ñ2ß‰VÐ&¨
°OÀUÈKÐXÓYß‰W�R˜ Ó,ß‰R�Óð	 	ô "ŸI™I k×&7Ñ&7Ó&9Ð<QÑ&QÐWXÔYÔ\a×\eÑ\eØ! qô]
ñ 
Ðð ˜4¢¢A tÐ+Ñ,ß‰VÐ&¨
°OÀ[ÐQÓRß‰W�R˜Ó%ß‰R�Óð	 	)ô "'§¡¨?Ð=]Ñ+]ÐcdÔ!eÔhm×hqÑhqØ,°!ôi
ñ "
Ðô —9‘9Ð.Ð1G×1QÑ1QÐRSÓ1TÑTÓU€LØ˜+Ñ%Ð%ùò[ .ks   ÁIc                   óD  ‡ — e Zd ZddiZddiZddgdgfiZdefˆ fd„Zee		 	 	 	 	 	 	 	 dd
e
j                  d	z  de
j                  d	z  de
j                  d	z  ded	z  de
j                  d	z  de
j                  d	z  ded	z  dee
j                  z  deez  fd„«       «       Zˆ xZS )ÚGraniteMoeHybridForCausalLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrB   Úlogitsrh   c                 óp  •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  |j                  d¬«      | _        |j                  | _	        |j                  | _        |j                  | _        |j                  | _        | j                  «        y )NFrk   )rm   rn   r   rê  r  r   ru   rp   r.  Úrouter_aux_loss_coefr”  r•  r–  Úlogits_scalingr  rg  s     €r4   rn   z$GraniteMoeHybridForCausalLM.__init__ñ  s�   ø€ Ü‰Ñ˜Ô Ü*¨6Ó2ˆŒ
Ø ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ$*×$?Ñ$?ˆÔ!Ø!×3Ñ3ˆÔØ#)×#=Ñ#=ˆÔ Ø$×3Ñ3ˆÔð 	�‰Õr6   Nr  rQ   r‰  r~   r  ÚlabelsÚoutput_router_logitsÚlogits_to_keeprD   c	           	      ó®  — |�|n| j                   j                  } | j                  d|||||dœ|	¤Ž}
|
j                  }t	        |t
        «      rt        | d«      n|}| j                  |dd…|dd…f   «      }|| j                   j                  z  }d}|�* | j                  ||fd| j                   j                  i|	¤Ž}d}|rYt        |
j                  | j                  | j                  |«      }|�+|| j                  |j!                  |j"                  «      z  z  }t%        ||||
j&                  |
j(                  |
j*                  |
j                  ¬«      S )a–  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, GraniteMoeHybridForCausalLM

        >>> model = GraniteMoeHybridForCausalLM.from_pretrained("ibm-granite/granite-4.0-h-tiny")
        >>> tokenizer = AutoTokenizer.from_pretrained("ibm-granite/granite-4.0-h-tiny")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r  rQ   r‰  r~   r  r  )ÚlossÚaux_lossr0  r~   rB   rë  rž  rÌ  )rh   r5  rê  r  r‡  r�   Úslicer.  r3  Úloss_functionr  r+  rž  r•  r–  r2  r_   r¥   r   r~   rB   rë  )r|   r  rQ   r‰  r~   r  r4  r5  r6  rT   ÚoutputsrB   Úslice_indicesr0  r8  r9  s                   r4   rŠ   z#GraniteMoeHybridForCausalLM.forwardþ  s|  € ðJ %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð �$—*‘*ð 
ØØ)Ø%Ø+Ø'ñ
ð ñ
ˆð  ×1Ñ1ˆÜ8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆØ˜$Ÿ+™+×4Ñ4Ñ4ˆàˆØÐà%�4×%Ñ%ØØñð  Ÿ;™;×1Ñ1ðð ñ	ˆDð ˆÙÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ð!Ø˜×1Ñ1°H·K±KÀÇÁÓ4LÑLÑL�Ü(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r6   )NNNNNNNr   )rŒ   r�   rŽ   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr(   rn   r   r    r/   rË  r‘   r
   rç  r«   r�   r’   r   rŠ   r“   r”   s   @r4   r-  r-  ë  s   ø„ à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HðÐ5õ ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø,0Ø-.ñQ
à×#Ñ# dÑ*ðQ
ð Ÿ™ tÑ+ðQ
ð ×&Ñ&¨Ñ-ð	Q
ð
  ™ðQ
ð ×(Ñ(¨4Ñ/ðQ
ð × Ñ  4Ñ'ðQ
ð # T™kðQ
ð ˜eŸl™lÑ*ðQ
ð 
Ð*Ñ	*òQ
ó ó ôQ
r6   r-  )r-  r   ré  )r'   )r�   )Nr+   N)]Úcollections.abcr   Útypingr   r   r/   Útorch.nn.functionalr   r\   rš  Ú r   rï  Úactivationsr	   Úcache_utilsr
   r   Ú
generationr   Úintegrationsr   r   r   r   Úintegrations.accelerater   Úintegrations.hub_kernelsr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r    r!   r"   Úutils.genericr#   r$   Úutils.import_utilsr%   Úutils.output_capturingr&   Úconfiguration_granitemoehybridr(   Ú
get_loggerrŒ   rë   r5   rA   r‘   r�   rL   ÚModuler  re   rg   rž   r¡   r±   r³   rµ   râ   ra  rm  r’  r£  r»  rÆ  rÐ  rÖ  ré  r   r’   r+  r-  Ú__all__rÌ  r6   r4   ú<module>rY     s  ðõ* %ß &ã ß Ð Ý å &Ý !ß .Ý )÷ó õ >Ý 8ß PÝ 9ß jÑ jß Kß FÝ &ß lÕ lß GÝ 9Ý 5Ý Bð 
ˆ×	Ñ	˜HÓ	%€ò(ñ Ð*Ó+òó ,ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(ó%ñ2 Ð)Ó*ôA) §	¡	ó A)ó +ðA)ðNV U§\¡\ð V¸Só Vò
ò(ò(	ôlO §¡ô lOô^; 5§8¡8§?¡?ô ;ô$˜"Ÿ)™)ô ô4>< b§i¡iô ><ôB9 §¡ô 9ð, ô$#˜bŸi™ió $#ó ð$#ôN?˜"Ÿ)™)ô ?ô" )°5õ ñ0 ˜YÓ'ôJ˜bŸi™ió Jó (ðJô(AÐ#=ô AðH ô& oó &ó ð&ðB ôQ
Ð;ó Q
ó ðQ
ðl #Ø
Ø*.ñ	O&Ø—‘  e§l¡lÑ 3Ñ3°dÑ:ðO&à�t‘ðO&ð —L‘L 4Ñ'ð	O&ð
 ‡\�\�CÑóO&ðd ôe
Ð"AÀ?ó e
ó ðe
òP f�r6   