Ë
    ÜÓ^j°�  ã                   óž  — d dl mZ d dlmZ d dlZd dlmc mZ d dlmZ ddl	m
Z ddlmZ ddlmZmZ dd	lmZ dd
lmZmZmZmZ ddlmZmZ ddlmZ ddlmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+m,Z, ddl-m.Z.m/Z/m0Z0 ddl1m2Z2m3Z3 ddl4m5Z5  G d„ dejl                  «      Z7 ed«       G d„ dejl                  «      «       Z8d„ Z9 ed«      dBd„«       Z:dejv                  d e<d!ejv                  fd"„Z=	 dCd#ejl                  d$ejv                  d%ejv                  d&ejv                  d'ejv                  dz  d(e>d)e>d*e)e.   fd+„Z? ee:«       G d,„ d-ejl                  «      «       Z@e G d.„ d/ejl                  «      «       ZA G d0„ d1ejl                  «      ZB G d2„ d3ejl                  «      ZC G d4„ d5ejl                  «      ZD G d6„ d7e«      ZEe+ G d8„ d9e'«      «       ZFe+ G d:„ d;eF«      «       ZG	 	 	 dDd<ejv                  eHejv                     z  dz  d=e<dz  d'ejv                  dz  d!ejv                  e<z  fd>„ZIe+ G d?„ d@eFe«      «       ZJg dA¢ZKy)Eé    )ÚCallable)ÚOptionalN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hubÚuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úauto_docstringÚcan_return_tuple)ÚTransformersKwargsÚmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚMellumConfigc                   óÐ   ‡ — e Zd ZU ej                  ed<   defˆ fd„Ze	 	 	 	 ddedz  de	d   de
dz  dedz  d	ed
ef   f
d„«       Z ej                  «       edd„«       «       Zˆ xZS )ÚMellumRotaryEmbeddingÚinv_freqÚconfigc                 ót  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        t        t        |j                  «      «      | _        i | _	        | j                  D ]Ç  }| j
                  j                  |   }|€Œ|d   | j                  |<   | j                  }| j                  |   dk7  rt        | j                  |      } || j
                  |¬«      \  }}| j                  |› d�|d¬«       | j                  |› d�|j                  «       d¬«       t        | |› d�|«       ŒÉ y )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqF)Ú
persistentÚ_original_inv_freqÚ_attention_scaling)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr'   ÚlistÚsetÚlayer_typesr)   Úrope_parametersÚcompute_default_rope_parametersr   Úregister_bufferÚcloneÚsetattr)Úselfr'   r,   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingÚ	__class__s          €úu/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/mellum/modeling_mellum.pyr2   zMellumRotaryEmbedding.__init__6   s6  ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØˆŒÜ¤ F×$6Ñ$6Ó 7Ó8ˆÔØˆŒØ×*Ñ*ò 	UˆJØŸ+™+×5Ñ5°jÑAˆKØÐ"Øà)4°[Ñ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÑ)¨YÒ6Ü2°4·>±>À*Ñ3MÑN�Ù4@ÀÇÁÐYcÔ4dÑ1ˆMÐ1Ø× Ñ  J <¨yÐ!9¸=ÐUZÐ Ô[Ø× Ñ  J <Ð/AÐ!BÀM×DWÑDWÓDYÐfkÐ ÔlÜ�D˜Z˜LÐ(:Ð;Ð=SÕTñ	Uó    NÚdeviceztorch.deviceÚseq_lenr,   Úreturnztorch.Tensorc                 óz  — | j                   |   d   }| j                   |   j                  dd«      }t        | dd«      xs | j                  | j                  z  }t        ||z  «      }d}d|t        j                  d|dt        j                  ¬«      j                  |t        j                  ¬	«      |z  z  z  }	|	|fS )
a{  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimNr   é   ©Údtype)rF   rO   )r9   ÚgetÚgetattrÚhidden_sizeÚnum_attention_headsÚintÚtorchÚarangeÚint64ÚtoÚfloat)
r'   rF   rG   r,   ÚbaserK   rL   ÚdimÚattention_factorr&   s
             rD   r:   z5MellumRotaryEmbedding.compute_default_rope_parametersK   sÅ   € ð. ×%Ñ% jÑ1°,Ñ?ˆà &× 6Ñ 6°zÑ B× FÑ FÐG^Ð`cÓ dÐÜ˜6 :¨tÓ4Òh¸×8JÑ8JÈf×NhÑNhÑ8hˆÜ�(Ð2Ñ2Ó3ˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)rE   c                 óN  — t        | |› d�«      }t        | |› d�«      }|d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬	«      5  |j                  «       |j                  «       z  j                  dd
«      }	t        j                  |	|	fd¬«      }
|
j                  «       |z  }|
j                  «       |z  }d d d «       j	                  |j                  ¬«      j	                  |j                  ¬«      fS # 1 sw Y   ŒAxY w)Nr-   r0   r   éÿÿÿÿr"   ÚmpsÚcpuF)Údevice_typeÚenabledrM   ©r[   rN   )rQ   rY   ÚexpandÚshaperX   rF   Ú
isinstanceÚtypeÚstrr   Ú	transposerU   ÚcatÚcosÚsinrO   )r>   ÚxÚposition_idsr,   r&   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedra   ÚfreqsÚembrk   rl   s                rD   ÚforwardzMellumRotaryEmbedding.forwardp   sl  € ô ˜4 J <¨yÐ!9Ó:ˆÜ# D¨Z¨LÐ8JÐ*KÓLÐà$ Tª1¨d ]Ñ3×9Ñ9Ó;×BÑBÀ<×CUÑCUÐVWÑCXÐZ\Ð^_Ó`×cÑcÐde×dlÑdlÓmÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	0Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)Ð/Ñ/ˆCØ—'‘'“)Ð/Ñ/ˆC÷		0ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	0ð 	0ús   Ã*A1FÆF$)NNNN©N)Ú__name__Ú
__module__Ú__qualname__rU   ÚTensorÚ__annotations__r#   r2   Ústaticmethodr   rT   rh   ÚtuplerY   r:   Úno_gradr   rt   Ú__classcell__©rC   s   @rD   r%   r%   3   sª   ø… Ø�l‰lÓðU˜|õ Uð* à&*Ø+/Ø"Ø!%ñ	"*Ø˜tÑ#ð"*à˜Ñ(ð"*ð �t‘ð"*ð ˜$‘Jð	"*ð
 
ˆ~˜uÐ$Ñ	%ò"*ó ð"*ðH €U‡]�]ƒ_Øò<ó ó ô<rE   r%   ÚRMSNormc                   óh   ‡ — e Zd Zddeddfˆ fd„Zdej                  dej                  fd„Zd„ Zˆ xZ	S )	ÚMellumRMSNormÚepsrH   Nc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)z<
        MellumRMSNorm is equivalent to T5LayerNorm
        N)r1   r2   r   Ú	ParameterrU   ÚonesÚweightÚvariance_epsilon)r>   rR   rƒ   rC   s      €rD   r2   zMellumRMSNorm.__init__…   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕrE   Úhidden_statesc                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S )NrM   r^   T)Úkeepdim)	rO   rX   rU   Úfloat32ÚpowÚmeanÚrsqrtrˆ   r‡   )r>   r‰   Úinput_dtypeÚvariances       rD   rt   zMellumRMSNorm.forward�   sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:rE   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)r|   r‡   re   rˆ   )r>   s    rD   Ú
extra_reprzMellumRMSNorm.extra_repr”   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIrE   )g�íµ ÷Æ°>)
rv   rw   rx   rY   r2   rU   ry   rt   r“   r~   r   s   @rD   r‚   r‚   ƒ   s7   ø„ ñ$¨ð $¸$õ $ð; U§\¡\ð ;°e·l±ló ;öJrE   r‚   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Nr^   rM   rc   )re   rU   rj   )rm   Úx1Úx2s      rD   Úrotate_halfr—   ˜   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'rE   Úrotary_pos_embc                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer—   )ÚqÚkrk   rl   Úunsqueeze_dimÚq_embedÚk_embeds          rD   Úapply_rotary_pos_embr    Ÿ   sY   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐrE   r‰   Ún_reprH   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r"   N)re   rd   Úreshape)r‰   r¡   ÚbatchÚnum_key_value_headsÚslenrL   s         rD   Ú	repeat_kvr§   ¹   so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTrE   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t        || j                  «      }t        || j                  «      }	t        j                  ||j	                  dd«      «      |z  }
|�|
|z   }
t
        j                  j                  |
dt        j                  ¬«      j                  |j                  «      }
t
        j                  j                  |
|| j                  ¬«      }
t        j                  |
|	«      }|j	                  dd«      j                  «       }||
fS )NrM   r   r^   )r[   rO   )ÚpÚtrainingr"   )r§   Únum_key_value_groupsrU   Úmatmulri   r   Ú
functionalÚsoftmaxrŒ   rX   rO   r®   r²   Ú
contiguous)r¨   r©   rª   r«   r¬   r­   r®   r¯   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               rD   Úeager_attention_forwardr¼   Å   sà   € ô ˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$rE   c                   óô   ‡ — e Zd ZdZdedefˆ fd„Z	 ddej                  de	ej                  ej                  f   dej                  dz  d	e
dz  d
ee   de	ej                  ej                  dz  f   fd„Zˆ xZS )ÚMellumAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr'   Ú	layer_idxc                 óR  •— t         ‰| �  «        || _        || _        t	        |d|j
                  |j                  z  «      | _        |j                  |j                  z  | _	        | j                  dz  | _
        |j                  | _        d| _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j                  | j                  z  |j
                  |j                  ¬«      | _        t)        | j                  |j*                  ¬«      | _        t)        | j                  |j*                  ¬«      | _        |j0                  |   dk(  r|j2                  | _        y d | _        y )NrL   g      à¿T©Úbias©rƒ   Úsliding_attention)r1   r2   r'   r¿   rQ   rR   rS   rL   r¥   r³   r­   Úattention_dropoutÚ	is_causalr   ÚLinearÚattention_biasÚq_projÚk_projÚv_projÚo_projr‚   Úrms_norm_epsÚq_normÚk_normr8   Úsliding_window©r>   r'   r¿   rC   s      €rD   r2   zMellumAttention.__init__â   s¨  ø€ Ü‰ÑÔØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒô $ D§M¡M°v×7JÑ7JÔKˆŒÜ# D§M¡M°v×7JÑ7JÔKˆŒØ7=×7IÑ7IÈ)Ñ7TÐXkÒ7k˜f×3Ñ3ˆÕÐquˆÕrE   Nr‰   Úposition_embeddingsr¬   Úpast_key_valuesr¯   rH   c                 ó\  — |j                   d d }g |¢d‘| j                  ‘­}| j                  | j                  |«      j	                  |«      «      j                  dd«      }| j                  | j                  |«      j	                  |«      «      j                  dd«      }	| j                  |«      j	                  |«      j                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t         «      } || ||	|
|f| j"                  sdn| j$                  | j&                  | j(                  dœ|¤Ž\  }} |j*                  g |¢d‘­Ž j-                  «       }| j/                  |«      }||fS )Nr^   r"   rM   ç        )r®   r­   rÐ   )re   rL   rÎ   rÉ   Úviewri   rÏ   rÊ   rË   r    Úupdater¿   r   Úget_interfacer'   Ú_attn_implementationr¼   r²   rÅ   r­   rÐ   r£   r·   rÌ   )r>   r‰   rÒ   r¬   rÓ   r¯   Úinput_shapeÚhidden_shapeÚquery_statesr¸   r¹   rk   rl   Úattention_interfacer»   rº   s                   rD   rt   zMellumAttention.forwardü   s®  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ 4§;¡;¨}Ó#=×#BÑ#BÀ<Ó#PÓQ×[Ñ[Ð\]Ð_`ÓaˆØ—[‘[ §¡¨]Ó!;×!@Ñ!@ÀÓ!NÓO×YÑYÐZ[Ð]^Ó_ˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(rE   ru   )rv   rw   rx   Ú__doc__r#   rT   r2   rU   ry   r|   r	   r   r   rt   r~   r   s   @rD   r¾   r¾   Þ   s    ø„ áGðv˜|ð v¸õ vð> )-ñ')à—|‘|ð')ð # 5§<¡<°·±Ð#=Ñ>ð')ð Ÿ™ tÑ+ð	')ð
  ™ð')ð Ð-Ñ.ð')ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷')rE   r¾   c                   óŠ   ‡ — e Zd ZdZˆ fd„Zdej                  dej                  dej                  dej                  fd„Zˆ xZS )ÚMellumExpertsz2Collection of expert weights stored as 3D tensors.c                 óò  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        t        j                  t        j                  | j                  d| j                  z  | j                  «      «      | _        t        j                  t        j                  | j                  | j                  | j                  «      «      | _        t        |j                     | _        y )NrM   )r1   r2   Únum_expertsrR   Ú
hidden_dimÚmoe_intermediate_sizeÚintermediate_dimr   r…   rU   ÚemptyÚgate_up_projÚ	down_projr   Ú
hidden_actÚact_fn©r>   r'   rC   s     €rD   r2   zMellumExperts.__init__*  s­   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ ×,Ñ,ˆŒØ &× <Ñ <ˆÔÜŸL™L¬¯©°T×5EÑ5EÀqÈ4×K`ÑK`ÑG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸ™¤e§k¡k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Ñ/ˆ�rE   r‰   Útop_k_indexÚtop_k_weightsrH   c                 óf  — t        j                  |«      }t        j                  «       5  t         j                  j                  j                  || j                  ¬«      }|j                  ddd«      }t        j                  |j                  d¬«      d«      j                  «       }d d d «       D ]ð  }|d   }|| j                  k(  rŒt        j                  |   «      \  }}	||	   }
t        j                  j                  |
| j                  |   «      j                  dd¬«      \  }}| j                  |«      |z  }t        j                  j                  || j                   |   «      }|||	|d f   z  }|j#                  d|	|j%                  |j&                  «      «       Œò |S # 1 sw Y   �ŒxY w)N)Únum_classesrM   r"   r   )r^   éþÿÿÿrc   r^   )rU   Ú
zeros_liker}   r   rµ   Úone_hotrâ   ÚpermuteÚgreaterÚsumÚnonzeroÚwhereÚlinearrç   Úchunkrê   rè   Ú
index_add_rX   rO   )r>   r‰   rì   rí   Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess                 rD   rt   zMellumExperts.forward3  s™  € ô $×.Ñ.¨}Ó=ÐÜ�]‰]‹_ñ 	SÜŸ(™(×-Ñ-×5Ñ5°kÈt×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸ™ {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷	Sð
 %ò 
	nˆJØ# A™ˆJØ˜T×-Ñ-Ò-ØÜ#(§;¡;¨{¸:Ñ/FÓ#GÑ ˆI�yØ)¨)Ñ4ˆMÜ—}‘}×+Ñ+¨M¸4×;LÑ;LÈZÑ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆD�"Ø$(§K¡K°Ó$5¸Ñ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9NÐPT×P^ÑP^Ð_iÑPjÓ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÑ<eÑ$eÐ!Ø×*Ñ*¨1¨iÐ9N×9QÑ9QÐRe×RkÑRkÓ9lÕmð
	nð #Ð"÷#	Sñ 	Sús   ªA=F&Æ&F0)	rv   rw   rx   rÞ   r2   rU   ry   rt   r~   r   s   @rD   rà   rà   &  sF   ø„ á<ô0ð#à—|‘|ð#ð —\‘\ð#ð —|‘|ð	#ð
 
�‰÷#rE   rà   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚMellumTopKRouterc                 ó.  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        t        j                  t        j                  | j                  | j                  «      «      | _        y ru   )r1   r2   Únum_experts_per_tokÚtop_krâ   Únorm_topk_probrR   rã   r   r…   rU   Úzerosr‡   rë   s     €rD   r2   zMellumTopKRouter.__init__O  si   ø€ Ü‰ÑÔØ×/Ñ/ˆŒ
Ø!×-Ñ-ˆÔØ$×3Ñ3ˆÔØ ×,Ñ,ˆŒÜ—l‘l¤5§;¡;¨t×/?Ñ/?ÀÇÁÓ#QÓRˆ�rE   c                 óÀ  — |j                  d| j                  «      }t        j                  || j                  «      }t
        j                  j                  j                  |t
        j                  d¬«      }t        j                  || j                  d¬«      \  }}| j                  r||j                  dd¬«      z  }|j                  |j                  «      }|}|||fS )Nr^   )rO   r[   rc   T)r[   r‹   )r£   rã   ÚFrø   r‡   rU   r   rµ   r¶   rY   Útopkr	  r
  rõ   rX   rO   )r>   r‰   Úrouter_logitsÚrouter_probsÚrouter_top_valueÚrouter_indicesÚrouter_scoress          rD   rt   zMellumTopKRouter.forwardW  s½   € Ø%×-Ñ-¨b°$·/±/ÓBˆÜŸ™ °·±Ó<ˆÜ—x‘x×*Ñ*×2Ñ2°=ÌÏÉÐY[Ð2Ó\ˆÜ+0¯:©:°lÀDÇJÁJÐTVÔ+WÑ(Ð˜.Ø×ÒØÐ 0× 4Ñ 4¸ÀTÐ 4Ó JÑJÐØ+×.Ñ.¨}×/BÑ/BÓCÐØ(ˆØ˜m¨^Ð;Ð;rE   ©rv   rw   rx   r2   rt   r~   r   s   @rD   r  r  N  s   ø„ ôSö	<rE   r  c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚMellumSparseMoeBlockr'   c                 ób   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        y ru   )r1   r2   rà   Úexpertsr  r  rë   s     €rD   r2   zMellumSparseMoeBlock.__init__d  s&   ø€ Ü‰ÑÔÜ$ VÓ,ˆŒÜ$ VÓ,ˆ�	rE   r‰   rH   c                 ó¼   — |j                   \  }}}|j                  d|«      }| j                  |«      \  }}}| j                  |||«      }	|	j	                  |||«      S )Nr^   )re   rÖ   r  r  r£   )
r>   r‰   Ú
batch_sizeÚsequence_lengthrã   Úhidden_states_reshapedÚ_Úrouting_weightsÚselected_expertsrû   s
             rD   rt   zMellumSparseMoeBlock.forwardi  si   € Ø2?×2EÑ2EÑ/ˆ
�O ZØ!.×!3Ñ!3°B¸
Ó!CÐØ/3¯y©yÐ9OÓ/PÑ,ˆˆ?Ð,Ø"Ÿl™lÐ+AÐCSÐUdÓeÐØ"×*Ñ*¨:°È
ÓSÐSrE   )	rv   rw   rx   r#   r2   rU   ry   rt   r~   r   s   @rD   r  r  c  s-   ø„ ð-˜|õ -ð
T U§\¡\ð T°e·l±l÷ TrE   r  c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )Ú	MellumMLPc                 óÒ  •— t         ‰| �  «        || _        |j                  | _        |€|j                  n|| _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _	        t        |j                     | _        y ©NFrÁ   )r1   r2   r'   rR   Úintermediate_sizer   rÇ   Ú	gate_projÚup_projrè   r   ré   rê   )r>   r'   r$  rC   s      €rD   r2   zMellumMLP.__init__r  s®   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ=NÐ=V ×!9Ò!9Ð\mˆÔÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Ñ/ˆ�rE   c                 óˆ   — | j                  | j                  | j                  |«      «      | j                  |«      z  «      }|S ru   )rè   rê   r%  r&  )r>   rm   rè   s      rD   rt   zMellumMLP.forward|  s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÑ#SÓTˆ	ØÐrE   ru   r  r   s   @rD   r!  r!  q  s   ø„ õ0örE   r!  c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
eej                  ej                  f   dz  dee   dej                  fd„Zˆ xZS )ÚMellumDecoderLayerr'   r¿   c                 ó|  •— t         ‰| �  «        |j                  | _        t        ||«      | _        |j
                  |   dk(  rt        |«      | _        nt        ||j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        y )NÚsparse)r$  rÃ   )r1   r2   rR   r¾   Ú	self_attnÚmlp_layer_typesr  Úmlpr!  r$  r‚   rÍ   Úinput_layernormÚpost_attention_layernormrÑ   s      €rD   r2   zMellumDecoderLayer.__init__‚  s“   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ(¨°Ó;ˆŒØ×!Ñ! )Ñ,°Ò8Ü+¨FÓ3ˆD�Hä  ¸6×;SÑ;SÔTˆDŒHÜ,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔÜ(5°f×6HÑ6HÈf×NaÑNaÔ(bˆÕ%rE   Nr‰   r¬   rn   rÓ   Ú	use_cacherÒ   r¯   rH   c           
      óÀ   — |}| j                  |«      } | j                  d||||||dœ|¤Ž\  }}	||z   }|}| j                  |«      }| j                  |«      }||z   }|S )N)r‰   r¬   rn   rÓ   r1  rÒ   © )r/  r,  r0  r.  )
r>   r‰   r¬   rn   rÓ   r1  rÒ   r¯   Úresidualr  s
             rD   rt   zMellumDecoderLayer.forward�  s�   € ð !ˆØ×,Ñ,¨]Ó;ˆà)˜4Ÿ>™>ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
Ñˆ�qð ! =Ñ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ  =Ñ0ˆØÐrE   )NNNFN)rv   rw   rx   r#   rT   r2   rU   ry   Ú
LongTensorr	   Úboolr|   r   r   rt   r~   r   s   @rD   r)  r)  �  s¾   ø„ ð	c˜|ð 	c¸õ 	cð /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð+Ñ,ðð 
�‰÷rE   r)  c                   ó–   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZ eed¬«      eedœZ ej(                  «       ˆ fd	„«       Zˆ xZS )
ÚMellumPreTrainedModelr'   ÚmodelTr)  rÓ   r   )Úindex)r  r‰   Ú
attentionsc                 óÄ  •— t         ‰| �  |«       | j                  j                  }t	        |t
        «      rEt        j                  |j                  d|¬«       t        j                  |j                  d|¬«       n2t	        |t        «      r"t        j                  |j                  d|¬«       t	        |t        «      r£|j                  D ]“  }|j                  }|j                  |   dk7  rt         |j                  |      } ||j                  |¬«      \  }}t        j"                  t%        ||› d�«      |«       t        j"                  t%        ||› d�«      |«       Œ• y y )NrÕ   )rŽ   Ústdr*   r+   r-   r/   )r1   Ú_init_weightsr'   Úinitializer_rangerf   rà   ÚinitÚnormal_rç   rè   r  r‡   r%   r8   r:   r)   r   Úcopy_rQ   )r>   r¨   r=  r,   r@   rA   r  rC   s          €rD   r>  z#MellumPreTrainedModel._init_weightsÀ  s  ø€ ä‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fœmÔ,Ü�L‰L˜×,Ñ,°3¸CÕ@Ü�L‰L˜×)Ñ)°¸Ö=Ü˜Ô 0Ô1Ü�L‰L˜Ÿ™¨S°cÕ:Ü�fÔ3Ô4Ø$×0Ñ0ò ^�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ò<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ô#UÑ �˜qÜ—
‘
œ7 6¨j¨\¸Ð+CÓDÀmÔTÜ—
‘
œ7 6¨j¨\Ð9KÐ+LÓMÈ}Õ]ñ^ð 5rE   )rv   rw   rx   r#   rz   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr    r  r)  r¾   Ú_can_record_outputsrU   r}   r>  r~   r   s   @rD   r8  r8  ­  s{   ø… àÓØÐØ&*Ð#Ø-Ð.ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðá'Ð(8ÀÔBØ+Ø%ñÐð €U‡]�]ƒ_ó^ó ô^rE   r8  c                   óò   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   defd„«       «       «       Zˆ xZS )ÚMellumModelr'   c           	      ó  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                  ¬«      | _        t#        |¬«      | _        d| _        | j)                  «        y c c}w )NrÃ   ©r'   F)r1   r2   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	EmbeddingrR   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersr)  Úlayersr‚   rÍ   Únormr%   Ú
rotary_embÚgradient_checkpointingÚ	post_initrÑ   s      €rD   r2   zMellumModel.__init__Õ  sÆ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m‘mÜDIÈ&×JbÑJbÓDcÖd°yÔ ¨	Õ2Òdó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÔNˆŒ	Ü/°vÔ>ˆŒØ&+ˆÔ#ð 	�‰Õùò es   ÂDNÚ	input_idsr¬   rn   rÓ   Úinputs_embedsr1  r¯   rH   c           	      ó°  ‡— |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      sP| j                  ||||dœŠˆfd„ˆfd„d	œ}
i }	t        | j                  j                  «      D ]  } |
|   «       |	|<   Œ |}i }t        | j                  j                  «      D ]  }| j                  |||«      ||<   Œ t        | j                   d | j                  j"                   «      D ]G  \  }} ||f|	| j                  j                  |      || j                  j                  |      ||d
œ|¤Ž}ŒI | j%                  |«      }t'        ||r|¬«      S d ¬«      S )Nz:You must specify exactly one of input_ids or inputs_embedsrP  r   r"   )rF   )r'   r_  r¬   rÓ   rn   c                  ó   •— t        di ‰ ¤ŽS ©Nr3  )r   ©Úmask_kwargss   €rD   ú<lambda>z%MellumModel.forward.<locals>.<lambda>	  s   ø€ Ô*<Ñ*K¸{Ñ*K€ rE   c                  ó   •— t        di ‰ ¤ŽS rb  )r   rc  s   €rD   re  z%MellumModel.forward.<locals>.<lambda>
  s   ø€ Ô-NÑ-]ÐQ\Ñ-]€ rE   )Úfull_attentionrÄ   )r¬   rÒ   rn   rÓ   )Úlast_hidden_staterÓ   )Ú
ValueErrorrU  r
   r'   Úget_seq_lengthrU   rV   re   rF   rš   rf   Údictr7   r8   r[  Ú	enumeraterY  rX  rZ  r   )r>   r^  r¬   rn   rÓ   r_  r1  r¯   Úpast_seen_tokensÚcausal_mask_mappingÚmask_creation_functionsr,   r‰   rÒ   ÚiÚdecoder_layerrd  s                   @rD   rt   zMellumModel.forwardå  s  ø€ ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-ÄÔFàŸ+™+Ø!.Ø"0Ø#2Ø ,ñˆKó #LÛ%]ñ'Ð#ð #%ÐÜ! $§+¡+×"9Ñ"9Ó:ò X�
Ø2UÐ2IÈ*Ñ2UÓ2WÐ# JÒ/ðXð &ˆØ ÐÜ˜dŸk™k×5Ñ5Ó6ò 	gˆJØ.2¯o©o¸mÈ\Ð[eÓ.fÐ 
Ò+ð	gô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÓ Wò 	ÑˆAˆ}Ù)Øðà2°4·;±;×3JÑ3JÈ1Ñ3MÑNØ$7¸¿¹×8OÑ8OÐPQÑ8RÑ$SØ)Ø /ñð ñ‰Mð	ð Ÿ	™	 -Ó0ˆä%Ø+Ù/8˜Oô
ð 	
à>Bô
ð 	
rE   )NNNNNN)rv   rw   rx   r#   r2   r   r!   r   rU   r5  ry   r	   ÚFloatTensorr6  r   r   r   rt   r~   r   s   @rD   rN  rN  Ó  sÍ   ø„ ð˜|õ ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ<
à×#Ñ# dÑ*ð<
ð Ÿ™ tÑ+ð<
ð ×&Ñ&¨Ñ-ð	<
ð
  ™ð<
ð ×(Ñ(¨4Ñ/ð<
ð ˜$‘;ð<
ð Ð+Ñ,ð<
ð 
 ò<
ó ó ó  ô<
rE   rN  Úgate_logitsrâ   c                 ó¾  — | �t        | t        «      syt        | t        «      rC| d   j                  }t        j                  | D �cg c]  }|j                  |«      ‘Œ c}d¬«      }t        j                  j                  j                  d¬«      }t        j                  ||d¬«      \  }}	t        j                  j                  j                  |	|«      }
|€>t        j                  |
j                  «       d¬«      }t        j                  |d¬«      }�n|j                  \  }}|j                  d   ||z  z  }|ddd…dd…ddf   j                  |||||f«      j                  d||«      j                  «      }t        j                   |
j                  «       |z  d¬«      t        j                   |d¬«      z  }|ddd…dd…df   j                  ||||f«      j                  d|«      j                  |«      }t        j                   ||z  d¬«      t        j                   |d¬«      z  }t        j                   ||j#                  d«      z  «      }||z  S c c}w )aÄ  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   rc   r^   )rf   r|   rF   rU   rj   rX   r   rµ   r¶   r  rò   rŽ   rY   re   rd   r£   rõ   rš   )rs  râ   r	  r¬   Úcompute_deviceÚ
layer_gateÚconcatenated_gate_logitsr  r  r  rü   Útokens_per_expertÚrouter_prob_per_expertr  r  rX  Úexpert_attention_maskÚ router_per_expert_attention_maskÚoverall_losss                      rD   Úload_balancing_loss_funcr}  '  s9  € ð: Ð¤*¨[¼%Ô"@Øä�+œuÔ%Ø$ Q™×.Ñ.ˆÜ#(§9¡9Ð^iÖ-jÐPZ¨j¯m©m¸NÕ.KÒ-jÐpqÔ#rÐ ä—h‘h×)Ñ)×1Ñ1Ð2JÐPRÐ1ÓS€OäŸ*™* _°eÀÔDÑ€AÐä—(‘(×%Ñ%×-Ñ-Ð.>ÀÓL€KàÐä!ŸJ™J {×'8Ñ'8Ó':ÀÔBÐô "'§¡¨OÀÔ!CÒà&4×&:Ñ&:Ñ#ˆ
�OØ4×:Ñ:¸1Ñ=À*ÈÑB^Ñ_Ðð ˜4¢¢A t¨TÐ1Ñ2ß‰VÐ&¨
°OÀUÈKÐXÓYß‰W�R˜ Ó,ß‰R�Óð	 	ô "ŸI™I k×&7Ñ&7Ó&9Ð<QÑ&QÐWXÔYÔ\a×\eÑ\eØ! qô]
ñ 
Ðð ˜4¢¢A tÐ+Ñ,ß‰VÐ&¨
°OÀ[ÐQÓRß‰W�R˜Ó%ß‰R�Óð	 	)ô "'§¡¨?Ð=]Ñ+]ÐcdÔ!eÔhm×hqÑhqØ,°!ôi
ñ "
Ðô —9‘9Ð.Ð1G×1QÑ1QÐRSÓ1TÑTÓU€LØ˜+Ñ%Ð%ùò[ .ks   ÁIc                   óN  ‡ — e Zd ZddiZddiZddgdgfiZˆ fd„Zee	 	 	 	 	 	 	 	 	 dd	e	j                  dz  d
e	j                  dz  de	j                  dz  dedz  de	j                  dz  de	j                  dz  dedz  dedz  dee	j                  z  dee   defd„«       «       Zˆ xZS )ÚMellumForCausalLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr‰   Úlogitsc                 óN  •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  |j                  d¬«      | _        |j                  | _	        |j                  | _
        |j                  | _        | j                  «        y r#  )r1   r2   rN  r9  rS  r   rÇ   rR   r€  Úrouter_aux_loss_coefrâ   r  r]  rë   s     €rD   r2   zMellumForCausalLM.__init__  s   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ$*×$?Ñ$?ˆÔ!Ø!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ ð 	�‰ÕrE   Nr^  r¬   rn   rÓ   r_  Úlabelsr1  Úoutput_router_logitsÚlogits_to_keepr¯   rH   c
                 ój  — |�|n| j                   j                  } | j                  d|||||||dœ|
¤Ž}|j                  }t	        |	t
        «      rt        |	 d«      n|	}| j                  |dd…|dd…f   «      }d}|� | j                  ||| j                  fi |
¤Ž}d}|rYt        |j                  | j                  | j                  |«      }|�+|| j                  |j                  |j                   «      z  z  }t#        ||||j$                  |j&                  |j(                  |j                  ¬«      S )ar  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, MellumForCausalLM

        >>> model = MellumForCausalLM.from_pretrained("Qwen/Qwen3-MoE-15B-A2B")
        >>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-MoE-15B-A2B")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r^  r¬   rn   rÓ   r_  r1  r†  )ÚlossÚaux_lossr‚  rÓ   r‰   r;  r  r3  )r'   r†  r9  rh  rf   rT   Úslicer€  Úloss_functionrS  r}  r  râ   r  r„  rX   rF   r   rÓ   r‰   r;  )r>   r^  r¬   rn   rÓ   r_  r…  r1  r†  r‡  r¯   Úoutputsr‰   Úslice_indicesr‚  r‰  rŠ  s                    rD   rt   zMellumForCausalLM.forward‹  sU  € ðN %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð
 +5¨$¯*©*ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ñ	+
ð ñ	+
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ% f¨f°d·o±oÑPÈÑPˆDàˆÙÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ð!Ø˜×1Ñ1°H·K±KÀÇÁÓ4LÑLÑL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
rE   )	NNNNNNNNr   )rv   rw   rx   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr2   r   r   rU   r5  ry   r	   rr  r6  rT   r   r   r   rt   r~   r   s   @rD   r  r  y  s5  ø„ à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€Hô
ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ñP
à×#Ñ# dÑ*ðP
ð Ÿ™ tÑ+ðP
ð ×&Ñ&¨Ñ-ð	P
ð
  ™ðP
ð ×(Ñ(¨4Ñ/ðP
ð × Ñ  4Ñ'ðP
ð ˜$‘;ðP
ð # T™kðP
ð ˜eŸl™lÑ*ðP
ð Ð+Ñ,ðP
ð 
#òP
ó ó ôP
rE   r  )r  rN  r8  )r"   )rÕ   )NrM   N)LÚcollections.abcr   Útypingr   rU   Útorch.nn.functionalr   rµ   r  Ú r   r@  Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úintegrationsr   r   r   r   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.genericr   r   r   Úutils.output_capturingr    r!   Úconfiguration_mellumr#   ÚModuler%   r‚   r—   r    ry   rT   r§   rY   r¼   r¾   rà   r  r  r!  r)  r8  rN  r|   r}  r  Ú__all__r3  rE   rD   ú<module>r§     s£  ðõ* %Ý ã ß Ð Ý å &Ý !ß .Ý )÷ó ÷ SÝ BÝ 9ß Qß Kß FÝ &ß 5ß [Ñ [ß EÝ .ôM<˜BŸI™Iô M<ñ` ˜YÓ'ôJ�B—I‘Ió Jó (ðJò((ñ Ð*Ó+òó ,ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(ó%ñ2 Ð)Ó*ôD)�b—i‘ió D)ó +ðD)ðN ô$#�B—I‘Ió $#ó ð$#ôN<�r—y‘yô <ô*T˜2Ÿ9™9ô Tô�—	‘	ô ô )Ð3ô )ðX ô"^˜Oó "^ó ð"^ðJ ôP
Ð'ó P
ó ðP
ðj #Ø
Ø*.ñ	O&Ø—‘  e§l¡lÑ 3Ñ3°dÑ:ðO&à�t‘ðO&ð —L‘L 4Ñ'ð	O&ð
 ‡\�\�CÑóO&ðd ôc
Ð-¨ó c
ó ðc
òL H�rE   