Ë
    ÛÓ^jè  ã                   ó@  — d dl mZ d dlZd dlmZ ddlmZ ddlmZ ddlm	Z	 ddl
mZ d	d
lmZmZmZmZ ddlmZ  ej&                  e«      Z G d„ ded¬«      Z G d„ dej.                  «      Z G d„ de«      Z G d„ de«      Z G d„ de«      Z G d„ de«      Zg d¢Zy)é    )Ú	TypedDictN)Únné   )ÚACT2FN)ÚCache)ÚUnpack)Úloggingé   )ÚGraniteMoeDecoderLayerÚGraniteMoeForCausalLMÚGraniteMoeModelÚGraniteMoePreTrainedModelé   )ÚGraniteMoeSharedConfigc                   ó€   — e Zd ZU dZej
                  ed<   ej
                  ed<   eed<   eed<   ej                  ed<   y)ÚGraniteFlashAttentionKwargsaT  
    Keyword arguments for advanced Flash Attention, causal-conv1d, and mamba_ssm kernel usage.
    Use cases include padding-free training and fewer `torch.compile` graph breaks.

    cu_seq_lens_q (`torch.LongTensor`):
        Gets cumulative sequence length for query state.
    cu_seq_lens_k (`torch.LongTensor`):
        Gets cumulative sequence length for key state.
    max_length_q (`int`):
        Maximum sequence length for query state.
    max_length_k (`int`):
        Maximum sequence length for key state.
    seq_idx (`torch.IntTensor):
        Index of each packed sequence.
    Úcu_seq_lens_qÚcu_seq_lens_kÚmax_length_qÚmax_length_kÚseq_idxN)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚtorchÚ
LongTensorÚ__annotations__ÚintÚ	IntTensor© ó    úˆ/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/granitemoeshared/modular_granitemoeshared.pyr   r   $   s7   … ñð  ×#Ñ#Ó#Ø×#Ñ#Ó#ØÓØÓØ�_‰_Ôr"   r   F)Útotalc                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚGraniteMoeSharedMLPz~
    MLP layer for shared experts

    Args:
        config:
            Configuration object with model hyperparameters.
    Úconfigc                 ó`  •— t         ‰| �  «        |j                  | _        |j                  | _        t
        |j                     | _        t        j                  | j                  | j                  dz  d¬«      | _
        t        j                  | j                  | j                  d¬«      | _        y )Nr
   F)Úbias)ÚsuperÚ__init__Úhidden_sizeÚ
input_sizeÚshared_intermediate_sizer   Ú
hidden_actÚ
activationr   ÚLinearÚinput_linearÚoutput_linear©Úselfr'   Ú	__class__s     €r#   r+   zGraniteMoeSharedMLP.__init__E   s   ø€ Ü‰ÑÔà ×,Ñ,ˆŒØ!×:Ñ:ˆÔÜ  ×!2Ñ!2Ñ3ˆŒÜŸI™I d§o¡o°t×7GÑ7GÈ!Ñ7KÐRWÔXˆÔÜŸY™Y t×'7Ñ'7¸¿¹ÈuÔUˆÕr"   Úhidden_statesÚreturnc                 ó¤   — | j                  |«      }|j                  dd¬«      }| j                  |d   «      |d   z  }| j                  |«      }|S )Nr
   éÿÿÿÿ)Údimr   r   )r2   Úchunkr0   r3   )r5   r7   Úchunked_hidden_statess      r#   ÚforwardzGraniteMoeSharedMLP.forwardN   s^   € Ø×)Ñ)¨-Ó8ˆØ -× 3Ñ 3°A¸2Ð 3Ó >ÐØŸ™Ð(=¸aÑ(@ÓAÐDYÐZ[ÑD\Ñ\ˆØ×*Ñ*¨=Ó9ˆØÐr"   )
r   r   r   r   r   r+   r   ÚTensorr>   Ú__classcell__©r6   s   @r#   r&   r&   <   s2   ø„ ñðVÐ5õ Vð U§\¡\ð °e·l±l÷ r"   r&   c                   óP  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
e
dz  deej                  ej                  f   dz  dee   deej                  eej                  ej                  f   dz  f   fd„Zˆ xZS )ÚGraniteMoeSharedDecoderLayerr'   Ú	layer_idxc                 ót   •— t         ‰| �  ||«       |j                  dk(  rd | _        y t        |«      | _        y )Nr   )r*   r+   r.   r&   Ú
shared_mlp©r5   r'   rD   r6   s      €r#   r+   z%GraniteMoeSharedDecoderLayer.__init__W   s3   ø€ Ü‰Ñ˜ Ô+Ø"(×"AÑ"AÀQÒ"F˜$ˆ�ÔL_Ð`fÓLgˆ�r"   Nr7   Úattention_maskÚposition_idsÚpast_key_valuesÚoutput_attentionsÚ	use_cacheÚposition_embeddingsÚkwargsr8   c                 ó<  — |}	| j                  |«      } | j                  d|||||||dœ|¤Ž\  }}
|	|| j                  z  z   }|}	| j                  |«      }| j	                  |«      }| j
                  €|}n|| j                  |«      z   }|	|| j                  z  z   }|S )N)r7   rH   rI   rJ   rK   rL   rM   r!   )Úinput_layernormÚ	self_attnÚresidual_multiplierÚpost_attention_layernormÚblock_sparse_moerF   )r5   r7   rH   rI   rJ   rK   rL   rM   rN   ÚresidualÚ_Úmoe_hidden_statess               r#   r>   z$GraniteMoeSharedDecoderLayer.forward[   sÎ   € ð !ˆØ×,Ñ,¨]Ó;ˆð *˜4Ÿ>™>ð 	
Ø'Ø)Ø%Ø+Ø/ØØ 3ñ	
ð ñ	
Ñˆ�qð ! =°4×3KÑ3KÑ#KÑKˆà ˆØ×5Ñ5°mÓDˆØ ×1Ñ1°-Ó@Ðà�?‰?Ð"Ø-‰Mà-°·±ÀÓ0NÑNˆMØ  =°4×3KÑ3KÑ#KÑKˆØÐr"   )NNNFFN)r   r   r   r   r   r+   r   r?   r   r   ÚboolÚtupler   r   ÚFloatTensorr>   r@   rA   s   @r#   rC   rC   V   sø   ø„ ðhÐ5ð hÀ#õ hð /3Ø04Ø(,Ø).Ø!&ØHLñ%à—|‘|ð%ð Ÿ™ tÑ+ð%ð ×&Ñ&¨Ñ-ð	%ð
  ™ð%ð   $™;ð%ð ˜$‘;ð%ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð%ð Ð4Ñ5ð%ð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷%r"   rC   c                   ó   — e Zd ZU eed<   dgZy)ÚGraniteMoeSharedPreTrainedModelr'   rC   N)r   r   r   r   r   Ú_no_split_modulesr!   r"   r#   r\   r\   ƒ   s   … Ø"Ó"Ø7Ð8Ñr"   r\   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚGraniteMoeSharedModelr'   c           	      ó¸   •— t         ‰| �  |«       t        j                  t	        |j
                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        y c c}w ©N)r*   r+   r   Ú
ModuleListÚrangeÚnum_hidden_layersrC   ÚlayersrG   s      €r#   r+   zGraniteMoeSharedModel.__init__‰   sE   ø€ Ü‰Ñ˜Ô Ü—m‘mÜNSÐTZ×TlÑTlÓNmÖnÀÔ)¨&°)Õ<Ònó
ˆ�ùÚns   ·A)r   r   r   r   r+   r@   rA   s   @r#   r_   r_   ˆ   s   ø„ ð
Ð5÷ 
ñ 
r"   r_   c                   ó,   ‡ — e Zd ZddiZdefˆ fd„Zˆ xZS )ÚGraniteMoeSharedForCausalLMzlm_head.weightzmodel.embed_tokens.weightr'   c                 ód   •— t         ‰| �  |«       t        |«      | _        | j	                  «        y ra   )r*   r+   r_   ÚmodelÚ	post_initr4   s     €r#   r+   z$GraniteMoeSharedForCausalLM.__init__“   s&   ø€ Ü‰Ñ˜Ô Ü*¨6Ó2ˆŒ
à�‰Õr"   )r   r   r   Ú_tied_weights_keysr   r+   r@   rA   s   @r#   rg   rg   �   s!   ø„ Ø*Ð,GÐHÐðÐ5÷ ñ r"   rg   )rg   r_   r\   )Útypingr   r   r   Úactivationsr   Úcache_utilsr   Úprocessing_utilsr   Úutilsr	   Úgranitemoe.modeling_granitemoer   r   r   r   Úconfiguration_granitemoesharedr   Ú
get_loggerr   Úloggerr   ÚModuler&   rC   r\   r_   rg   Ú__all__r!   r"   r#   ú<module>rw      sš   ðõ ã Ý å !Ý  Ý &Ý ÷ó õ Cð 
ˆ×	Ñ	˜HÓ	%€ô )°5õ ô0˜"Ÿ)™)ô ô4*Ð#9ô *ôZ9Ð&?ô 9ô

˜Oô 
ôÐ"7ô ò f�r"   