Ë
    ÜÓ^j¯  ã                   ó†  — d dl Z d dlmZ ddlmZ ddlmZ ddlm	Z	m
Z
 ddlmZmZ dd	lmZmZmZ dd
lmZ ddlmZmZmZmZ  ej2                  e«      Z ed¬«      e G d„ de«      «       «       Z G d„ de«      Z G d„ de«      Z G d„ de«      Z G d„ de«      Z  G d„ de«      Z! G d„ de«      Z" G d„ de«      Z#g d¢Z$y)é    N)Ústricté   )Úinitialization)ÚPreTrainedConfig)ÚROPE_INIT_FUNCTIONSÚRopeParameters)Úauto_docstringÚloggingé   )ÚLagunaDecoderLayerÚLagunaModelÚLagunaRotaryEmbedding)ÚQwen3MoeConfig)ÚQwen3MoeAttentionÚQwen3MoeForCausalLMÚQwen3MoePreTrainedModelÚQwen3MoeSparseMoeBlockz JetBrains/Mellum2-12B-A2.5B-Base)Ú
checkpointc                   ó,  — e Zd ZU dZdZdZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZedz  ed<   dZeed<   dZeed<   dZeed<   dZee   dz  ed<   dZee   dz  ed<   dZeez  dz  ed<    e«       Z e«       Z e«       Zd„ Zd„ Zy)ÚMellumConfigu}  
    mlp_layer_types (`list[str]`, *optional*):
        Per-layer MLP type â€” `"dense"` or `"sparse"`. Length must equal
        `num_hidden_layers`. Defaults to all sparse.

    ```python
    >>> from transformers import MellumModel, MellumConfig

    >>> configuration = MellumConfig()
    >>> model = MellumModel(configuration)
    >>> configuration = model.config
    ```
    Úmellumi € Ú
vocab_sizei 	  Úhidden_sizei   Úintermediate_sizeé   Únum_hidden_layersé€   Úhead_dimi   Úmax_position_embeddingsi   NÚsliding_windowé@   Únum_expertsi€  Úmoe_intermediate_sizeTÚnorm_topk_probÚlayer_typesÚmlp_layer_typesÚrope_parametersc                 óø   — | j                   €dg| j                  z  | _         | j                  €dg| j                  z  | _        | j                  €dddœdddœdœ| _        t	        j
                  | fi |¤dd	dhi¤Ž y )
NÚfull_attentionÚsparseÚdefaultg    €„A)Ú	rope_typeÚ
rope_thetag     ˆÃ@)r)   Úsliding_attentionÚignore_keys_at_rope_validationr.   )r%   r   r&   r'   r   Ú__post_init__©ÚselfÚkwargss     út/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/mellum/modular_mellum.pyr0   zMellumConfig.__post_init__K   s—   € Ø×ÑÐ#Ø 0Ð1°D×4JÑ4JÑJˆDÔØ×ÑÐ'Ø$, :°×0FÑ0FÑ#FˆDÔ à×ÑÐ'à09ÈÑ"RØ3<ÈGÑ%Tñ$ˆDÔ ô
 	×&Ñ&Øñ	
àñ	
ð -@ÐAQÐ+Rô	
ó    c                 ó   — |S ©N© r1   s     r4   Úconvert_rope_params_to_dictz(MellumConfig.convert_rope_params_to_dict]   s   € àˆr5   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typer   ÚintÚ__annotations__r   r   r   r   r   r    r"   r#   r$   Úboolr%   ÚlistÚstrr&   r'   Údictr   ÚAttributeErrorÚuse_sliding_windowÚdecoder_sparse_stepÚmlp_only_layersr0   r9   r8   r5   r4   r   r   &   sØ   … ñð €Jà€J�ÓØ€K�ÓØ!Ð�sÓ!ØÐ�sÓØ€HˆcÓØ#)Ð˜SÓ)Ø!%€N�C˜$‘JÓ%Ø€K�ÓØ!$Ð˜3Ó$Ø€N�DÓØ$(€K��c‘˜TÑ!Ó(Ø(,€O�T˜#‘Y Ñ%Ó,Ø48€O�T˜NÑ*¨TÑ1Ó8á'Ó)ÐÙ(Ó*ÐÙ$Ó&€Oò
ó$r5   r   c                   ó   — e Zd Zy)ÚMellumRotaryEmbeddingN©r:   r;   r<   r8   r5   r4   rJ   rJ   b   ó   „ Ør5   rJ   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚMellumAttentionÚconfigÚ	layer_idxc                 ó|   •— t         ‰| �  ||«       |j                  |   dk(  r|j                  | _        y d | _        y )Nr.   )ÚsuperÚ__init__r%   r    ©r2   rO   rP   Ú	__class__s      €r4   rS   zMellumAttention.__init__g   s;   ø€ Ü‰Ñ˜ Ô+Ø7=×7IÑ7IÈ)Ñ7TÐXkÒ7k˜f×3Ñ3ˆÕÐquˆÕr5   ©r:   r;   r<   r   r?   rS   Ú__classcell__©rU   s   @r4   rN   rN   f   s   ø„ ðv˜|ð v¸÷ vñ vr5   rN   c                   ó   — e Zd Zy)ÚMellumSparseMoeBlockNrK   r8   r5   r4   rZ   rZ   l   rL   r5   rZ   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚMellumDecoderLayerrO   rP   c                 óD   •— t         ‰| �  «        t        ||«      | _        y r7   )rR   rS   rN   Ú	self_attnrT   s      €r4   rS   zMellumDecoderLayer.__init__q   s   ø€ Ü‰ÑÔÜ(¨°Ó;ˆ�r5   rV   rX   s   @r4   r\   r\   p   s   ø„ ð<˜|ð <¸÷ <ñ <r5   r\   c                   óF   ‡ — e Zd Z ej                  «       ˆ fd„«       Zˆ xZS )ÚMellumPreTrainedModelc                 óŠ  •— t         ‰| �  |«       t        |t        «      r£|j                  D ]“  }|j
                  }|j                  |   dk7  rt        |j                  |      } ||j                  |¬«      \  }}t        j                  t        ||› d�«      |«       t        j                  t        ||› d�«      |«       Œ• y y )Nr+   )Ú
layer_typeÚ	_inv_freqÚ_original_inv_freq)rR   Ú_init_weightsÚ
isinstancerJ   r%   Úcompute_default_rope_parametersr,   r   rO   ÚinitÚcopy_Úgetattr)r2   Úmodulerb   Úrope_init_fnÚcurr_inv_freqÚ_rU   s         €r4   re   z#MellumPreTrainedModel._init_weightsw   s¸   ø€ ä‰Ñ˜fÔ%Ü�fÔ3Ô4Ø$×0Ñ0ò ^�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ò<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ô#UÑ �˜qÜ—
‘
œ7 6¨j¨\¸Ð+CÓDÀmÔTÜ—
‘
œ7 6¨j¨\Ð9KÐ+LÓMÈ}Õ]ñ^ð 5r5   )r:   r;   r<   ÚtorchÚno_gradre   rW   rX   s   @r4   r`   r`   v   s   ø„ Ø€U‡]�]ƒ_ó	^ó ô	^r5   r`   c                   ó   — e Zd Zy)ÚMellumModelNrK   r8   r5   r4   rr   rr   „   rL   r5   rr   c                   ó   — e Zd Zy)ÚMellumForCausalLMNrK   r8   r5   r4   rt   rt   ˆ   rL   r5   rt   )r   rt   rr   r`   )%ro   Úhuggingface_hub.dataclassesr   Ú r   rh   Úconfiguration_utilsr   Úmodeling_rope_utilsr   r   Úutilsr	   r
   Úlaguna.modeling_lagunar   r   r   Ú!qwen3_moe.configuration_qwen3_moer   Úqwen3_moe.modeling_qwen3_moer   r   r   r   Ú
get_loggerr:   Úloggerr   rJ   rN   rZ   r\   r`   rr   rt   Ú__all__r8   r5   r4   ú<module>r€      sÓ   ðó Ý .å &Ý 3÷÷ -ß [Ñ [Ý >÷ó ð 
ˆ×	Ñ	˜HÓ	%€ñ Ð=Ô>Øô7�>ó 7ó ó ?ð7ôt	Ð1ô 	ôvÐ'ô vô	Ð1ô 	ô<Ð+ô <ô^Ð3ô ^ô	�+ô 	ô	Ð+ô 	ò�r5   