
    ^j                        d dl Z d dlmZ ddlmZ ddlmZ ddlm	Z	m
Z
 ddlmZmZ dd	lmZmZmZ dd
lmZ ddlmZmZmZmZ  ej2                  e      Z ed      e G d de                    Z G d de      Z G d de      Z G d de      Z G d de      Z  G d de      Z! G d de      Z" G d de      Z#g dZ$y)    N)strict   )initialization)PreTrainedConfig)ROPE_INIT_FUNCTIONSRopeParameters)auto_docstringlogging   )LagunaDecoderLayerLagunaModelLagunaRotaryEmbedding)Qwen3MoeConfig)Qwen3MoeAttentionQwen3MoeForCausalLMQwen3MoePreTrainedModelQwen3MoeSparseMoeBlockz JetBrains/Mellum2-12B-A2.5B-Base)
checkpointc                   ,   e Zd ZU dZdZdZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZedz  ed<   dZeed<   dZeed<   dZeed<   dZee   dz  ed<   dZee   dz  ed<   dZeez  dz  ed<    e       Z e       Z e       Zd Zd Zy)MellumConfigu}  
    mlp_layer_types (`list[str]`, *optional*):
        Per-layer MLP type — `"dense"` or `"sparse"`. Length must equal
        `num_hidden_layers`. Defaults to all sparse.

    ```python
    >>> from transformers import MellumModel, MellumConfig

    >>> configuration = MellumConfig()
    >>> model = MellumModel(configuration)
    >>> configuration = model.config
    ```
    mellumi  
vocab_sizei 	  hidden_sizei   intermediate_size   num_hidden_layers   head_dimi   max_position_embeddingsi   Nsliding_window@   num_expertsi  moe_intermediate_sizeTnorm_topk_problayer_typesmlp_layer_typesrope_parametersc                     | j                   dg| j                  z  | _         | j                  dg| j                  z  | _        | j                  ddddddd| _        t	        j
                  | fi |dd	dhi y )
Nfull_attentionsparsedefaultg    A)	rope_type
rope_thetag     @)r)   sliding_attentionignore_keys_at_rope_validationr.   )r%   r   r&   r'   r   __post_init__selfkwargss     t/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/mellum/modular_mellum.pyr0   zMellumConfig.__post_init__K   s    # 01D4J4JJD'$,:0F0F#FD '09"R3<G%T$D 
 	&&	
	
 -@AQ+R	
    c                     |S N r1   s     r4   convert_rope_params_to_dictz(MellumConfig.convert_rope_params_to_dict]   s    r5   )__name__
__module____qualname____doc__
model_typer   int__annotations__r   r   r   r   r   r    r"   r#   r$   boolr%   liststrr&   r'   dictr   AttributeErroruse_sliding_windowdecoder_sparse_stepmlp_only_layersr0   r9   r8   r5   r4   r   r   &   s     JJK!s!sHc#)S)!%NC$J%K!$3$ND$(KcT!((,OT#Y%,48OTN*T18')(*$&O
$r5   r   c                       e Zd Zy)MellumRotaryEmbeddingNr:   r;   r<   r8   r5   r4   rJ   rJ   b       r5   rJ   c                   (     e Zd Zdedef fdZ xZS )MellumAttentionconfig	layer_idxc                 |    t         |   ||       |j                  |   dk(  r|j                  | _        y d | _        y )Nr.   )super__init__r%   r    r2   rO   rP   	__class__s      r4   rS   zMellumAttention.__init__g   s;    +7=7I7I)7TXk7kf33qur5   r:   r;   r<   r   r?   rS   __classcell__rU   s   @r4   rN   rN   f   s    v| v v vr5   rN   c                       e Zd Zy)MellumSparseMoeBlockNrK   r8   r5   r4   rZ   rZ   l   rL   r5   rZ   c                   (     e Zd Zdedef fdZ xZS )MellumDecoderLayerrO   rP   c                 D    t         |           t        ||      | _        y r7   )rR   rS   rN   	self_attnrT   s      r4   rS   zMellumDecoderLayer.__init__q   s    (;r5   rV   rX   s   @r4   r\   r\   p   s    <| < < <r5   r\   c                   F     e Zd Z ej                          fd       Z xZS )MellumPreTrainedModelc                    t         |   |       t        |t              r|j                  D ]  }|j
                  }|j                  |   dk7  rt        |j                  |      } ||j                  |      \  }}t        j                  t        || d      |       t        j                  t        || d      |        y y )Nr+   )
layer_type	_inv_freq_original_inv_freq)rR   _init_weights
isinstancerJ   r%   compute_default_rope_parametersr,   r   rO   initcopy_getattr)r2   modulerb   rope_init_fncurr_inv_freq_rU   s         r4   re   z#MellumPreTrainedModel._init_weightsw   s    f%f34$00 ^
%EE##J/9<#6v7G7G
7S#TL#/*#U q

76j\+CDmT

76j\9K+LM}]^ 5r5   )r:   r;   r<   torchno_gradre   rW   rX   s   @r4   r`   r`   v   s    U]]_	^ 	^r5   r`   c                       e Zd Zy)MellumModelNrK   r8   r5   r4   rr   rr      rL   r5   rr   c                       e Zd Zy)MellumForCausalLMNrK   r8   r5   r4   rt   rt      rL   r5   rt   )r   rt   rr   r`   )%ro   huggingface_hub.dataclassesr    r   rh   configuration_utilsr   modeling_rope_utilsr   r   utilsr	   r
   laguna.modeling_lagunar   r   r   !qwen3_moe.configuration_qwen3_moer   qwen3_moe.modeling_qwen3_moer   r   r   r   
get_loggerr:   loggerr   rJ   rN   rZ   r\   r`   rr   rt   __all__r8   r5   r4   <module>r      s     . & 3 - [ [ >  
		H	% =>7> 7  ?7t	1 	v' v	1 	<+ <^3 ^	+ 		+ 	r5   