Ë
    ÜÓ^jÙ  ã                   ór   — d dl mZ ddlmZ ddlmZ ddlmZ  ed¬«      e G d„ d	e«      «       «       Zd	gZ	y
)é    )Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringz JetBrains/Mellum2-12B-A2.5B-Base)Ú
checkpointc                   ód  ‡ — e Zd ZU dZdZdgZddiZddddddd	dd
ddddœZdddd
dœZdgdgfddgdgfdgdgfdœZ	dZ
eed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   d Zeed!<   d"Zeed#<   d$Zeed%<   d&Zeed'<   d(Zeed)<   d*Zeed+<   d,Zeez  d,z  ed-<   d*Zeed.<   d/Zed,z  ed0<   d1Z eez  ed2<   d3Z!eed4<   d5Z"eed6<   d7Z#eed<   d(Z$eed8<   d*Z%eed9<   d:Z&eed;<   d,Z'ed,z  ed<<   d,Z(ed,z  ed=<   d,Z)ee*e   z  d,z  ed><   d?Z+eed@<   d,Z,e*e   d,z  edA<   d,Z-e*e   d,z  edB<   ˆ fdC„Z.dD„ Z/ˆ xZ0S )EÚMellumConfigu}  
    mlp_layer_types (`list[str]`, *optional*):
        Per-layer MLP type â€” `"dense"` or `"sparse"`. Length must equal
        `num_hidden_layers`. Defaults to all sparse.

    ```python
    >>> from transformers import MellumModel, MellumConfig

    >>> configuration = MellumConfig()
    >>> model = MellumModel(configuration)
    >>> configuration = model.config
    ```
    ÚmellumÚpast_key_valuesÚnum_expertsÚnum_local_expertsÚcolwiseÚreplicated_with_grad_allreduceÚrowwiseÚpacked_colwiseÚmoe_tp_experts)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.q_normzlayers.*.self_attn.k_normzlayers.*.self_attn.o_projú!layers.*.mlp.experts.gate_up_projúlayers.*.mlp.experts.down_projúlayers.*.mlp.expertszlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	ep_routerÚgrouped_gemm)zlayers.*.mlp.gater   r   r   Ú	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormi € Ú
vocab_sizei 	  Úhidden_sizei   Úintermediate_sizeé   Únum_hidden_layersé    Únum_attention_headsé   Únum_key_value_headsÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeg�íµ ÷Æ°>Úrms_norm_epsTÚ	use_cacheFÚtie_word_embeddingsNÚrope_parametersÚattention_biasi   Úsliding_windowg        Úattention_dropouti€  Úmoe_intermediate_sizeé   Únum_experts_per_toké@   Únorm_topk_probÚoutput_router_logitsgü©ñÒMbP?Úrouter_aux_loss_coefÚpad_token_idÚbos_token_idÚeos_token_idé€   Úhead_dimÚlayer_typesÚmlp_layer_typesc                 óì   •— | j                   €dg| j                  z  | _         | j                  €dg| j                  z  | _        | j                  €dddœdddœdœ| _        t	        ‰| �  d
i |¤dd	dhi¤Ž y )NÚfull_attentionÚsparseÚdefaultg    €„A)Ú	rope_typeÚ
rope_thetag     ˆÃ@)rC   Úsliding_attentionÚignore_keys_at_rope_validationrH   © )r@   r$   rA   r0   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úz/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/mellum/configuration_mellum.pyrL   zMellumConfig.__post_init__o   s‘   ø€ Ø×ÑÐ#Ø 0Ð1°D×4JÑ4JÑJˆDÔØ×ÑÐ'Ø$, :°×0FÑ0FÑ#FˆDÔ à×ÑÐ'à09ÈÑ"RØ3<ÈGÑ%Tñ$ˆDÔ ô
 	‰Ññ 	
Øñ	
à,?ÐAQÐ+Rô	
ó    c                 ó   — |S )NrJ   )rM   rN   s     rP   Úconvert_rope_params_to_dictz(MellumConfig.convert_rope_params_to_dict€   s   € àˆrQ   )1Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚattribute_mapÚbase_model_tp_planÚbase_model_ep_planÚbase_model_pp_planr    ÚintÚ__annotations__r!   r"   r$   r&   r(   r*   Ústrr+   r,   Úfloatr-   r.   Úboolr/   r0   Údictr   r1   r2   r3   r4   r6   r   r8   r9   r:   r;   r<   r=   Úlistr?   r@   rA   rL   rS   Ú__classcell__)rO   s   @rP   r
   r
      s
  ø… ñð €JØ#4Ð"5Ðð 	Ð*ð€Mð &/Ø%.Ø%.Ø%EØ%EØ%.Ø-=Ø*3Ø 0Ø"+Ø )Ø"+ñÐð$ )Ø-;Ø*8Ø 0ñ	Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð €J�ÓØ€K�ÓØ!Ð�sÓ!ØÐ�sÓØ!Ð˜Ó!Ø Ð˜Ó Ø€J�ÓØ#)Ð˜SÓ)Ø#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ %Ð˜Ó%Ø48€O�T˜NÑ*¨TÑ1Ó8Ø €N�DÓ Ø!%€N�C˜$‘JÓ%Ø%(Ð�u˜s‘{Ó(Ø!$Ð˜3Ó$Ø Ð˜Ó Ø€K�ÓØ€N�DÓØ!&Ð˜$Ó&Ø"'Ð˜%Ó'Ø#€L�#˜‘*Ó#Ø#€L�#˜‘*Ó#Ø+/€L�#˜˜S™	‘/ DÑ(Ó/Ø€HˆcÓØ$(€K��c‘˜TÑ!Ó(Ø(,€O�T˜#‘Y Ñ%Ó,ô
ö"rQ   r
   N)
Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r
   Ú__all__rJ   rQ   rP   ú<module>rk      sK   ðõ* /å 3Ý 1Ý #ñ Ð=Ô>ØôdÐ#ó dó ó ?ðdðN Ð
�rQ   