Ë
    ÛÓ^j…  ã                   ór   — d dl mZ ddlmZ ddlmZ ddlmZ  ed¬«      e G d„ d	e«      «       «       Zd	gZ	y
)é    )Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringzSmallDoge/Doge-320M)Ú
checkpointc                   ó:  ‡ — e Zd ZU dZdZdgZddddddddddddœZd	gd
gfddgdgfdgdgfdœZdZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZee	z  e
d<   dZee
d<   dZee
d<   dZee
d<   dZee
d<   d Zee
d!<   dZe	e
d"<   d#Zeez  d#z  e
d$<   d%Ze	e
d&<   d#Ze	d#z  e
d'<   d Zee
d(<   dZed#z  e
d)<   d Zee
d*<   d#Z e	d#z  e
d+<   dZ!e	e
d,<   d Z"ee
d-<   d.Z#e	e
d/<   d0Z$e	e
d1<   d Z%ee
d2<   d Z&ee
d3<   d4Z'ee
d5<   d#Z(e	d#z  e
d6<   d#Z)e	d#z  e
d7<   d#Z*e	e+e	   z  d#z  e
d8<   ˆ fd9„Z,ˆ xZ-S ):Ú
DogeConfigaè  
    keep_window_size (`int`, *optional*, defaults to 2048):
        The window size of tokens that are not dynamically masked, and dynamic masking is only performed when the sequence length exceeds this value.
    is_moe (`bool`, *optional*, defaults to `False`):
        Whether to use the Cross Domain Mixture of Experts, if `True`, the MoE will inherit the MLP to initialize.

    ```python
    >>> from transformers import DogeConfig, DogeModel

    >>> # Initializing a Doge-320M style configuration
    >>> configuration = DogeConfig()

    >>> # Initializing a model from the Doge-320M style configuration
    >>> model = DogeModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```ÚdogeÚpast_key_valuesÚcolwiseÚrowwiseÚcolwise_gather_outputÚrowwise_split_input)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.dt_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projzlayers.*.mlp.router_gatezlayers.*.mlp.down_embedzlayers.*.mlp.up_embedÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormi €  Ú
vocab_sizei   Úhidden_sizei   Úintermediate_sizeé    Únum_hidden_layersg        Úhidden_dropoutÚsiluÚ
hidden_actg{®Gáz”?Úinitializer_rangeg�íµ ÷Æ°>Úrms_norm_epsTÚ	use_cacheFÚtie_word_embeddingsÚmax_position_embeddingsNÚrope_parametersé   Únum_attention_headsÚnum_key_value_headsÚattention_biasÚattention_dropoutÚmlp_biasÚsliding_windowÚkeep_window_sizeÚis_moei @  Únum_expertsé@   Únum_experts_per_tokÚnorm_topk_probÚoutput_router_logitsgü©ñÒMbP?Úrouter_aux_loss_coefÚpad_token_idÚbos_token_idÚeos_token_idc                 ó^   •— | j                   €| j                  | _         t        ‰| �  di |¤Ž y )N© )r(   r'   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úv/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/doge/configuration_doge.pyr;   zDogeConfig.__post_init__f   s-   ø€ à×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$ä‰ÑÑ' Ó'ó    ).Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planr   ÚintÚ__annotations__r   r   r   r   Úfloatr   Ústrr    r!   r"   Úboolr#   r$   r%   r   Údictr'   r(   r)   r*   r+   r,   r-   r.   r/   r1   r2   r3   r4   r5   r6   r7   Úlistr;   Ú__classcell__)r>   s   @r?   r
   r
      sØ  ø… ñð& €JØ#4Ð"5Ðð &/Ø%.Ø%.Ø&/Ø%.Ø"+Ø )Ø"+Ø$;Ø#8Ø!6ñÐð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð €J�ÓØ€K�ÓØ!Ð�sÓ!ØÐ�sÓØ"%€N�E˜C‘KÓ%Ø€J�ÓØ#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ %Ð˜Ó%Ø#'Ð˜SÓ'Ø48€O�^ dÑ*¨TÑ1Ó8Ø Ð˜Ó Ø&*Ð˜˜t™Ó*Ø €N�DÓ Ø&)Ð�u˜t‘|Ó)Ø€HˆdÓØ!%€N�C˜$‘JÓ%Ø Ð�cÓ Ø€FˆDÓØ€K�ÓØ!Ð˜Ó!Ø €N�DÓ Ø!&Ð˜$Ó&Ø"'Ð˜%Ó'Ø#€L�#˜‘*Ó#Ø#€L�#˜‘*Ó#Ø+/€L�#˜˜S™	‘/ DÑ(Ó/÷(ð (r@   r
   N)
Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r
   Ú__all__r9   r@   r?   ú<module>rV      sJ   ðõ, /å 3Ý 1Ý #ñ Ð0Ô1ØôL(Ð!ó L(ó ó 2ðL(ð^ ˆ.�r@   