Ë
    ÛÓ^jZ  ã                   óx   — d Z ddlmZ ddlmZ ddlmZ ddlmZ e edd¬	«       G d
„ de«      «       «       Z	dgZ
y)zAFMoE model configurationé    )Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringzÏ
    AFMoE is an Adaptive Feedforward MoE (Mixture of Experts) model with token-choice routing, shared experts, and a
    hybrid attention mechanism combining sliding window and full attention patterns.
    zarcee-ai/Trinity-Mini)Úcustom_introÚ
checkpointc                   ó„  ‡ — e Zd ZU dZdZdgZdgdgfddgdgfdgdgfdœZd	d
d
ddœZdZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	dz  e
d<   dZe	e
d<   dZe	dz  e
d<   dZe	dz  e
d<   dZee
d <   d!Ze	e
d"<   d#Zee
d$<   d%Zee
d&<   d'Zee
d(<   d)Zee
d*<   dZeez  dz  e
d+<   d,Ze	dz  e
d-<   d.Z e	dz  e
d/<   d0Z!e	dz  e
d1<   d2Z"edz  e
d3<   d)Z#ee
d4<   d5Z$e	dz  e
d6<   d7Z%e	dz  e
d8<   dZ&e'e   dz  e
d9<   d:Z(ee	z  dz  e
d;<   d)Z)edz  e
d<<   dZ*e	e'e	   z  dz  e
d=<   dZ+e	dz  e
d><   dZ,e	dz  e
d?<   d)Z-ee
d@<   ˆ fdA„Z.ˆ xZ/S )BÚAfmoeConfiga%  
    global_attn_every_n_layers (`int`, *optional*, defaults to 4):
        The frequency of full attention layers. Every Nth layer will use full attention, while others use sliding
        window attention.
    mup_enabled (`bool`, *optional*, defaults to `False`):
        Whether to enable muP (Maximal Update Parametrization) input scaling. When enabled, input embeddings
        are scaled by `sqrt(hidden_size)`.

    Example:
    ```python
    >>> from transformers import AfmoeModel, AfmoeConfig

    >>> # Initializing an AFMoE configuration
    >>> configuration = AfmoeConfig()

    >>> # Initializing a model from the afmoe-small-sft-v1 style configuration
    >>> model = AfmoeModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    ÚafmoeÚpast_key_valuesÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormÚ	ep_routerÚgrouped_gemmÚmoe_tp_experts)zlayers.*.mlp.routerz!layers.*.mlp.experts.gate_up_projzlayers.*.mlp.experts.down_projzlayers.*.mlp.expertsi  Ú
vocab_sizei   Úhidden_sizei   Úintermediate_sizei€  Úmoe_intermediate_sizeé    Únum_hidden_layersé   NÚnum_dense_layersé   Únum_attention_headsÚnum_key_value_headsé€   Úhead_dimÚsiluÚ
hidden_acti @  Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsTÚ	use_cacheFÚtie_word_embeddingsÚrope_parametersé@   Únum_expertsé   Únum_experts_per_toké   Únum_shared_expertsg      ð?Úroute_scaleÚoutput_router_logitsé   Úglobal_attn_every_n_layersi   Úsliding_windowÚlayer_typesg        Úattention_dropoutÚmup_enabledÚeos_token_idÚpad_token_idÚbos_token_idÚattention_biasc                 ó
  •— | j                   €Et        | j                  «      D �cg c]!  }t        |dz   | j                  z  «      rdnd‘Œ# c}| _         | j
                  €| j                  | _        t        ‰| �   di |¤Ž y c c}w )Nr   Úsliding_attentionÚfull_attention© )	r8   Úranger   Úboolr6   r"   r!   ÚsuperÚ__post_init__)ÚselfÚkwargsÚiÚ	__class__s      €úx/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/afmoe/configuration_afmoe.pyrF   zAfmoeConfig.__post_init__f   s…   ø€ Ø×ÑÐ#ô ˜t×5Ñ5Ó6ö àô (,¨Q°©U°d×6UÑ6UÑ,UÔ'VÑ#Ð\lÑlò ˆDÔð
 ×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$ä‰ÑÑ' Ó'ùò s   ¥&B )0Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_pp_planÚbase_model_ep_planr   ÚintÚ__annotations__r   r   r   r   r   r!   r"   r$   r&   Ústrr'   r(   Úfloatr)   r*   rD   r+   r,   r   Údictr.   r0   r2   r3   r4   r6   r7   r8   Úlistr9   r:   r;   r<   r=   r>   rF   Ú__classcell__)rJ   s   @rK   r   r      s  ø… ñð. €JØ#4Ð"5Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð  +Ø-;Ø*8Ø 0ñ	Ðð €J�ÓØ€K�ÓØ!Ð�sÓ!Ø!%Ð˜3Ó%ØÐ�sÓØ#$Ð�c˜D‘jÓ$Ø!Ð˜Ó!Ø&*Ð˜˜t™Ó*Ø€Hˆc�D‰jÓØ€J�ÓØ#(Ð˜SÓ(Ø#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ %Ð˜Ó%Ø48€O�^ dÑ*¨TÑ1Ó8Ø €K��t‘Ó Ø&'Ð˜˜t™Ó'Ø%&Ð˜˜d™
Ó&Ø #€K�˜‘Ó#Ø!&Ð˜$Ó&Ø-.Ð  d¡
Ó.Ø!%€N�C˜$‘JÓ%Ø$(€K��c‘˜TÑ!Ó(Ø,/Ð�u˜s‘{ TÑ)Ó/Ø$€K�˜‘Ó$Ø+/€L�#˜˜S™	‘/ DÑ(Ó/Ø#€L�#˜‘*Ó#Ø#€L�#˜‘*Ó#Ø €N�DÓ ÷
(ð 
(ó    r   N)rO   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r   Ú__all__rB   r[   rK   ú<module>ra      sV   ðñ  å .å 3Ý 1Ý #ð Ùðð 'ôôQ(Ð"ó Q(óó ðQ(ðh ˆ/�r[   