Ë
    ÛÓ^jÿ  ã                   ór   — d dl mZ ddlmZ ddlmZ ddlmZ  ed¬«      e G d„ d	e«      «       «       Zd	gZ	y
)é    )Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringzrednote-hilab/dots.llm1.base)Ú
checkpointc                   ó°  ‡ — e Zd ZU dZdZdgZdddddddddddddddd	œZd
gdgfddgdgfdgdgfdœZdddddœZddiZ	dZ
eed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZed z  ed!<   d Zed z  ed"<   d Zed z  ed<   d#Zed z  ed$<   d#Zed z  ed%<   d Zed z  ed&<   d'Zed z  ed(<   d)Zed z  ed*<   d+Zeed,<   d-Zeed.<   d/Zeed0<   d1Z eed2<   d3Z!eed4<   d)Z"eed5<   d Z#e$e%z  d z  ed6<   d)Z&eed7<   d8Z'eez  d z  ed9<   d:Z(eed;<   d<Z)ed z  ed=<   dZ*ed z  ed><   d Z+e,e   d z  ed?<   d Z-ed z  ed@<   d Z.ed z  edA<   d Z/ee,e   z  d z  edB<   ˆ fdC„Z0ˆ xZ1S )DÚDots1Configa  
    n_group (`int`, *optional*, defaults to 1):
        Number of groups for routed experts.
    first_k_dense_replace (`int`, *optional*, defaults to 0):
        Number of dense layers at the beginning of the model before the first MoE layer.

    Examples:

    ```python
    >>> from transformers import Dots1Model, Dots1Config
    >>> # Initializing a Dots1 style configuration
    >>> configuration = Dots1Config()
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    Údots1Úpast_key_valuesÚcolwiseÚrowwiseÚreplicated_with_grad_allreduceÚpacked_colwiseÚmoe_tp_experts)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.self_attn.q_normzlayers.*.self_attn.k_normú!layers.*.mlp.experts.gate_up_projúlayers.*.mlp.experts.down_projúlayers.*.mlp.expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormÚ	ep_routerÚgrouped_gemm)zlayers.*.mlp.gater   r   r   Únum_local_expertsÚn_routed_expertsi R Ú
vocab_sizei   Úhidden_sizeiÀ*  Úintermediate_sizei€  Úmoe_intermediate_sizeé>   Únum_hidden_layersé    Únum_attention_headsNÚnum_key_value_headsÚn_shared_expertsé   Ún_groupÚ
topk_groupÚnum_experts_per_tokr   Úfirst_k_dense_replaceFÚnorm_topk_probÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeg�íµ ÷Æ°>Úrms_norm_epsTÚ	use_cacheÚtie_word_embeddingsÚrope_parametersÚattention_biasg        Úattention_dropoutg      ð?Úrouted_scaling_factori   Úsliding_windowÚmax_window_layersÚlayer_typesÚpad_token_idÚbos_token_idÚeos_token_idc                 ó
  •— | j                   €| j                  | _         | j                  €Et        | j                  «      D �cg c]!  }| j
                  �|| j                  k\  rdnd‘Œ# c}| _        t        ‰| �   di |¤Ž y c c}w )NÚsliding_attentionÚfull_attention© )	r(   r'   r=   Úranger%   r;   r<   ÚsuperÚ__post_init__)ÚselfÚkwargsÚiÚ	__class__s      €úx/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/dots1/configuration_dots1.pyrG   zDots1Config.__post_init__r   s�   ø€ Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$à×ÑÐ#ô
 ˜t×5Ñ5Ó6ö	 ð ð ×&Ñ&Ð2°q¸D×<RÑ<RÒ7Rñ $à%ñ&ò ˆDÔô 	‰ÑÑ' Ó'ùò s   Á&B )2Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planÚbase_model_ep_planÚattribute_mapr    ÚintÚ__annotations__r!   r"   r#   r%   r'   r(   r)   r   r+   r,   r-   r.   r/   Úboolr1   Ústrr2   r3   Úfloatr4   r5   r6   r7   r   Údictr8   r9   r:   r;   r<   r=   Úlistr>   r?   r@   rG   Ú__classcell__)rK   s   @rL   r
   r
      sJ  ø… ñð" €JØ#4Ð"5Ðð &/Ø%.Ø%.Ø%.Ø%EØ%EØ-=Ø*3Ø 0Ø1:Ø/8Ø1:Ø"+Ø )Ø"+ñÐð& &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð )Ø-;Ø*8Ø 0ñ	Ðð 	Ð/ð€Mð €J�ÓØ€K�ÓØ"Ð�sÓ"Ø!%Ð˜3Ó%ØÐ�sÓØ!Ð˜Ó!Ø&(Ð˜˜t™Ó(Ø#'Ð�c˜D‘jÓ'Ø#'Ð�c˜D‘jÓ'Ø€GˆS�4‰ZÓØ€J��d‘
ÓØ&*Ð˜˜t™Ó*Ø()Ð˜3 ™:Ó)Ø"'€N�D˜4‘KÓ'Ø€J�ÓØ#'Ð˜SÓ'Ø#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ %Ð˜Ó%Ø48€O�^ dÑ*¨TÑ1Ó8Ø €N�DÓ Ø,/Ð�u˜s‘{ TÑ)Ó/Ø#&Ð˜5Ó&Ø!%€N�C˜$‘JÓ%Ø$&Ð�s˜T‘zÓ&Ø$(€K��c‘˜TÑ!Ó(Ø#€L�#˜‘*Ó#Ø#€L�#˜‘*Ó#Ø+/€L�#˜˜S™	‘/ DÑ(Ó/÷(ð (ó    r
   N)
Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r
   Ú__all__rD   r_   rL   ú<module>re      sJ   ðõ( /å 3Ý 1Ý #ñ Ð9Ô:Øôa(Ð"ó a(ó ó ;ða(ðH ˆ/�r_   