
    ^j                     v    d Z ddlmZ ddlmZ ddlmZ ddlmZ  ed      e G d	 d
e                    Z	d
gZ
y)zMixtral model configuration    )strict   )PreTrainedConfig)RopeParameters)auto_docstringzmistralai/Mixtral-8x7B-v0.1)
checkpointc                       e Zd ZU dZdZdgZdZdddddddd	Zd
gdgfddgdgfdgdgfdZdddddZ	ddiZ
dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZedz  ed <   d!Zeed"<   d#Zeed$<   d%Zeed&<   d'Zeed(<   d)Zeed*<   dZedz  ed+<   d,Zedz  ed-<   d.Zeee   z  dz  ed/<   d0Z eed1<   dZ!edz  ed2<   d3Z"eez  ed4<   d.Z#eed5<   dZ$eed<   d0Z%eed6<   d7Z&eed8<   d3Z'eed9<   dZ(e)e*z  dz  ed:<    fd;Z+ xZ,S )<MixtralConfiga  
    Example:

    ```python
    >>> from transformers import MixtralModel, MixtralConfig

    >>> # Initializing a Mixtral 7B style configuration
    >>> configuration = MixtralConfig()

    >>> # Initializing a model from the Mixtral 7B style configuration
    >>> model = MixtralModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```mixtralpast_key_valuesg    .Acolwiserowwisepacked_colwisemoe_tp_experts)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_proj!layers.*.mlp.experts.gate_up_projlayers.*.mlp.experts.down_projlayers.*.mlp.experts	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnorm	ep_routergrouped_gemm)zlayers.*.mlp.gater   r   r   num_expertsnum_local_expertsi }  
vocab_sizei   hidden_sizei 8  intermediate_size    num_hidden_layersnum_attention_heads   num_key_value_headsNhead_dimsilu
hidden_acti   max_position_embeddingsg{Gz?initializer_rangegh㈵>rms_norm_epsT	use_cachepad_token_id   bos_token_id   eos_token_idFtie_word_embeddingssliding_windowg        attention_dropoutnum_experts_per_tokoutput_router_logitsgMbP?router_aux_loss_coefrouter_jitter_noiserope_parametersc                 ^    | j                   | j                  | _         t        |   di | y )N )r&   r$   super__post_init__)selfkwargs	__class__s     |/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/mixtral/configuration_mixtral.pyr>   zMixtralConfig.__post_init__\   s-    ##+'+'?'?D$''    )-__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencedefault_thetabase_model_tp_planbase_model_pp_planbase_model_ep_planattribute_mapr   int__annotations__r    r!   r#   r$   r&   r'   r)   strr*   r+   floatr,   r-   boolr.   r0   r2   listr3   r4   r5   r6   r   r7   r8   r9   r:   r   dictr>   __classcell__)rA   s   @rB   r
   r
      s     J#4"5M%.%.%.%.-=*3 0 &(9:#%568IJ!"_$56 )-;*8 0	 #$78MJK"s"s!!  HcDjJ#,S,#u#L%It#L#*# L#* +,L#S	/D(, %%!%NC$J%%(us{(  s!&$&"'%'!$$48O^d*T18( (rC   r
   N)rG   huggingface_hub.dataclassesr   configuration_utilsr   modeling_rope_utilsr   utilsr   r
   __all__r<   rC   rB   <module>r\      sN    " . 3 1 # 89G($ G(  :G(T 
rC   