
    ^j                     v    d Z ddlmZ ddlmZ ddlmZ ddlmZ  ed      e G d	 d
e                    Z	d
gZ
y)zPyTorch Phi-MoE model.    )strict   )PreTrainedConfig)RopeParameters)auto_docstringzmicrosoft/Phi-3.5-MoE-instruct)
checkpointc                       e Zd ZU dZdZdgZdddddZdZd	Ze	e
d
<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZee
d<   dZe	e
d<   dZee
d<   dZee
d<   dZee
d<   dZe	dz  e
d<   d Ze	dz  e
d!<   d"Ze	ee	   z  dz  e
d#<   d$Zee
d%<   dZeez  dz  e
d&<   dZ e	dz  e
d'<   d(Z!ee	z  e
d)<   d"Z"e	e
d*<   d+Z#e	e
d,<   d$Z$ee
d-<   d.Z%ee
d/<   d0Z&ee
d1<   d(Z'ee
d2<   d$Z(ee
d3<   d$Z)ee
d4<    fd5Z* fd6Z+ xZ,S )7PhimoeConfiga  
    num_local_experts (`int`, *optional*, defaults to 16):
        Number of experts per Sparse MLP layer.
    input_jitter_noise (`float`, *optional*, defaults to 0.0):
        Input jitter noise
    lm_head_bias (`bool`, *optional*, defaults to `False`):
        LM head bias

    Example:

    ```python
    >>> from transformers import PhimoeModel, PhimoeConfig
    >>> # Initializing a Phi-3 style configuration
    >>> configuration = PhimoeConfig.from_pretrained("microsoft/Phi-3.5-MoE-instruct")
    >>> # Initializing a model from the configuration
    >>> model = PhimoeModel(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```phimoepast_key_values	ep_routergrouped_gemmmoe_tp_experts)zlayers.*.mlp.routerz!layers.*.mlp.experts.gate_up_projzlayers.*.mlp.experts.down_projzlayers.*.mlp.expertsg    .Ai@}  
vocab_sizei   hidden_sizei   intermediate_size    num_hidden_layersnum_attention_heads   num_key_value_headssilu
hidden_acti   max_position_embeddingsg{Gz?initializer_rangegh㈵>rms_norm_epsT	use_cacheNpad_token_id   bos_token_id   eos_token_idFtie_word_embeddingsrope_parameterssliding_windowg        attention_dropoutnum_experts_per_tok   num_local_expertsoutput_router_logitsgMbP?router_aux_loss_coefg{Gz?router_jitter_noiseinput_jitter_noiseattention_biaslm_head_biasc                 ^    | j                   | j                  | _         t        |   di | y )N )r   r   super__post_init__)selfkwargs	__class__s     z/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/phimoe/configuration_phimoe.pyr3   zPhimoeConfig.__post_init__T   s-    ##+'+'?'?D$''    c                    t         |           | j                  d   dk7  rd| j                  v r| j                  d   | _        | j                  j	                  dd      }| j                  j	                  dd      }t        |t        t        f      st        d|       t        |t        t        f      st        d|       yy)	z?
        Validate the `rope_parameters` configuration.
        	rope_typedefault original_max_position_embeddingsshort_mscaleNlong_mscalez=`rope_parameters`'s short_mscale field must be a number, got z<`rope_parameters`'s long_mscale field must be a number, got )	r2   validate_roper$   r<   get
isinstanceintfloat	TypeError)r4   rope_parameters_short_mscalerope_parameters_long_mscaler6   s      r7   r?   zPhimoeConfig.validate_ropeY   s     	 ,	91T5I5II8<8L8LMo8p5+/+?+?+C+CNTX+Y(*.*>*>*B*B=RV*W':S%LISTpSqr  9C<HRSnRop  I :r8   )-__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencebase_model_ep_plandefault_thetar   rB   __annotations__r   r   r   r   r   r   strr   r   rC   r   r   boolr   r    r"   listr#   r$   r   dictr%   r&   r'   r)   r*   r+   r,   r-   r.   r/   r3   r?   __classcell__)r6   s   @r7   r
   r
      s{   ( J#4"5*-;*8 0	 MJK!s!s!!  J#,S,#u#L%It#L#*# L#* +,L#S	/D(, %%48O^d*T18!%NC$J%%(us{(  s!&$&"'%'!%% ## ND L$(
 r8   r
   N)rJ   huggingface_hub.dataclassesr   configuration_utilsr   modeling_rope_utilsr   utilsr   r
   __all__r1   r8   r7   <module>rZ      sN     . 3 1 # ;<R# R  =Rj 
r8   