Ë
    ÜÓ^j–  ã                   óv   — d Z ddlmZ ddlmZ ddlmZ ddlmZ  ed¬«      e G d	„ d
e«      «       «       Z	d
gZ
y)zPyTorch Phi-MoE model.é    )Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringzmicrosoft/Phi-3.5-MoE-instruct)Ú
checkpointc                   óì  ‡ — e Zd ZU dZdZdgZdddddœZdZd	Ze	e
d
<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZee
d<   dZe	e
d<   dZee
d<   dZee
d<   dZee
d<   dZe	dz  e
d<   d Ze	dz  e
d!<   d"Ze	ee	   z  dz  e
d#<   d$Zee
d%<   dZeez  dz  e
d&<   dZ e	dz  e
d'<   d(Z!ee	z  e
d)<   d"Z"e	e
d*<   d+Z#e	e
d,<   d$Z$ee
d-<   d.Z%ee
d/<   d0Z&ee
d1<   d(Z'ee
d2<   d$Z(ee
d3<   d$Z)ee
d4<   ˆ fd5„Z*ˆ fd6„Z+ˆ xZ,S )7ÚPhimoeConfiga²  
    num_local_experts (`int`, *optional*, defaults to 16):
        Number of experts per Sparse MLP layer.
    input_jitter_noise (`float`, *optional*, defaults to 0.0):
        Input jitter noise
    lm_head_bias (`bool`, *optional*, defaults to `False`):
        LM head bias

    Example:

    ```python
    >>> from transformers import PhimoeModel, PhimoeConfig
    >>> # Initializing a Phi-3 style configuration
    >>> configuration = PhimoeConfig.from_pretrained("microsoft/Phi-3.5-MoE-instruct")
    >>> # Initializing a model from the configuration
    >>> model = PhimoeModel(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```ÚphimoeÚpast_key_valuesÚ	ep_routerÚgrouped_gemmÚmoe_tp_experts)zlayers.*.mlp.routerz!layers.*.mlp.experts.gate_up_projzlayers.*.mlp.experts.down_projzlayers.*.mlp.expertsg    €„.Ai@}  Ú
vocab_sizei   Úhidden_sizei   Úintermediate_sizeé    Únum_hidden_layersÚnum_attention_headsé   Únum_key_value_headsÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsTÚ	use_cacheNÚpad_token_idé   Úbos_token_idé   Úeos_token_idFÚtie_word_embeddingsÚrope_parametersÚsliding_windowg        Úattention_dropoutÚnum_experts_per_toké   Únum_local_expertsÚoutput_router_logitsgü©ñÒMbP?Úrouter_aux_loss_coefg{®Gáz„?Úrouter_jitter_noiseÚinput_jitter_noiseÚattention_biasÚlm_head_biasc                 ó^   •— | j                   €| j                  | _         t        ‰| �  di |¤Ž y )N© )r   r   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úz/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/phimoe/configuration_phimoe.pyr3   zPhimoeConfig.__post_init__T   s-   ø€ Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$Ü‰ÑÑ' Ó'ó    c                 óŒ  •— t         ‰| �  «        | j                  d   dk7  r£d| j                  v r| j                  d   | _        | j                  j	                  dd«      }| j                  j	                  dd«      }t        |t        t        f«      st        d|› �«      ‚t        |t        t        f«      st        d|› �«      ‚yy)	z?
        Validate the `rope_parameters` configuration.
        Ú	rope_typeÚdefaultÚ original_max_position_embeddingsÚshort_mscaleNÚlong_mscalez=`rope_parameters`'s short_mscale field must be a number, got z<`rope_parameters`'s long_mscale field must be a number, got )	r2   Úvalidate_roper$   r<   ÚgetÚ
isinstanceÚintÚfloatÚ	TypeError)r4   Úrope_parameters_short_mscaleÚrope_parameters_long_mscaler6   s      €r7   r?   zPhimoeConfig.validate_ropeY   sØ   ø€ ô 	‰ÑÔð ×Ñ Ñ,°	Ò9Ø1°T×5IÑ5IÑIØ8<×8LÑ8LÐMoÑ8p�Ô5Ø+/×+?Ñ+?×+CÑ+CÀNÐTXÓ+YÐ(Ø*.×*>Ñ*>×*BÑ*BÀ=ÐRVÓ*WÐ'ÜÐ:¼SÄ%¸LÔIÜØSÐTpÐSqÐróð ô Ð9¼CÄ¸<ÔHÜØRÐSnÐRoÐpóð ð Ið :r8   )-Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_ep_planÚdefault_thetar   rB   Ú__annotations__r   r   r   r   r   r   Ústrr   r   rC   r   r   Úboolr   r    r"   Úlistr#   r$   r   Údictr%   r&   r'   r)   r*   r+   r,   r-   r.   r/   r3   r?   Ú__classcell__)r6   s   @r7   r
   r
      s{  ø… ñð( €JØ#4Ð"5Ðà*Ø-;Ø*8Ø 0ñ	Ðð €Mà€J�ÓØ€K�ÓØ!Ð�sÓ!ØÐ�sÓØ!Ð˜Ó!Ø Ð˜Ó Ø€J�ÓØ#,Ð˜SÓ,Ø#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ#€L�#˜‘*Ó#Ø €L�#˜‘*Ó Ø+,€L�#˜˜S™	‘/ DÑ(Ó,Ø %Ð˜Ó%Ø48€O�^ dÑ*¨TÑ1Ó8Ø!%€N�C˜$‘JÓ%Ø%(Ð�u˜s‘{Ó(Ø Ð˜Ó ØÐ�sÓØ!&Ð˜$Ó&Ø"'Ð˜%Ó'Ø!%Ð˜Ó%Ø #Ð˜Ó#Ø €N�DÓ Ø€L�$Óô(÷
ð r8   r
   N)rJ   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r
   Ú__all__r1   r8   r7   ú<module>rZ      sN   ðñ å .å 3Ý 1Ý #ñ Ð;Ô<ØôRÐ#ó Ró ó =ðRðj Ð
�r8   