Ë
    ÜÓ^jŒ  ã                   óv   — d Z ddlmZ ddlmZ ddlmZ ddlmZ  ed¬«      e G d	„ d
e«      «       «       Z	d
gZ
y)zMistral4 model configurationé    )Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringz#mistralai/Mistral-Small-4-119B-2603)Ú
checkpointc            
       óÐ  ‡ — e Zd ZU dZdZdgZddddddddddœ	Zd	gd
gfddgdgfdgdgfdœZdddddœZddiZ	dZ
eed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZedz  ed <   d!Zeed"<   d#Zeed<   d$Zeed%<   d&Zeed'<   d(Zedz  ed)<   d*Zeed+<   d#Zedz  ed,<   d*Zeed-<   d!Zedz  ed.<   d!Zedz  ed/<   d0Zedz  ed1<   d2Zedz  ed3<   d4Z e!dz  ed5<   d6Z"e#ed7<   d8Z$eed9<   d:Z%eed;<   d<Z&eed=<   d4Z'e!ed><   d?Z(edz  ed@<   d!Z)edz  edA<   dBZ*ee+e   z  dz  edC<   d!Z,edz  edD<   dEZ-e!edF<   dZ.e/e0z  dz  edG<   d4Z1e!dz  edH<   dEZ2e!edI<   dJZ3eez  dz  edK<   ˆ fdL„Z4ˆ xZ5S )MÚMistral4Configaã  
    n_group (`int`, *optional*, defaults to 1):
        Number of groups for routed experts.
    first_k_dense_replace (`int`, *optional*, defaults to 0):
        Number of dense layers in shallow layers(embed->dense->dense->...->dense->moe->moe...->lm_head).
                                                        \--k dense layers--/
    rope_interleave (`bool`, *optional*, defaults to `True`):
        Whether to interleave the rotary position embeddings.

    Example:

    ```python
    >>> from transformers import Mistral4Model, Mistral4Config

    >>> # Initializing a Mistral4 style configuration
    >>> configuration = Mistral4Config()

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Úmistral4Úpast_key_valuesÚpacked_colwiseÚrowwiseÚmoe_tp_expertsÚcolwise)	ú!layers.*.mlp.experts.gate_up_projúlayers.*.mlp.experts.down_projúlayers.*.mlp.expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormÚ	ep_routerÚgrouped_gemm)zlayers.*.mlp.gater   r   r   Únum_local_expertsÚn_routed_expertsi   Ú
vocab_sizei   Úhidden_sizei 0  Úintermediate_sizei   Úmoe_intermediate_sizeé$   Únum_hidden_layersé    Únum_attention_headsNÚnum_key_value_headsé   Ún_shared_expertsé€   ç      ð?Úrouted_scaling_factoré   Úkv_lora_ranki   Úq_lora_ranké@   Úqk_rope_head_dimÚ
v_head_dimÚqk_nope_head_dimÚn_groupÚ
topk_groupé   Únum_experts_per_tokr   Úfirst_k_dense_replaceTÚnorm_topk_probÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeg�íµ ÷Æ°>Úrms_norm_epsÚ	use_cacheé   Úpad_token_idÚbos_token_idé   Úeos_token_idÚpretraining_tpFÚtie_word_embeddingsÚrope_parametersÚrope_interleaveÚattention_biasg        Úattention_dropoutc                 óÞ  •— | j                   €Adddd| j                  ddddd| j                  | j                  | j                  z   z  dœ| _         | j                  €| j
                  | _        | j                  | j                  z   | _        | j                  | j                  z   | _        | j                   j                  d	| j                  | j                  z  «       t        ‰| �(  dd
ddhi|¤Ž y )NÚyarng     ˆÃ@g      `@i    g      @@r+   gš™™™™™¹?)ÚtypeÚ
rope_thetaÚfactorÚ original_max_position_embeddingsr<   Ú	beta_fastÚ	beta_slowÚmscale_all_dimÚmscaleÚllama_4_scaling_betaÚpartial_rotary_factorrV   Úignore_keys_at_rope_validationrU   r<   © )rG   r<   r1   r3   r'   r&   Úqk_head_dimÚhead_dimÚ
setdefaultÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €ú~/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/mistral4/configuration_mistral4.pyr]   zMistral4Config.__post_init__n   sö   ø€ Ø×ÑÐ'àØ%ØØ48Ø+/×+GÑ+GØ!Ø Ø"%ØØ(+Ø)-×)>Ñ)>À$×BWÑBWÐZ^×ZoÑZoÑBoÑ)pñ$ˆDÔ ð ×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$à×0Ñ0°4×3HÑ3HÑHˆÔØ×-Ñ-°×0EÑ0EÑEˆŒØ×Ñ×'Ñ'Ð(?À×AVÑAVÐY]×YfÑYfÑAfÔgÜ‰Ññ 	
Ø,BÐD]Ð+^ð	
Øbhó	
ó    )6Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planÚbase_model_ep_planÚattribute_mapr   ÚintÚ__annotations__r    r!   r"   r$   r&   r'   r)   r   r,   Úfloatr.   r/   r1   r2   r3   r4   r5   r7   r8   r9   Úboolr;   Ústrr<   r=   r>   r?   rA   rB   rD   ÚlistrE   rF   rG   r   ÚdictrH   rI   rJ   r]   Ú__classcell__)r`   s   @ra   r
   r
      sV  ø… ñð* €JØ#4Ð"5Ðà-=Ø*3Ø 0Ø1:Ø/8Ø1:Ø"+Ø )Ø"+ñ
Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð )Ø-;Ø*8Ø 0ñ	Ðð 	Ð/ð€Mð €J�ÓØ€K�ÓØ"Ð�sÓ"Ø!%Ð˜3Ó%ØÐ�sÓØ!Ð˜Ó!Ø&(Ð˜˜t™Ó(ØÐ�cÓØÐ�cÓØ#&Ð˜5Ó&Ø€L�#ÓØ"€K��t‘Ó"ØÐ�cÓØ €J��d‘
Ó ØÐ�cÓØ€GˆS�4‰ZÓØ€J��d‘
ÓØ&'Ð˜˜t™Ó'Ø()Ð˜3 ™:Ó)Ø"&€N�D˜4‘KÓ&Ø€J�ÓØ#*Ð˜SÓ*Ø#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ!€L�#˜‘*Ó!Ø €L�#˜‘*Ó Ø+,€L�#˜˜S™	‘/ DÑ(Ó,Ø!"€N�C˜$‘JÓ"Ø %Ð˜Ó%Ø48€O�^ dÑ*¨TÑ1Ó8Ø#'€O�T˜D‘[Ó'Ø €N�DÓ Ø,/Ð�u˜s‘{ TÑ)Ó/÷
ð 
rb   r
   N)rf   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r
   Ú__all__rX   rb   ra   ú<module>rz      sO   ðñ #å .å 3Ý 1Ý #ñ Ð@ÔAØôm
Ð%ó m
ó ó Bðm
ð` Ð
�rb   