Ë
    ÜÓ^jå  ã                   ó  — d dl Z d dlmZ d dl mZ ddlmZmZ ddlmZ ddl	m
Z
mZ ddlmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$  ed¬«      e G d„ de«      «       «       Z% G d„ de «      Z& G d„ de«      Z' G d„ de#«      Z( G d„ de«      Z) G d„ de"«      Z* G d„ de$«      Z+ G d „ d!e!«      Z, G d"„ d#e«      Z- G d$„ d%e«      Z. G d&„ d'e«      Z/ G d(„ d)e«      Z0g d*¢Z1y)+é    N)Ústrict)Únné   )ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚBaseModelOutputWithPast)ÚUnpack)ÚTransformersKwargsÚauto_docstring)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚMistralConfig)ÚQwen2AttentionÚQwen2DecoderLayerÚQwen2ForCausalLMÚQwen2ForQuestionAnsweringÚQwen2ForSequenceClassificationÚQwen2ForTokenClassificationÚQwen2MLPÚ
Qwen2ModelÚQwen2PreTrainedModelÚQwen2RMSNormÚQwen2RotaryEmbeddingz$mistralai/Ministral-8B-Instruct-2410)Ú
checkpointc                   ó6   — e Zd ZU dZdZdZee   dz  ed<   d„ Z	y)ÚMinistralConfiga”  
    Example:

    ```python
    >>> from transformers import MinistralModel, MinistralConfig

    >>> # Initializing a Ministral 8B style configuration
    >>> configuration = MinistralConfig()

    >>> # Initializing a model from the Ministral 8B style configuration
    >>> model = MinistralModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Ú	ministralNÚlayer_typesc                 óÈ   — | j                   €| j                  | _         | j                  €#| j                  �dndg| j                  z  | _        t        j                  | fi |¤Ž y )NÚsliding_attentionÚfull_attention)Únum_key_value_headsÚnum_attention_headsr"   Úsliding_windowÚnum_hidden_layersr   Ú__post_init__)ÚselfÚkwargss     úz/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/ministral/modular_ministral.pyr*   zMinistralConfig.__post_init__C   sg   € Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$à×ÑÐ#à'+×':Ñ':Ð'FÑ#ÐL\ð à×&Ñ&ñ 'ˆDÔô 	×&Ñ& tÑ6¨vÓ6ó    )
Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typer"   ÚlistÚstrÚ__annotations__r*   © r.   r-   r    r    ,   s&   … ñð  €Jà$(€K��c‘˜TÑ!Ó(ó	7r.   r    c                   ó   — e Zd Zy)ÚMinistralMLPN©r/   r0   r1   r7   r.   r-   r9   r9   O   ó   „ Ør.   r9   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚMinistralAttentionÚ	layer_idxc                 óš  •— t         ‰| �  ||«       t        j                  |j                  |j
                  | j                  z  d¬«      | _        t        j                  |j                  |j                  | j                  z  d¬«      | _	        t        j                  |j                  |j                  | j                  z  d¬«      | _
        y )NF)Úbias)ÚsuperÚ__init__r   ÚLinearÚhidden_sizer'   Úhead_dimÚq_projr&   Úk_projÚv_proj)r+   Úconfigr>   Ú	__class__s      €r-   rB   zMinistralAttention.__init__T   s™   ø€ Ü‰Ñ˜ Ô+ä—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆ�r.   )r/   r0   r1   ÚintrB   Ú__classcell__©rJ   s   @r-   r=   r=   S   s   ø„ ðl¨#÷ lñ lr.   r=   c                   ó   — e Zd Zy)ÚMinistralRMSNormNr:   r7   r.   r-   rO   rO   \   r;   r.   rO   c                   ó   — e Zd Zy)ÚMinistralDecoderLayerNr:   r7   r.   r-   rQ   rQ   `   r;   r.   rQ   c                   ó   — e Zd Zy)ÚMinistralPreTrainedModelNr:   r7   r.   r-   rS   rS   d   r;   r.   rS   c                   ó   — e Zd Zy)ÚMinistralRotaryEmbeddingNr:   r7   r.   r-   rU   rU   h   r;   r.   rU   c                   óò   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   defd„«       «       «       Zˆ xZS )ÚMinistralModelrI   c                 ó(   •— t         ‰| �  |«       | `y )N)rA   rB   Úhas_sliding_layers)r+   rI   rJ   s     €r-   rB   zMinistralModel.__init__m   s   ø€ Ü‰Ñ˜Ô ØÑ#r.   NÚ	input_idsÚattention_maskÚposition_idsÚpast_key_valuesÚinputs_embedsÚ	use_cacher,   Úreturnc           
      óÚ  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      s)| j                  ||||dœ}
t        d
i |
¤Žt        d
i |
¤Ždœ}	|}| j                  ||«      }t        | j                   d | j                  j"                   «      D ].  \  }} ||f|	| j                  j$                  |      ||||dœ|¤Ž}Œ0 | j'                  |«      }t)        ||r|¬	«      S d ¬	«      S )Nz:You must specify exactly one of input_ids or inputs_embeds)rI   r   é   )Údevice)rI   r^   r[   r]   r\   )r%   r$   )r[   r\   r]   r_   Úposition_embeddings)Úlast_hidden_stater]   r7   )Ú
ValueErrorÚembed_tokensr   rI   Úget_seq_lengthÚtorchÚarangeÚshaperc   Ú	unsqueezeÚ
isinstanceÚdictr	   r
   Ú
rotary_embÚ	enumerateÚlayersr)   r"   Únormr   )r+   rZ   r[   r\   r]   r^   r_   r,   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚhidden_statesrd   ÚiÚdecoder_layers                  r-   ÚforwardzMinistralModel.forwardq   s¨  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-ÄÔFð Ÿ+™+Ø!.Ø"0Ø#2Ø ,ñˆKô #5Ñ"C°{Ñ"CÜ%FÑ%UÈÑ%Uñ#Ðð
 &ˆØ"Ÿo™o¨m¸\ÓJÐä )¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÓ Wò 		ÑˆAˆ}Ù)Øðà2°4·;±;×3JÑ3JÈ1Ñ3MÑNØ)Ø /Ø#Ø$7ñð ñ‰Mð		ð Ÿ	™	 -Ó0ˆÜ&Ø+Ù/8˜Oô
ð 	
à>Bô
ð 	
r.   )NNNNNN)r/   r0   r1   r    rB   r   r   r   ri   Ú
LongTensorÚTensorr   ÚFloatTensorÚboolr   r   r   ry   rL   rM   s   @r-   rW   rW   l   sÍ   ø„ ð$˜õ $ð  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ:
à×#Ñ# dÑ*ð:
ð Ÿ™ tÑ+ð:
ð ×&Ñ&¨Ñ-ð	:
ð
  ™ð:
ð ×(Ñ(¨4Ñ/ð:
ð ˜$‘;ð:
ð Ð+Ñ,ð:
ð 
!ò:
ó ó ó  ô:
r.   rW   c                   ó   — e Zd Zy)ÚMinistralForCausalLMNr:   r7   r.   r-   r   r   ±   r;   r.   r   c                   ó   — e Zd Zy)Ú"MinistralForSequenceClassificationNr:   r7   r.   r-   r�   r�   µ   r;   r.   r�   c                   ó   — e Zd Zy)ÚMinistralForTokenClassificationNr:   r7   r.   r-   rƒ   rƒ   ¹   r;   r.   rƒ   c                   ó   — e Zd Zy)ÚMinistralForQuestionAnsweringNr:   r7   r.   r-   r…   r…   ½   r;   r.   r…   )r    rS   rW   r   r�   rƒ   r…   )2ri   Úhuggingface_hub.dataclassesr   r   Úcache_utilsr   r   Úconfiguration_utilsr   Úmasking_utilsr	   r
   Úmodeling_outputsr   Úprocessing_utilsr   Úutilsr   r   Úutils.genericr   Úutils.output_capturingr   Úmistral.configuration_mistralr   Úqwen2.modeling_qwen2r   r   r   r   r   r   r   r   r   r   r   r    r9   r=   rO   rQ   rS   rU   rW   r   r�   rƒ   r…   Ú__all__r7   r.   r-   ú<module>r’      s  ðó  Ý .Ý ç .Ý 3ß RÝ 7Ý &ß 7Ý 7Ý 5Ý 9÷÷ ÷ ñ ñ ÐAÔBØô7�mó 7ó ó Cð7ôB	�8ô 	ôl˜ô lô	�|ô 	ô	Ð-ô 	ô	Ð3ô 	ô	Ð3ô 	ôB
�Zô B
ôJ	Ð+ô 	ô	Ð)Gô 	ô	Ð&Aô 	ô	Ð$=ô 	ò�r.   