Ë
    ÜÓ^jí  ã                   óÐ  — d dl mZ d dlZddlmZ ddlmZ ddlmZm	Z	m
Z
 ddlmZ ddlmZ dd	lmZmZ d
dlmZmZmZmZmZmZmZ  ej4                  e«      Zdej:                  dededej:                  fd„Z  G d„ de«      Z! G d„ de«      Z"e G d„ de«      «       Z#e G d„ de«      «       Z$e G d„ de«      «       Z% G d„ de
e#«      Z& G d„ de	e#«      Z' G d„ d ee#«      Z(g d!¢Z)y)"é    )ÚCallableNé   )ÚCache)ÚFlashAttentionKwargs)ÚGenericForQuestionAnsweringÚ GenericForSequenceClassificationÚGenericForTokenClassification)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)Úauto_docstringÚloggingé   )ÚMistralAttentionÚMistralDecoderLayerÚMistralForCausalLMÚMistralModelÚMistralPreTrainedModelÚapply_rotary_pos_embÚeager_attention_forwardÚpositions_idsÚbetaÚmax_position_embeddingsÚreturnc           	      ó„   — d|t        j                  dt        j                  | |z  «      z   «      z  z   }|d d …d d d …d f   S )Né   )ÚtorchÚlogÚfloor)r   r   r   Úscalings       ú|/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/ministral3/modular_ministral3.pyÚget_llama_4_attn_scaler!      sB   € Ø�$œŸ™ 1¤u§{¡{°=ÐCZÑ3ZÓ'[Ñ#[Ó\Ñ\Ñ\€GØ’1�dšA˜tÐ#Ñ$Ð$ó    c                   óì   — e Zd Z	 d
dej                  deej                  ej                  f   dej                  dz  dej                  dedz  dee   deej                  ej                  dz  f   fd	„Z	y)ÚMinistral3AttentionNÚhidden_statesÚposition_embeddingsÚattention_maskÚposition_idsÚpast_key_valuesÚkwargsr   c           
      ó  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
| j                  |«      j                  |«      j	                  dd«      }|\  }}t        |	|
||«      \  }	}
|	t        || j                  j                  j                  d«      | j                  j                  j                  d«      «      j                  |	j                  «      z  }	|� |j                  |
|| j                  «      \  }
}t!        j"                  | j                  j$                  t&        «      } || |	|
||f| j(                  sdn| j*                  | j,                  t/        | j                  dd «      dœ|¤Ž\  }} |j0                  g |¢d‘­Ž j3                  «       }| j5                  |«      }||fS )	Néÿÿÿÿr   r   Úllama_4_scaling_betaÚ original_max_position_embeddingsg        Úsliding_window)Údropoutr   r/   )ÚshapeÚhead_dimÚq_projÚviewÚ	transposeÚk_projÚv_projr   r!   ÚconfigÚrope_parametersÚgetÚtoÚdtypeÚupdateÚ	layer_idxr
   Úget_interfaceÚ_attn_implementationr   ÚtrainingÚattention_dropoutr   ÚgetattrÚreshapeÚ
contiguousÚo_proj)Úselfr%   r&   r'   r(   r)   r*   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                    r    ÚforwardzMinistral3Attention.forward#   sý  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jØ#Ô&<ØØ�K‰K×'Ñ'×+Ñ+Ð,BÓCØ�K‰K×'Ñ'×+Ñ+Ð,NÓOó'
÷ ‰"ˆ\×ÑÓ
 ñ	!ˆð Ð&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LÜ" 4§;¡;Ð0@À$ÓGñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r"   )N)
Ú__name__Ú
__module__Ú__qualname__r   ÚTensorÚtupler   r   r   rR   © r"   r    r$   r$   "   s”   „ ð )-ñ-)à—|‘|ð-)ð # 5§<¡<°·±Ð#=Ñ>ð-)ð Ÿ™ tÑ+ð	-)ð
 —l‘lð-)ð  ™ð-)ð Ð-Ñ.ð-)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1ô-)r"   r$   c                   ó   — e Zd Zy)ÚMinistral3DecoderLayerN©rS   rT   rU   rX   r"   r    rZ   rZ   S   ó   „ Ør"   rZ   c                   ó   — e Zd Zy)ÚMinistral3PreTrainedModelNr[   rX   r"   r    r^   r^   W   ó   „ àr"   r^   c                   ó   — e Zd Zy)ÚMinistral3ModelNr[   rX   r"   r    ra   ra   \   r_   r"   ra   c                   ó   — e Zd Zy)ÚMinistral3ForCausalLMNr[   rX   r"   r    rc   rc   a   r_   r"   rc   c                   ó   — e Zd Zy)Ú Ministral3ForTokenClassificationNr[   rX   r"   r    re   re   f   r\   r"   re   c                   ó   — e Zd Zy)Ú#Ministral3ForSequenceClassificationNr[   rX   r"   r    rg   rg   j   r\   r"   rg   c                   ó   — e Zd Zy)ÚMinistral3ForQuestionAnsweringNr[   rX   r"   r    ri   ri   n   r\   r"   ri   )rc   ri   ra   r^   rg   re   )*Úcollections.abcr   r   Úcache_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   r   r	   Úmodeling_utilsr
   Úprocessing_utilsr   Úutilsr   r   Úmistral.modeling_mistralr   r   r   r   r   r   r   Ú
get_loggerrS   ÚloggerrV   ÚfloatÚintr!   r$   rZ   r^   ra   rc   re   rg   ri   Ú__all__rX   r"   r    ú<module>rw      s  ðÝ $ã å  Ý B÷ñ õ
 6Ý &ß ,÷÷ ñ ð 
ˆ×	Ñ	˜HÓ	%€ð%¨%¯,©,ð %¸eð %Ð^að %Ðfk×frÑfró %ô
.)Ð*ô .)ôb	Ð0ô 	ð ô	Ð 6ó 	ó ð	ð ô	�ló 	ó ð	ð ô	Ð.ó 	ó ð	ô	Ð'DÐF_ô 	ô	Ð*JÐLeô 	ô	Ð%@ÐB[ô 	ò�r"   