Ë
    ÜÓ^j„  ã                   óÜ  — d dl mZ d dlZd dlmZ ddlmZmZ ddlmZm	Z	 ddl
mZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZ ddlmZ ddlmZ ddlmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& ddl'm(Z(  ejR                  e*«      Z+ G d„ de"«      Z, G d„ de«      Z- G d„ de«      Z. G d„ de$«      Z/ G d„ de#«      Z0 G d„ de«      Z1 G d„ d e!«      Z2 G d!„ d"e «      Z3 G d#„ d$ee/«      Z4g d%¢Z5y)&é    )ÚCallableN)Únné   )ÚCacheÚDynamicCache)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGenericForQuestionAnswering)ÚBaseModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úmerge_with_config_defaults)Úcapture_outputsé   )
ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaForCausalLMÚLlamaForSequenceClassificationÚLlamaForTokenClassificationÚLlamaMLPÚ
LlamaModelÚLlamaPreTrainedModelÚapply_rotary_pos_embÚeager_attention_forwardé   )ÚMistralConfigc                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )Ú
MistralMLPc                 óJ  •— t         ‰| �  |«       t        j                  | j                  | j
                  d¬«      | _        t        j                  | j                  | j
                  d¬«      | _        t        j                  | j
                  | j                  d¬«      | _        y )NF©Úbias)	ÚsuperÚ__init__r   ÚLinearÚhidden_sizeÚintermediate_sizeÚ	gate_projÚup_projÚ	down_proj)ÚselfÚconfigÚ	__class__s     €úv/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/mistral/modular_mistral.pyr'   zMistralMLP.__init__%   ss   ø€ Ü‰Ñ˜Ô ÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆ�ó    )Ú__name__Ú
__module__Ú__qualname__r'   Ú__classcell__©r0   s   @r1   r"   r"   $   s   ø„ ÷Yð Yr2   r"   c                   óð   ‡ — e Zd Zdedefˆ fd„Z	 ddej                  deej                  ej                  f   dej                  dz  de	dz  d	e
e   d
eej                  ej                  dz  f   fd„Zˆ xZS )ÚMistralAttentionr/   Ú	layer_idxc                 óp  •— t         ‰| �  ||«       t        |dd «      xs |j                  |j                  z  | _        t        j                  |j                  |j                  | j
                  z  d¬«      | _        t        j                  |j                  |j                  | j
                  z  d¬«      | _
        t        j                  |j                  |j                  | j
                  z  d¬«      | _        t        j                  |j                  | j
                  z  |j                  d¬«      | _        y )NÚhead_dimFr$   )r&   r'   Úgetattrr)   Únum_attention_headsr<   r   r(   Úq_projÚnum_key_value_headsÚk_projÚv_projÚo_proj©r.   r/   r:   r0   s      €r1   r'   zMistralAttention.__init__-   sê   ø€ Ü‰Ñ˜ Ô+Ü ¨
°DÓ9Òm¸V×=OÑ=OÐSY×SmÑSmÑ=mˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆŒÜ—i‘i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^ÐejÔkˆ�r2   NÚhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valuesÚkwargsÚreturnc           
      ó6  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t        «      } || ||	|
|f| j                  sdn| j                   | j"                  t%        | j                  dd «      dœ|¤Ž\  }} |j&                  g |¢d‘­Ž j)                  «       }| j+                  |«      }||fS )Néÿÿÿÿr   r   g        Úsliding_window)ÚdropoutÚscalingrM   )Úshaper<   r?   ÚviewÚ	transposerA   rB   r   Úupdater:   r   Úget_interfacer/   Ú_attn_implementationr   ÚtrainingÚattention_dropoutrO   r=   ÚreshapeÚ
contiguousrC   )r.   rE   rF   rG   rH   rI   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                   r1   ÚforwardzMistralAttention.forward5   s¢  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LÜ" 4§;¡;Ð0@À$ÓGñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r2   )N)r3   r4   r5   r    Úintr'   ÚtorchÚTensorÚtupler   r   r
   rd   r6   r7   s   @r1   r9   r9   ,   s�   ø„ ðl˜}ð l¸õ lð )-ñ')à—|‘|ð')ð # 5§<¡<°·±Ð#=Ñ>ð')ð Ÿ™ tÑ+ð	')ð
  ™ð')ð Ð-Ñ.ð')ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷')r2   r9   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚMistralDecoderLayerr/   r:   c                 ój   •— t         ‰| �  ||«       t        ||¬«      | _        t	        |«      | _        y )N)r/   r:   )r&   r'   r9   Ú	self_attnr"   ÚmlprD   s      €r1   r'   zMistralDecoderLayer.__init__`   s,   ø€ Ü‰Ñ˜ Ô+Ü)°À9ÔMˆŒÜ˜fÓ%ˆ�r2   )r3   r4   r5   r    re   r'   r6   r7   s   @r1   rj   rj   _   s   ø„ ð&˜}ð &¸÷ &ñ &r2   rj   c                   ó   — e Zd ZeedœZy)ÚMistralPreTrainedModel)rE   Ú
attentionsN)r3   r4   r5   rj   r9   Ú_can_record_outputs© r2   r1   ro   ro   f   s   „ à,Ø&ñÑr2   ro   c                   óÚ   — e Zd Zeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de	dz  dej                  dz  dedz  dee   d	efd
„«       «       «       Zy)ÚMistralModelNÚ	input_idsrG   Úposition_idsrH   Úinputs_embedsÚ	use_cacherI   rJ   c           
      óŒ  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }| j                  j                  €t        nt        }	 |	| j                  ||||¬«      }
|}| j                  ||¬«      }| j                  d | j                  j                   D ]  } ||f|
||||dœ|¤Ž}Œ | j!                  |«      }t#        ||r|¬	«      S d ¬	«      S )
Nz:You must specify exactly one of input_ids or inputs_embeds)r/   r   r   )Údevice)r/   rw   rG   rH   rv   )rv   )rG   rv   rH   rx   rF   )Úlast_hidden_staterH   )Ú
ValueErrorÚembed_tokensr   r/   Úget_seq_lengthrf   ÚarangerP   rz   Ú	unsqueezerM   r   r	   Ú
rotary_embÚlayersÚnum_hidden_layersÚnormr   )r.   ru   rG   rv   rH   rw   rx   rI   Úpast_seen_tokensÚmask_functionÚcausal_maskrE   rF   Údecoder_layers                 r1   rd   zMistralModel.forwardn   ss  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLà.2¯k©k×.HÑ.HÐ.PÕ*ÔVwˆÙ#Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨mÈ,˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÐIò 		ˆMÙ)Øðà*Ø)Ø /Ø#Ø$7ñð ñ‰Mð		ð Ÿ	™	 -Ó0ˆÜ&Ø+Ù/8˜Oô
ð 	
à>Bô
ð 	
r2   )NNNNNN)r3   r4   r5   r   r   r   rf   Ú
LongTensorrg   r   ÚFloatTensorÚboolr   r   r   rd   rr   r2   r1   rt   rt   m   s¾   „ ØØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ2
à×#Ñ# dÑ*ð2
ð Ÿ™ tÑ+ð2
ð ×&Ñ&¨Ñ-ð	2
ð
  ™ð2
ð ×(Ñ(¨4Ñ/ð2
ð ˜$‘;ð2
ð Ð+Ñ,ð2
ð 
!ò2
ó ó ó  ñ2
r2   rt   c                   ó   — e Zd Zy)ÚMistralForCausalLMN©r3   r4   r5   rr   r2   r1   r�   r�   ¦   ó   „ Ør2   r�   c                   ó   — e Zd Zy)ÚMistralForTokenClassificationNrŽ   rr   r2   r1   r‘   r‘   ª   r�   r2   r‘   c                   ó   — e Zd Zy)Ú MistralForSequenceClassificationNrŽ   rr   r2   r1   r“   r“   ®   r�   r2   r“   c                   ó   — e Zd Zy)ÚMistralForQuestionAnsweringNrŽ   rr   r2   r1   r•   r•   ²   s   … r2   r•   )r�   r•   rt   ro   r“   r‘   )6Úcollections.abcr   rf   r   Úcache_utilsr   r   Úmasking_utilsr   r	   Úmodeling_flash_attention_utilsr
   Úmodeling_layersr   Úmodeling_outputsr   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   r   Úconfiguration_mistralr    Ú
get_loggerr3   Úloggerr"   r9   rj   ro   rt   r�   r‘   r“   r•   Ú__all__rr   r2   r1   ú<module>r¦      sÚ   ðÝ $ã Ý ç .ß RÝ Bõõ 8Ý 5Ý &ß @Ñ @Ý 7Ý 5÷÷ ÷ õ 1ð 
ˆ×	Ñ	˜HÓ	%€ôY�ô Yô0)�~ô 0)ôf&Ð+ô &ôÐ1ô ô6
�:ô 6
ôr	Ð)ô 	ô	Ð$?ô 	ô	Ð'Eô 	ô \Ð"=Ð?UÔ [ò�r2   