Ë
    ÜÓ^j”  ã                   óü  — d dl mZ d dlZd dlmZ ddlmZmZ ddlmZm	Z	 ddl
mZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZ ddlmZ ddlmZ ddlmZ ddlmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& ddl'm(Z( ddl)m*Z*  ejV                  e,«      Z- G d„ de#«      Z. G d„ de«      Z/ G d„ de«      Z0 G d„ de«      Z1 G d„ de$«      Z2 G d„ de(«      Z3 G d „ d!e«      Z4 G d"„ d#e!«      Z5 G d$„ d%e"«      Z6 G d&„ d'e «      Z7g d(¢Z8y))é    )ÚCallableN)Únné   )ÚCacheÚDynamicCache)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚBaseModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚGemma2RotaryEmbedding)
ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaForCausalLMÚLlamaForQuestionAnsweringÚLlamaForSequenceClassificationÚLlamaForTokenClassificationÚLlamaMLPÚLlamaPreTrainedModelÚapply_rotary_pos_embÚeager_attention_forward)ÚMistralModelé   )ÚQwen2Configc                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚQwen2MLPc                 óJ  •— t         ‰| �  |«       t        j                  | j                  | j
                  d¬«      | _        t        j                  | j                  | j
                  d¬«      | _        t        j                  | j
                  | j                  d¬«      | _        y )NF©Úbias)	ÚsuperÚ__init__r   ÚLinearÚhidden_sizeÚintermediate_sizeÚ	gate_projÚup_projÚ	down_proj©ÚselfÚconfigÚ	__class__s     €úr/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/qwen2/modular_qwen2.pyr(   zQwen2MLP.__init__&   ss   ø€ Ü‰Ñ˜Ô ÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆ�ó    )Ú__name__Ú
__module__Ú__qualname__r(   Ú__classcell__©r2   s   @r3   r#   r#   %   s   ø„ ÷Yð Yr4   r#   c                   ó   — e Zd Zy)ÚQwen2RotaryEmbeddingN©r5   r6   r7   © r4   r3   r;   r;   -   ó   „ Ør4   r;   c                   óð   ‡ — e Zd Zdedefˆ fd„Z	 ddej                  deej                  ej                  f   dej                  dz  de	dz  d	e
e   d
eej                  ej                  dz  f   fd„Zˆ xZS )ÚQwen2Attentionr1   Ú	layer_idxc                 óª  •— t        |d«      r|j                  |   nd | _        t        ‰| �  ||«       t        j                  |j                  |j                  | j                  z  d¬«      | _
        t        j                  |j                  |j                  | j                  z  d¬«      | _        t        j                  |j                  |j                  | j                  z  d¬«      | _        t        j                  |j                  | j                  z  |j                  d¬«      | _        | j                  dk(  r|j                  | _        y d | _        y )NÚlayer_typesTr%   FÚsliding_attention)ÚhasattrrC   Ú
layer_typer'   r(   r   r)   r*   Únum_attention_headsÚhead_dimÚq_projÚnum_key_value_headsÚk_projÚv_projÚo_projÚsliding_window)r0   r1   rA   r2   s      €r3   r(   zQwen2Attention.__init__2   s  ø€ Ü;BÀ6È=Ô;Y˜&×,Ñ,¨YÒ7Ð_cˆŒÜ‰Ñ˜ Ô+Ü—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐeiÔjˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐeiÔjˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐeiÔjˆŒÜ—i‘i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^ÐejÔkˆŒØ7;·±ÐJ]Ò7]˜f×3Ñ3ˆÕÐcgˆÕr4   NÚhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valuesÚkwargsÚreturnc                 ó   — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t        «      } || ||	|
|f| j                  sdn| j                   | j"                  | j$                  dœ|¤Ž\  }} |j&                  g |¢d‘­Ž j)                  «       }| j+                  |«      }||fS )Néÿÿÿÿr    r   g        )ÚdropoutÚscalingrN   )ÚshaperH   rI   ÚviewÚ	transposerK   rL   r   ÚupdaterA   r   Úget_interfacer1   Ú_attn_implementationr   ÚtrainingÚattention_dropoutrX   rN   ÚreshapeÚ
contiguousrM   )r0   rO   rP   rQ   rR   rS   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                   r3   ÚforwardzQwen2Attention.forward;   sš  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r4   )N)r5   r6   r7   r!   Úintr(   ÚtorchÚTensorÚtupler   r   r
   rm   r8   r9   s   @r3   r@   r@   1   s�   ø„ ðh˜{ð h°sõ hð )-ñ')à—|‘|ð')ð # 5§<¡<°·±Ð#=Ñ>ð')ð Ÿ™ tÑ+ð	')ð
  ™ð')ð Ð-Ñ.ð')ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷')r4   r@   c                   ó   — e Zd Zy)ÚQwen2DecoderLayerNr<   r=   r4   r3   rs   rs   e   r>   r4   rs   c                   ó   — e Zd Zy)ÚQwen2PreTrainedModelNr<   r=   r4   r3   ru   ru   i   r>   r4   ru   c                   óò   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   defd„«       «       «       Zˆ xZS )Ú
Qwen2Modelr1   c                 ó^   •— t         ‰| �  |«       d| j                  j                  v | _        y )NrD   )r'   r(   r1   rC   Úhas_sliding_layersr/   s     €r3   r(   zQwen2Model.__init__n   s'   ø€ Ü‰Ñ˜Ô Ø"5¸¿¹×9PÑ9PÐ"PˆÕr4   NÚ	input_idsrQ   Úposition_idsrR   Úinputs_embedsÚ	use_cacherS   rT   c           
      óú  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      s9| j                  ||||dœ}
dt        di |
¤Ži}	| j                  rt        di |
¤Ž|	d<   |}| j                  ||«      }t!        | j"                  d | j                  j$                   «      D ].  \  }} ||f|	| j                  j&                  |      ||||d	œ|¤Ž}Œ0 | j)                  |«      }t+        ||r|¬
«      S d ¬
«      S )Nz:You must specify exactly one of input_ids or inputs_embeds)r1   r   r    )Údevice)r1   r|   rQ   rR   r{   Úfull_attentionrD   )rQ   rP   r{   rR   r}   )Úlast_hidden_staterR   r=   )Ú
ValueErrorÚembed_tokensr   r1   Úget_seq_lengthro   ÚarangerY   r   Ú	unsqueezeÚ
isinstanceÚdictr   ry   r	   Ú
rotary_embÚ	enumerateÚlayersÚnum_hidden_layersrC   Únormr   )r0   rz   rQ   r{   rR   r|   r}   rS   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsrO   rP   ÚiÚdecoder_layers                  r3   rm   zQwen2Model.forwardr   s¾  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-ÄÔFð Ÿ+™+Ø!.Ø"0Ø#2Ø ,ñˆKð !Ô"4Ñ"C°{Ñ"Cð#Ðð ×&Ò&Ü;\Ñ;kÐ_jÑ;kÐ#Ð$7Ñ8à%ˆØ"Ÿo™o¨m¸\ÓJÐä )¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÓ Wò 		ÑˆAˆ}Ù)Øðà2°4·;±;×3JÑ3JÈ1Ñ3MÑNØ$7Ø)Ø /Ø#ñð ñ‰Mð		ð Ÿ	™	 -Ó0ˆÜ&Ø+Ù/8˜Oô
ð 	
à>Bô
ð 	
r4   )NNNNNN)r5   r6   r7   r!   r(   r   r   r   ro   Ú
LongTensorrp   r   ÚFloatTensorÚboolr   r   r   rm   r8   r9   s   @r3   rw   rw   m   sÏ   ø„ ðQ˜{õ Qð  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ<
à×#Ñ# dÑ*ð<
ð Ÿ™ tÑ+ð<
ð ×&Ñ&¨Ñ-ð	<
ð
  ™ð<
ð ×(Ñ(¨4Ñ/ð<
ð ˜$‘;ð<
ð Ð+Ñ,ð<
ð 
!ò<
ó ó ó  ô<
r4   rw   c                   ó   — e Zd Zy)ÚQwen2ForCausalLMNr<   r=   r4   r3   r—   r—   ´   r>   r4   r—   c                   ó   — e Zd Zy)ÚQwen2ForSequenceClassificationNr<   r=   r4   r3   r™   r™   ¸   r>   r4   r™   c                   ó   — e Zd Zy)ÚQwen2ForTokenClassificationNr<   r=   r4   r3   r›   r›   ¼   r>   r4   r›   c                   ó   — e Zd Zy)ÚQwen2ForQuestionAnsweringNr<   r=   r4   r3   r�   r�   À   r>   r4   r�   )ru   rw   r—   ÚQwen2RMSNormr™   r›   r�   )9Úcollections.abcr   ro   r   Úcache_utilsr   r   Úmasking_utilsr   r	   Úmodeling_flash_attention_utilsr
   Úmodeling_outputsr   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   Úgemma2.modeling_gemma2r   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   r   Úmistral.modeling_mistralr   Úconfiguration_qwen2r!   Ú
get_loggerr5   Úloggerr#   r;   r@   rs   ru   rw   r—   r™   r›   r�   Ú__all__r=   r4   r3   ú<module>r°      sê   ðÝ $ã Ý ç .ß RÝ Bõõ 6Ý &ß @Ñ @Ý 7Ý 5Ý :÷÷ ÷ õ 4Ý ,ð 
ˆ×	Ñ	˜HÓ	%€ôYˆxô Yô	Ð0ô 	ô1)�^ô 1)ôh	Ð)ô 	ô	Ð/ô 	ôD
�ô D
ôN	Ð'ô 	ô	Ð%Cô 	ô	Ð"=ô 	ô	Ð 9ô 	ò�r4   