Ë
    ÜÓ^jO  ã                   ó”  — d Z ddlmZ ddlZddlmZ ddlmZ ddlm	Z	 ddl
mZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZmZmZmZmZmZmZmZ ddlmZ  ej@                  e!«      Z"dZ# G d„ de«      Z$ G d„ de«      Z% G d„ de«      Z& G d„ de«      Z' G d„ de«      Z( G d„ de«      Z) G d„ de«      Z* G d „ d!e«      Z+g d"¢Z,y)#zPyTorch Qwen3 model.é    )ÚCallableNé   )ÚCache)ÚFlashAttentionKwargs)ÚCausalLMOutputWithPast)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚloggingé   )ÚGemmaMLP)ÚLlamaAttention)ÚQwen2ForCausalLMÚQwen2ForQuestionAnsweringÚQwen2ForSequenceClassificationÚQwen2ForTokenClassificationÚQwen2RMSNormÚQwen2RotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forwardé   )ÚQwen3ConfigzQwen/Qwen3-8Bc                   ó   — e Zd Zy)ÚQwen3RMSNormN©Ú__name__Ú
__module__Ú__qualname__© ó    úr/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/qwen3/modular_qwen3.pyr   r   0   ó   „ Ør    r   c                   ó   — e Zd Zy)ÚQwen3MLPNr   r   r    r!   r$   r$   4   r"   r    r$   c                   ó   — e Zd Zy)ÚQwen3RotaryEmbeddingNr   r   r    r!   r&   r&   8   r"   r    r&   c                   óð   ‡ — e Zd Zdedefˆ fd„Z	 ddej                  deej                  ej                  f   dej                  dz  de	dz  d	e
e   d
eej                  ej                  dz  f   fd„Zˆ xZS )ÚQwen3AttentionÚconfigÚ	layer_idxc                 óR  •— t        |d«      r|j                  |   nd | _        t        ‰| �  ||«       t        | j                  |j                  ¬«      | _        t        | j                  |j                  ¬«      | _	        | j                  dk(  r|j                  | _
        y d | _
        y )NÚlayer_types)ÚepsÚsliding_attention)Úhasattrr,   Ú
layer_typeÚsuperÚ__init__r   Úhead_dimÚrms_norm_epsÚq_normÚk_normÚsliding_window)Úselfr)   r*   Ú	__class__s      €r!   r2   zQwen3Attention.__init__=   s‚   ø€ Ü;BÀ6È=Ô;Y˜&×,Ñ,¨YÒ7Ð_cˆŒÜ‰Ñ˜ Ô+Ü" 4§=¡=°f×6IÑ6IÔJˆŒÜ" 4§=¡=°f×6IÑ6IÔJˆŒØ7;·±ÐJ]Ò7]˜f×3Ñ3ˆÕÐcgˆÕr    NÚhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valuesÚkwargsÚreturnc                 ó\  — |j                   d d }g |¢d‘| j                  ‘­}| j                  | j                  |«      j	                  |«      «      j                  dd«      }| j                  | j                  |«      j	                  |«      «      j                  dd«      }	| j                  |«      j	                  |«      j                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t         «      } || ||	|
|f| j"                  sdn| j$                  | j&                  | j(                  dœ|¤Ž\  }} |j*                  g |¢d‘­Ž j-                  «       }| j/                  |«      }||fS )Néÿÿÿÿr   r   g        )ÚdropoutÚscalingr7   )Úshaper3   r5   Úq_projÚviewÚ	transposer6   Úk_projÚv_projr   Úupdater*   r   Úget_interfacer)   Ú_attn_implementationr   ÚtrainingÚattention_dropoutrC   r7   ÚreshapeÚ
contiguousÚo_proj)r8   r:   r;   r<   r=   r>   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                   r!   ÚforwardzQwen3Attention.forwardD   s®  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ 4§;¡;¨}Ó#=×#BÑ#BÀ<Ó#PÓQ×[Ñ[Ð\]Ð_`ÓaˆØ—[‘[ §¡¨]Ó!;×!@Ñ!@ÀÓ!NÓO×YÑYÐZ[Ð]^Ó_ˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r    )N)r   r   r   r   Úintr2   ÚtorchÚTensorÚtupler   r	   r   r\   Ú__classcell__©r9   s   @r!   r(   r(   <   s�   ø„ ðh˜{ð h°sõ hð )-ñ')à—|‘|ð')ð # 5§<¡<°·±Ð#=Ñ>ð')ð Ÿ™ tÑ+ð	')ð
  ™ð')ð Ð-Ñ.ð')ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷')r    r(   c                   ó.   ‡ — e Zd Zdee   defˆ fd„Zˆ xZS )ÚQwen3ForCausalLMÚsuper_kwargsr?   c                 ó"   •— t        ‰| �  di |¤ŽS )a^  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Qwen3ForCausalLM

        >>> model = Qwen3ForCausalLM.from_pretrained("Qwen/Qwen3-8B")
        >>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-8B")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```r   )r1   r\   )r8   re   r9   s     €r!   r\   zQwen3ForCausalLM.forwardo   s   ø€ ô4 ‰w‰Ñ. Ñ.Ð.r    )r   r   r   r	   r
   r   r\   ra   rb   s   @r!   rd   rd   n   s%   ø„ ð/àÐ1Ñ2ð/ð 
 ÷/ñ /r    rd   c                   ó   — e Zd Zy)ÚQwen3ForSequenceClassificationNr   r   r    r!   rh   rh   Œ   r"   r    rh   c                   ó   — e Zd Zy)ÚQwen3ForTokenClassificationNr   r   r    r!   rj   rj   �   r"   r    rj   c                   ó   — e Zd Zy)ÚQwen3ForQuestionAnsweringNr   r   r    r!   rl   rl   ”   r"   r    rl   )rd   rl   ÚQwen3PreTrainedModelÚ
Qwen3Modelrh   rj   )-Ú__doc__Úcollections.abcr   r^   Úcache_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_outputsr   Úmodeling_utilsr   Úprocessing_utilsr	   Úutilsr
   r   Úgemma.modeling_gemmar   Úllama.modeling_llamar   Úqwen2.modeling_qwen2r   r   r   r   r   r   r   r   Úconfiguration_qwen3r   Ú
get_loggerr   ÚloggerÚ_CHECKPOINT_FOR_DOCr   r$   r&   r(   rd   rh   rj   rl   Ú__all__r   r    r!   ú<module>r      sÅ   ðñ å $ã å  Ý BÝ 6Ý 5Ý &ß 0Ý +õ÷	÷ 	ó 	õ -ð 
ˆ×	Ñ	˜HÓ	%€à%Ð ô	�<ô 	ô	ˆxô 	ô	Ð/ô 	ô/)�^ô /)ôd/Ð'ô /ô<	Ð%Cô 	ô	Ð"=ô 	ô	Ð 9ô 	ò�r    