Ë
    ÜÓ^j!  ã                   óÈ  — d dl Z d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
mZ ddlmZ ddlmZmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z)  G d„ de%«      Z* G d„ de!«      Z+d„ Z, G d„ de'«      Z- G d„ de«      Z. G d„ de«      Z/e G d„ de «      «       Z0e G d „ d!e«      «       Z1e G d"„ d#e«      «       Z2g d$¢Z3y)%é    N)ÚCallableé   )Úinitialization)ÚCacheÚDynamicCache)Úcreate_causal_mask)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringé   )ÚCLIPMLP)ÚGemma2ForCausalLM)ÚLlamaDecoderLayerÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)ÚLlama4TextL2Norm)ÚQwen3Attentioné   )ÚNanoChatConfigc                   ó   — e Zd Zy)ÚNanoChatRMSNormN©Ú__name__Ú
__module__Ú__qualname__© ó    úx/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/nanochat/modular_nanochat.pyr   r   +   ó   „ Ør$   r   c                   ó   — e Zd Zy)ÚNanoChatRotaryEmbeddingNr   r#   r$   r%   r(   r(   /   r&   r$   r(   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  || fd¬«      S )zJRotates half the hidden dims of the input with flipped signs for NanoChat..Néÿÿÿÿr   )Údim)ÚshapeÚtorchÚcat)ÚxÚx1Úx2s      r%   Úrotate_halfr2   3   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�b˜2˜#�Y BÔ'Ð'r$   c                   óú   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  de	dz  d	e
e   d
eej                  ej                  dz  f   fd„Zˆ xZS )ÚNanoChatAttentionÚconfigÚ	layer_idxc                 óš   •— t         ‰| �  ||«       | `| `t	        |j
                  ¬«      | _        t	        |j
                  ¬«      | _        y ©N)Úeps)ÚsuperÚ__init__Úsliding_windowÚ
layer_typer   Úrms_norm_epsÚq_normÚk_norm©Úselfr5   r6   Ú	__class__s      €r%   r;   zNanoChatAttention.__init__;   sA   ø€ Ü‰Ñ˜ Ô+ØÐØˆOä%¨&×*=Ñ*=Ô>ˆŒÜ%¨&×*=Ñ*=Ô>ˆ�r$   NÚhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valuesÚkwargsÚreturnc                 óN  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
|\  }}t        ||	||«      \  }}	| j                  |«      }| j                  |	«      }	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t         «      } || ||	|
|f| j"                  sdn| j$                  | j&                  dœ|¤Ž\  }} |j(                  g |¢d‘­Ž j+                  «       }| j-                  |«      }||fS )Nr*   r   r   ç        )ÚdropoutÚscaling)r,   Úhead_dimÚq_projÚviewÚ	transposeÚk_projÚv_projr   r?   r@   Úupdater6   r   Úget_interfacer5   Ú_attn_implementationr   ÚtrainingÚattention_dropoutrM   ÚreshapeÚ
contiguousÚo_proj)rB   rD   rE   rF   rG   rH   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                   r%   ÚforwardzNanoChatAttention.forwardC   s¯  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jð —{‘{ <Ó0ˆØ—[‘[ Ó,ˆ
àÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r$   )NNN)r    r!   r"   r   Úintr;   r-   ÚTensorÚtupler   r   r   rf   Ú__classcell__©rC   s   @r%   r4   r4   :   s§   ø„ ð?˜~ð ?¸#õ ?ð IMØ.2Ø(,ñ*)à—|‘|ð*)ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð*)ð Ÿ™ tÑ+ð	*)ð
  ™ð*)ð Ð+Ñ,ð*)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷*)r$   r4   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚNanoChatMLPc                 óè   •— t         ‰| �  |«       t        j                  |j                  |j
                  d¬«      | _        t        j                  |j
                  |j                  d¬«      | _        y )NF)Úbias)r:   r;   ÚnnÚLinearÚhidden_sizeÚintermediate_sizeÚfc1Úfc2©rB   r5   rC   s     €r%   r;   zNanoChatMLP.__init__q   sR   ø€ Ü‰Ñ˜Ô Ü—9‘9˜V×/Ñ/°×1IÑ1IÐPUÔVˆŒÜ—9‘9˜V×5Ñ5°v×7IÑ7IÐPUÔVˆ�r$   )r    r!   r"   r;   rj   rk   s   @r%   rm   rm   p   s   ø„ ÷Wð Wr$   rm   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚNanoChatDecoderLayerr5   r6   c                 óŽ   •— t         ‰| �  «        t        |j                  ¬«      | _        t        |j                  ¬«      | _        y r8   )r:   r;   r   r>   Úinput_layernormÚpost_attention_layernormrA   s      €r%   r;   zNanoChatDecoderLayer.__init__x   s4   ø€ Ü‰ÑÔä.°6×3FÑ3FÔGˆÔÜ(7¸F×<OÑ<OÔ(PˆÕ%r$   )r    r!   r"   r   rg   r;   rj   rk   s   @r%   rx   rx   w   s   ø„ ðQ˜~ð Q¸#÷ Qñ Qr$   rx   c                   ó0   — e Zd Zdej                  ddfd„Zy)ÚNanoChatPreTrainedModelÚmodulerI   Nc           	      ó,  — t        j                  | |«       t        |t        «      rnt	        j
                  |j                  j                  d| j                  j                  t        j                  d| j                  j                  z  «      z  ¬«       y y )NrK   r   )ÚmeanÚstd)r   Ú_init_weightsÚ
isinstancer4   ÚinitÚnormal_r[   Úweightr5   Úinitializer_rangeÚmathÚsqrtÚnum_hidden_layers)rB   r~   s     r%   r‚   z%NanoChatPreTrainedModel._init_weights�   sh   € Ü×%Ñ% d¨FÔ3Ü�fÔ/Ô0Ü�L‰LØ—‘×$Ñ$ØØ—K‘K×1Ñ1´D·I±I¸aÀ$Ç+Á+×B_ÑB_Ñ>_Ó4`Ñ`öð 1r$   )r    r!   r"   rp   ÚModuler‚   r#   r$   r%   r}   r}      s   „ ð B§I¡Ið °$ô r$   r}   c                   óÔ   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	e
dz  d
ee   defd„Zˆ xZS )ÚNanoChatModelr5   c                 óZ   •— t         ‰| �  |«       t        |j                  ¬«      | _        y r8   )r:   r;   r   r>   Únormrv   s     €r%   r;   zNanoChatModel.__init__�   s"   ø€ Ü‰Ñ˜Ô ä#¨×(;Ñ(;Ô<ˆ�	r$   NÚ	input_idsrF   Úposition_idsrG   Úinputs_embedsÚ	use_cacherH   rI   c           	      ó\  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        | j                  ||||¬«      }	|}
| j                  |
|¬«      }| j                  |
«      }
| j                  d | j                  j                   D ]  } ||
f|	|||dœ|¤Ž}
Œ | j                  |
«      }
t        |
|¬	«      S )
Nz:You must specify exactly one of input_ids or inputs_embeds)r5   r   r   )Údevice)r5   r’   rF   rG   r‘   )r‘   )rF   rE   r‘   rG   )Úlast_hidden_staterG   )Ú
ValueErrorÚembed_tokensr   r5   Úget_seq_lengthr-   Úaranger,   r•   Ú	unsqueezer   Ú
rotary_embr�   ÚlayersrŠ   r	   )rB   r�   rF   r‘   rG   r’   r“   rH   Úpast_seen_tokensÚcausal_maskrD   rE   Údecoder_layers                r%   rf   zNanoChatModel.forward’   sW  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø*.×*;Ñ*;¸IÓ*FˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨mÈ,˜oÓWÐàŸ	™	 -Ó0ˆØ!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÐIò 	ˆMÙ)Øðà*Ø$7Ø)Ø /ñð ñ‰Mð	ð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ô
ð 	
r$   )NNNNNN)r    r!   r"   r   r;   r-   Ú
LongTensorrh   r   ÚFloatTensorÚboolr   r   r	   rf   rj   rk   s   @r%   r�   r�   ‹   s®   ø„ ð=˜~õ =ð .2Ø.2Ø04Ø(,Ø26Ø!%ñ2
à×#Ñ# dÑ*ð2
ð Ÿ™ tÑ+ð2
ð ×&Ñ&¨Ñ-ð	2
ð
  ™ð2
ð ×(Ñ(¨4Ñ/ð2
ð ˜$‘;ð2
ð Ð+Ñ,ð2
ð 
!÷2
r$   r�   c                   ó,   ‡ — e Zd ZddiZdefˆ fd„Zˆ xZS )ÚNanoChatForCausalLMÚlm_headÚcolwise_gather_outputrI   c                 ó$   •— t        ‰| �  di |¤Ž y)ak  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, AutoModelForCausalLM

        >>> model = AutoModelForCausalLM.from_pretrained("karpathy/nanochat-d32")

        >>> tokenizer = AutoTokenizer.from_pretrained("karpathy/nanochat-d32")

        >>> conversation = [
                {"role": "user", "content": "What is the capital of France?"},
            ]

        >>> inputs = tokenizer.apply_chat_template(
                conversation, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt"
            ).to(device)

        >>> with torch.no_grad():
        >>>     outputs = model.generate(**inputs, max_new_tokens=64, do_sample=False)

        >>> generated_tokens = outputs[0, inputs["input_ids"].shape[1] :]
        >>> output = tokenizer.decode(generated_tokens, skip_special_tokens=True)
        ```Nr#   )r:   rf   )rB   Úsuper_kwargsrC   s     €r%   rf   zNanoChatForCausalLM.forwardË   s   ø€ ô2 	‰‰Ñ'˜,Ó'r$   )r    r!   r"   Ú_tp_planr
   rf   rj   rk   s   @r%   r¥   r¥   Ç   s    ø„ àÐ2Ð3€Hð(Ð)?÷ (ñ (r$   r¥   )r}   r�   r¥   )4rˆ   Úcollections.abcr   r-   Útorch.nnrp   Ú r   r„   Úcache_utilsr   r   Úmasking_utilsr   Úmodeling_outputsr	   r
   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úclip.modeling_clipr   Úgemma2.modeling_gemma2r   Úllama.modeling_llamar   r   r   r   r   r   Úllama4.modeling_llama4r   Úqwen3.modeling_qwen3r   Úconfiguration_nanochatr   r   r(   r2   r4   rm   rx   r}   r�   r¥   Ú__all__r#   r$   r%   ú<module>r»      sç   ðó Ý $ã Ý å &ß .Ý /ß Oß FÝ &ß 7Ý (Ý 6÷÷ õ 6Ý 1Ý 2ô	Ð&ô 	ô	Ð2ô 	ò(ô3)˜ô 3)ôlW�'ô WôQÐ,ô Qð ôÐ2ó ó ðð ô8
�Jó 8
ó ð8
ðv ô(Ð+ó (ó ð(ò>�r$   