Ë
    ÜÓ^j(7  ã                   óN  — d dl Z d dlmZ d dlZd dlmc mZ d dlm	Z	 ddl
mZ ddlmZ ddlmZmZ ddlmZmZ dd	lmZ dd
lmZmZmZmZ ddlmZ ddlmZ ddl m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.  ej^                  e0«      Z1 ed¬«      e	 G d„ de#«      «       «       Z2 G d„ de!«      Z3 G d„ de+«      Z4 G d„ de,«      Z5 G d„ de(«      Z6 G d„ de«      Z7 G d„ d e%«      Z8 G d!„ d"e*«      Z9e G d#„ d$e)«      «       Z:e G d%„ d&e&«      «       Z; G d'„ d(e'«      Z<g d)¢Z=y)*é    N)ÚCallable)Ústricté   )Úinitialization)ÚCache)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úis_flash_attention_requestedé   )ÚDeepseekV2Attention)ÚGemma3TextScaledWordEmbedding)ÚLlamaConfig)
ÚLlamaDecoderLayerÚLlamaForCausalLMÚLlamaForSequenceClassificationÚLlamaMLPÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRMSNormÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forwardzopenbmb/MiniCPM3-4B)Ú
checkpointc            	       óz  ‡ — e Zd ZU dZdddddddddœZdZdZeed<   d	Z	eed
<   dZ
eed<   dZeed<   dZeed<   dZedz  ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZedz  ed<   dZeed <   d!Zeed"<   dZedz  ed#<   d$Zeez  ed%<   d&Zeez  dz  ed'<   dZedz  ed(<   ˆ fd)„Zed*efd+„«       Zˆ xZS ),ÚMiniCPM3Configa‰  
    kv_lora_rank (`int`, *optional*, defaults to 256):
        Rank of the low-rank KV projection in multi-head latent attention.
    q_lora_rank (`int`, *optional*, defaults to 768):
        Rank of the low-rank query projection in multi-head latent attention. If `None`, the query projection
        is a single dense projection rather than a low-rank one.
    qk_nope_head_dim (`int`, *optional*, defaults to 64):
        Dimension of the non-RoPE part of each query/key head.
    qk_rope_head_dim (`int`, *optional*, defaults to 32):
        Dimension of the RoPE part of each query/key head.
    v_head_dim (`int`, *optional*):
        Dimension of each value head. If `None`, defaults to `hidden_size // num_attention_heads`.
    scale_emb (`int` or `float`, *optional*, defaults to 12):
        Multiplier applied to input embeddings.
    scale_depth (`int` or `float`, *optional*, defaults to 1.4):
        Multiplier for residual connections; the effective scaling is `scale_depth / sqrt(num_hidden_layers)`.
        If `None`, defaults to `sqrt(num_hidden_layers)` (no-op scaling).
    dim_model_base (`int`, *optional*, defaults to 256):
        Base model dimension used to scale logits before the language model head. If `None`,
        defaults to `hidden_size` (no-op scaling).

    Example:

    ```python
    >>> from transformers import MiniCPM3Model, MiniCPM3Config
    >>> # Initializing a MiniCPM3 style configuration
    >>> configuration = MiniCPM3Config()
    >>> # Initializing a model from the configuration
    >>> model = MiniCPM3Model(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    ÚcolwiseÚmla_kv_a_projÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.q_b_projz%layers.*.self_attn.kv_a_proj_with_mqazlayers.*.self_attn.kv_b_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚminicpm3iè Ú
vocab_sizei 
  Úhidden_sizei   Úintermediate_sizeé>   Únum_hidden_layersé(   Únum_attention_headsNÚnum_key_value_headsi €  Úmax_position_embeddingsgš™™™™™¹?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsTÚtie_word_embeddingsé   Úkv_lora_ranki   Úq_lora_ranké@   Úqk_nope_head_dimé    Úqk_rope_head_dimÚ
v_head_dimé   Ú	scale_embgffffffö?Úscale_depthÚdim_model_basec                 ó4  •— | j                   | _        | j                  €| j                  | j                  z  | _        | j
                  €$t        j                  | j                  «      | _        | j                  €| j                  | _	        t        ‰| �,  di |¤Ž y )N© )r9   Úhead_dimr:   r(   r-   r=   ÚmathÚsqrtr+   r>   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úx/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/minicpm3/modular_minicpm3.pyrE   zMiniCPM3Config.__post_init__x   s�   ø€ à×-Ñ-ˆŒð �?‰?Ð"Ø"×.Ñ.°$×2JÑ2JÑJˆDŒOØ×ÑÐ#Ü#Ÿy™y¨×)?Ñ)?Ó@ˆDÔØ×ÑÐ&Ø"&×"2Ñ"2ˆDÔÜ‰ÑÑ' Ó'ó    Úreturnc                 ó4   — | j                   | j                  z  S ©N)r(   r>   )rF   s    rI   Úlogits_scalingzMiniCPM3Config.logits_scaling…   s   € ð ×Ñ $×"5Ñ"5Ñ5Ð5rJ   ) Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úbase_model_tp_planÚ
model_typer'   ÚintÚ__annotations__r(   r)   r+   r-   r.   r/   r0   Úfloatr1   r2   Úboolr4   r5   r7   r9   r:   r<   r=   r>   rE   ÚpropertyrN   Ú__classcell__©rH   s   @rI   r"   r"   1   s.  ø… ñ ðF &/Ø'0Ø1@Ø(1Ø%.Ø"+Ø )Ø"+ñ	Ðð €Jð €J�ÓØ€K�ÓØ!Ð�sÓ!ØÐ�sÓØ!Ð˜Ó!Ø&(Ð˜˜t™Ó(Ø#(Ð˜SÓ(Ø"Ð�uÓ"Ø€L�%ÓØ $Ð˜Ó$Ø€L�#ÓØ!€K��t‘Ó!ØÐ�cÓØÐ�cÓØ!€J��d‘
Ó!Ø€Iˆs�U‰{ÓØ&)€K��u‘˜tÑ#Ó)Ø!$€N�C˜$‘JÓ$ô(ð ð6 ò 6ó ô6rJ   r"   c                   ó   — e Zd Zy)ÚMiniCPM3ScaledWordEmbeddingN©rO   rP   rQ   r@   rJ   rI   r]   r]   ‹   ó   „ ØrJ   r]   c                   ó   — e Zd Zy)ÚMiniCPM3RMSNormNr^   r@   rJ   rI   ra   ra   �   r_   rJ   ra   c                   ó   — e Zd Zy)ÚMiniCPM3RotaryEmbeddingNr^   r@   rJ   rI   rc   rc   “   r_   rJ   rc   c                   ó   — e Zd Zy)ÚMiniCPM3MLPNr^   r@   rJ   rI   re   re   —   r_   rJ   re   c                   óÐ   — e Zd ZdZ	 	 d	dej
                  deej
                  ej
                  f   dej
                  dz  dedz  deej
                  ej
                  dz  f   f
d„Zy)
ÚMiniCPM3AttentionaE  
    Multi-head Latent Attention (MLA), structurally identical to `DeepseekV2Attention`.
    The only difference is the rotary convention: MiniCPM3 keeps the original cos/sin RoPE
    (`apply_rotary_pos_emb`) instead of DeepSeek-V2's complex rotary, so we inherit the
    module construction and override only `forward`.
    NÚhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valuesrK   c                 óò  — |j                   d d \  }}||d| j                  f}||d| j                  | j                  z   f}	| j                  €| j                  |«      }
n/| j                  | j                  | j                  |«      «      «      }
|
j                  |«      j                  dd«      }
t        j                  |
| j                  | j                  gd¬«      \  }}| j                  |«      }t        j                  || j                  | j                  gd¬«      \  }}| j!                  | j#                  |«      «      j                  |	«      j                  dd«      }t        j                  || j                  | j                  gd¬«      \  }}|j                  |d|| j                  «      }|\  }}t%        ||||«      \  }} |j&                  g |j                   d d ¢d‘­Ž }t        j(                  ||fd¬«      }t        j(                  ||fd¬«      }|� |j+                  ||| j,                  «      \  }}t/        | j0                  «      rH| j                  | j                  k7  r/t3        j4                  |d| j                  | j                  z
  g«      }t7        j8                  | j0                  j:                  t<        «      } || ||||f| j>                  sdn| j@                  | jB                  dœ|¤Ž\  }}t/        | j0                  «      r4| j                  | j                  k7  r|d d …d d …d d …d | j                  …f   }|jE                  ||d«      jG                  «       }| jI                  |«      }||fS )Néÿÿÿÿé   r   )Údimr   g        )ÚdropoutÚscaling)%ÚshapeÚqk_head_dimr7   r:   r5   Úq_projÚq_b_projÚq_a_layernormÚq_a_projÚviewÚ	transposeÚtorchÚsplitr9   Úkv_a_proj_with_mqar4   Ú	kv_b_projÚkv_a_layernormr   ÚexpandÚcatÚupdateÚ	layer_idxr   ÚconfigÚFÚpadr
   Úget_interfaceÚ_attn_implementationr   ÚtrainingÚattention_dropoutrq   ÚreshapeÚ
contiguousÚo_proj)rF   rh   ri   rj   rk   rG   Ú
batch_sizeÚ
seq_lengthÚquery_shapeÚ	key_shapeÚq_statesÚq_passÚq_rotÚcompressed_kvÚk_passÚk_rotÚvalue_statesÚcosÚsinÚquery_statesÚ
key_statesÚattention_interfaceÚattn_outputÚattn_weightss                           rI   ÚforwardzMiniCPM3Attention.forward£   s$  € ð "/×!4Ñ!4°S°bÐ!9Ñˆ
�JØ! :¨r°4×3CÑ3CÐDˆØ ¨R°×1FÑ1FÈÏÉÑ1XÐYˆ	à×ÑÐ#Ø—{‘{ =Ó1‰Hà—}‘} T×%7Ñ%7¸¿¹ÀmÓ8TÓ%UÓVˆHØ—=‘= Ó-×7Ñ7¸¸1Ó=ˆÜŸ™ H¨t×/DÑ/DÀd×F[ÑF[Ð.\ÐbdÔe‰ˆ�à×/Ñ/°Ó>ˆÜŸ™ M°D×4EÑ4EÀt×G\ÑG\Ð3]ÐceÔf‰ˆ�à—‘ × 3Ñ 3°FÓ ;Ó<×AÑAÀ)ÓL×VÑVÐWXÐZ[Ó\ˆÜ$Ÿ{™{¨6°D×4IÑ4IÈ4Ï?É?Ð3[ÐacÔdÑˆ�à—
‘
˜: q¨*°d×6KÑ6KÓLˆà&‰ˆˆSô ,¨E°5¸#¸sÓC‰ˆˆuØ�—‘Ð4˜fŸl™l¨3¨BÐ/Ð4°Ò4ˆä—y‘y &¨% °bÔ9ˆÜ—Y‘Y ¨˜°BÔ7ˆ
àÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä'¨¯©Ô4¸×9IÑ9IÈTÏ_É_Ò9\ÜŸ5™5 °°4×3CÑ3CÀdÇoÁoÑ3UÐ/VÓWˆLä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ô (¨¯©Ô4¸×9IÑ9IÈTÏ_É_Ò9\Ø%¢aªªAÐ/@°·±Ð/@Ð&@ÑAˆKà!×)Ñ)¨*°jÀ"ÓE×PÑPÓRˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(rJ   )NN)	rO   rP   rQ   rR   rz   ÚTensorÚtupler   rŸ   r@   rJ   rI   rg   rg   ›   s~   „ ñð /3Ø(,ñ>)à—|‘|ð>)ð # 5§<¡<°·±Ð#=Ñ>ð>)ð Ÿ™ tÑ+ð	>)ð
  ™ð>)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1ô>)rJ   rg   c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
eej                  ej                  f   dz  dee   dej                  fd„Zˆ xZS )ÚMiniCPM3DecoderLayerrƒ   r‚   c                 óˆ   •— t         ‰| �  ||«       |j                  t        j                  |j
                  «      z  | _        y rM   )rD   Ú__init__r=   rB   rC   r+   Úresidual_scale)rF   rƒ   r‚   rH   s      €rI   r¥   zMiniCPM3DecoderLayer.__init__å   s6   ø€ Ü‰Ñ˜ Ô+ð %×0Ñ0´4·9±9¸V×=UÑ=UÓ3VÑVˆÕrJ   Nrh   rj   Úposition_idsrk   Ú	use_cacheri   rG   rK   c           
      óô   — |}| j                  |«      } | j                  d||||||dœ|¤Ž\  }}	||| j                  z  z   }|}| j                  |«      }| j	                  |«      }||| j                  z  z   }|S )N)rh   rj   r§   rk   r¨   ri   r@   )Úinput_layernormÚ	self_attnr¦   Úpost_attention_layernormÚmlp)
rF   rh   rj   r§   rk   r¨   ri   rG   ÚresidualÚ_s
             rI   rŸ   zMiniCPM3DecoderLayer.forwardë   s¤   € ð !ˆØ×,Ñ,¨]Ó;ˆØ)˜4Ÿ>™>ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
Ñˆ�qð ! =°4×3FÑ3FÑ#FÑFˆà ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ  =°4×3FÑ3FÑ#FÑFˆØÐrJ   )NNNFN)rO   rP   rQ   r"   rU   r¥   rz   r    Ú
LongTensorr   rX   r¡   r   r   rŸ   rZ   r[   s   @rI   r£   r£   ä   s¾   ø„ ðW˜~ð W¸#õ Wð /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð+Ñ,ðð 
�‰÷rJ   r£   c                   ó:   — e Zd Z ej                  «       d„ «       Zy)ÚMiniCPM3PreTrainedModelc                 ó¦   — t        j                  | |«       t        |t        «      r+t	        j
                  |j                  |j                  «       y y rM   )r   Ú_init_weightsÚ
isinstancer]   ÚinitÚ	constant_Úembed_scaleÚscalar_embed_scale)rF   Úmodules     rI   r´   z%MiniCPM3PreTrainedModel._init_weights
  s<   € ä×%Ñ% d¨FÔ3Ü�fÔ9Ô:Ü�N‰N˜6×-Ñ-¨v×/HÑ/HÕIð ;rJ   N)rO   rP   rQ   rz   Úno_gradr´   r@   rJ   rI   r²   r²   	  s   „ Ø€U‡]�]ƒ_ñJó ñJrJ   r²   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚMiniCPM3Modelrƒ   c                 óœ   •— t         ‰| �  |«       t        |j                  |j                  | j
                  |j                  ¬«      | _        y )N)r¸   )rD   r¥   r]   r'   r(   Úpadding_idxr<   Úembed_tokens)rF   rƒ   rH   s     €rI   r¥   zMiniCPM3Model.__init__  s?   ø€ Ü‰Ñ˜Ô ä7Ø×Ñ˜v×1Ñ1°4×3CÑ3CÐQW×QaÑQaô
ˆÕrJ   )rO   rP   rQ   r"   r¥   rZ   r[   s   @rI   r½   r½     s   ø„ ð
˜~÷ 
ñ 
rJ   r½   c                   ó  — e Zd Zee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  dej                  dz  de
dz  d	eej                  z  d
ee   defd„«       «       Zy)ÚMiniCPM3ForCausalLMNÚ	input_idsrj   r§   rk   Úinputs_embedsÚlabelsr¨   Úlogits_to_keeprG   rK   c	           
      óª  —  | j                   d||||||dœ|	¤Ž}
|
j                  }|| j                  j                  z  }t	        |t
        «      rt        | d«      n|}| j                  |dd…|dd…f   «      }d}|�* | j                  d||| j                  j                  dœ|	¤Ž}t        |||
j                  |
j                  |
j                  ¬«      S )a´  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, MiniCPM3ForCausalLM

        >>> model = MiniCPM3ForCausalLM.from_pretrained("openbmb/MiniCPM3-4B")
        >>> tokenizer = AutoTokenizer.from_pretrained("openbmb/MiniCPM3-4B")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)rÃ   rj   r§   rk   rÄ   r¨   N)ÚlogitsrÅ   r'   )ÚlossrÈ   rk   rh   Ú
attentionsr@   )ÚmodelÚlast_hidden_staterƒ   rN   rµ   rU   ÚsliceÚlm_headÚloss_functionr'   r	   rk   rh   rÊ   )rF   rÃ   rj   r§   rk   rÄ   rÅ   r¨   rÆ   rG   Úoutputsrh   Úslice_indicesrÈ   rÉ   s                  rI   rŸ   zMiniCPM3ForCausalLM.forward  sí   € ð< ,6¨4¯:©:ð ,
ØØ)Ø%Ø+Ø'Øñ,
ð ñ,
ˆð  ×1Ñ1ˆà%¨¯©×(BÑ(BÑBˆÜ8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ%Ðp¨V¸FÈtÏ{É{×OeÑOeÑpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
rJ   )NNNNNNNr   )rO   rP   rQ   r   r   rz   r°   r    r   ÚFloatTensorrX   rU   r   r   r	   rŸ   r@   rJ   rI   rÂ   rÂ     sä   „ àØð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ6
à×#Ñ# dÑ*ð6
ð Ÿ™ tÑ+ð6
ð ×&Ñ&¨Ñ-ð	6
ð
  ™ð6
ð ×(Ñ(¨4Ñ/ð6
ð × Ñ  4Ñ'ð6
ð ˜$‘;ð6
ð ˜eŸl™lÑ*ð6
ð Ð+Ñ,ð6
ð 
 ò6
ó ó ñ6
rJ   rÂ   c                   ó   — e Zd Zy)Ú!MiniCPM3ForSequenceClassificationNr^   r@   rJ   rI   rÔ   rÔ   X  r_   rJ   rÔ   )r"   r²   r½   rÂ   rÔ   )>rB   Úcollections.abcr   rz   Útorch.nn.functionalÚnnÚ
functionalr„   Úhuggingface_hub.dataclassesr   Ú r   r¶   Úcache_utilsr   Úmodeling_outputsr   r	   Úmodeling_utilsr
   r   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   Ú deepseek_v2.modeling_deepseek_v2r   Úgemma3.modeling_gemma3r   Úllama.configuration_llamar   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   r   Ú
get_loggerrO   Úloggerr"   r]   ra   rc   re   rg   r£   r²   r½   rÂ   rÔ   Ú__all__r@   rJ   rI   ú<module>rè      s5  ðó Ý $ã ß Ð Ý .å &Ý  ß Oß FÝ &ß RÓ RÝ 9Ý BÝ BÝ 3÷÷ ÷ ð 
ˆ×	Ñ	˜HÓ	%€ñ Ð0Ô1ØôU6�[ó U6ó ó 2ðU6ôp	Ð"?ô 	ô	�lô 	ô	Ð2ô 	ô	�(ô 	ôF)Ð+ô F)ôR"Ð,ô "ôJJÐ2ô Jð ô
�Jó 
ó ð
ð ô9
Ð*ó 9
ó ð9
ôx	Ð(Fô 	ò�rJ   