
    ^j(7                     N   d dl Z d dlmZ d dlZd dlmc mZ d dlm	Z	 ddl
mZ ddlmZ ddlmZmZ ddlmZmZ dd	lmZ dd
lmZmZmZmZ ddlmZ ddlmZ ddl m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.  ej^                  e0      Z1 ed      e	 G d de#                    Z2 G d de!      Z3 G d de+      Z4 G d de,      Z5 G d de(      Z6 G d de      Z7 G d d e%      Z8 G d! d"e*      Z9e G d# d$e)             Z:e G d% d&e&             Z; G d' d(e'      Z<g d)Z=y)*    N)Callable)strict   )initialization)Cache)BaseModelOutputWithPastCausalLMOutputWithPast)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)is_flash_attention_requested   )DeepseekV2Attention)Gemma3TextScaledWordEmbedding)LlamaConfig)
LlamaDecoderLayerLlamaForCausalLMLlamaForSequenceClassificationLlamaMLP
LlamaModelLlamaPreTrainedModelLlamaRMSNormLlamaRotaryEmbeddingapply_rotary_pos_embeager_attention_forwardzopenbmb/MiniCPM3-4B)
checkpointc            	       z    e Zd ZU dZdddddddddZdZdZeed<   d	Z	eed
<   dZ
eed<   dZeed<   dZeed<   dZedz  ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZeed<   dZedz  ed<   dZeed <   d!Zeed"<   dZedz  ed#<   d$Zeez  ed%<   d&Zeez  dz  ed'<   dZedz  ed(<    fd)Zed*efd+       Z xZS ),MiniCPM3Configa  
    kv_lora_rank (`int`, *optional*, defaults to 256):
        Rank of the low-rank KV projection in multi-head latent attention.
    q_lora_rank (`int`, *optional*, defaults to 768):
        Rank of the low-rank query projection in multi-head latent attention. If `None`, the query projection
        is a single dense projection rather than a low-rank one.
    qk_nope_head_dim (`int`, *optional*, defaults to 64):
        Dimension of the non-RoPE part of each query/key head.
    qk_rope_head_dim (`int`, *optional*, defaults to 32):
        Dimension of the RoPE part of each query/key head.
    v_head_dim (`int`, *optional*):
        Dimension of each value head. If `None`, defaults to `hidden_size // num_attention_heads`.
    scale_emb (`int` or `float`, *optional*, defaults to 12):
        Multiplier applied to input embeddings.
    scale_depth (`int` or `float`, *optional*, defaults to 1.4):
        Multiplier for residual connections; the effective scaling is `scale_depth / sqrt(num_hidden_layers)`.
        If `None`, defaults to `sqrt(num_hidden_layers)` (no-op scaling).
    dim_model_base (`int`, *optional*, defaults to 256):
        Base model dimension used to scale logits before the language model head. If `None`,
        defaults to `hidden_size` (no-op scaling).

    Example:

    ```python
    >>> from transformers import MiniCPM3Model, MiniCPM3Config
    >>> # Initializing a MiniCPM3 style configuration
    >>> configuration = MiniCPM3Config()
    >>> # Initializing a model from the configuration
    >>> model = MiniCPM3Model(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    colwisemla_kv_a_projrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.q_b_projz%layers.*.self_attn.kv_a_proj_with_mqazlayers.*.self_attn.kv_b_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projminicpm3i 
vocab_sizei 
  hidden_sizei   intermediate_size>   num_hidden_layers(   num_attention_headsNnum_key_value_headsi   max_position_embeddingsg?initializer_rangegh㈵>rms_norm_epsTtie_word_embeddings   kv_lora_ranki   q_lora_rank@   qk_nope_head_dim    qk_rope_head_dim
v_head_dim   	scale_embgffffff?scale_depthdim_model_basec                 4   | j                   | _        | j                  | j                  | j                  z  | _        | j
                  $t        j                  | j                        | _        | j                  | j                  | _	        t        | ,  di | y )N )r9   head_dimr:   r(   r-   r=   mathsqrtr+   r>   super__post_init__)selfkwargs	__class__s     x/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/minicpm3/modular_minicpm3.pyrE   zMiniCPM3Config.__post_init__x   s    -- ??""..$2J2JJDO##yy)?)?@D&"&"2"2D''    returnc                 4    | j                   | j                  z  S N)r(   r>   )rF   s    rI   logits_scalingzMiniCPM3Config.logits_scaling   s     $"5"555rJ   ) __name__
__module____qualname____doc__base_model_tp_plan
model_typer'   int__annotations__r(   r)   r+   r-   r.   r/   r0   floatr1   r2   boolr4   r5   r7   r9   r:   r<   r=   r>   rE   propertyrN   __classcell__rH   s   @rI   r"   r"   1   s.    F &/'01@(1%."+ )"+	 J JK!s!s!!&(t(#(S("u"L% $$L#!Kt!cc!Jd
!IsU{&)Kut#)!$NC$J$( 6 6 6rJ   r"   c                       e Zd Zy)MiniCPM3ScaledWordEmbeddingNrO   rP   rQ   r@   rJ   rI   r]   r]          rJ   r]   c                       e Zd Zy)MiniCPM3RMSNormNr^   r@   rJ   rI   ra   ra      r_   rJ   ra   c                       e Zd Zy)MiniCPM3RotaryEmbeddingNr^   r@   rJ   rI   rc   rc      r_   rJ   rc   c                       e Zd Zy)MiniCPM3MLPNr^   r@   rJ   rI   re   re      r_   rJ   re   c                       e Zd ZdZ	 	 d	dej
                  deej
                  ej
                  f   dej
                  dz  dedz  deej
                  ej
                  dz  f   f
dZy)
MiniCPM3AttentionaE  
    Multi-head Latent Attention (MLA), structurally identical to `DeepseekV2Attention`.
    The only difference is the rotary convention: MiniCPM3 keeps the original cos/sin RoPE
    (`apply_rotary_pos_emb`) instead of DeepSeek-V2's complex rotary, so we inherit the
    module construction and override only `forward`.
    Nhidden_statesposition_embeddingsattention_maskpast_key_valuesrK   c                    |j                   d d \  }}||d| j                  f}||d| j                  | j                  z   f}	| j                  | j                  |      }
n/| j                  | j                  | j                  |                  }
|
j                  |      j                  dd      }
t        j                  |
| j                  | j                  gd      \  }}| j                  |      }t        j                  || j                  | j                  gd      \  }}| j!                  | j#                  |            j                  |	      j                  dd      }t        j                  || j                  | j                  gd      \  }}|j                  |d|| j                        }|\  }}t%        ||||      \  }} |j&                  g |j                   d d d }t        j(                  ||fd      }t        j(                  ||fd      }| |j+                  ||| j,                        \  }}t/        | j0                        rH| j                  | j                  k7  r/t3        j4                  |d| j                  | j                  z
  g      }t7        j8                  | j0                  j:                  t<              } || ||||f| j>                  sdn| j@                  | jB                  d|\  }}t/        | j0                        r4| j                  | j                  k7  r|d d d d d d d | j                  f   }|jE                  ||d      jG                         }| jI                  |      }||fS )N   r   )dimr   g        )dropoutscaling)%shapeqk_head_dimr7   r:   r5   q_projq_b_projq_a_layernormq_a_projview	transposetorchsplitr9   kv_a_proj_with_mqar4   	kv_b_projkv_a_layernormr   expandcatupdate	layer_idxr   configFpadr
   get_interface_attn_implementationr   trainingattention_dropoutrq   reshape
contiguouso_proj)rF   rh   ri   rj   rk   rG   
batch_size
seq_lengthquery_shape	key_shapeq_statesq_passq_rotcompressed_kvk_passk_rotvalue_statescossinquery_states
key_statesattention_interfaceattn_outputattn_weightss                           rI   forwardzMiniCPM3Attention.forward   s$    "/!4!4Sb!9
J!:r43C3CDR1F1F1XY	#{{=1H}}T%7%7m8T%UVH==-771=Ht/D/DdF[F[.\bde//>MD4E4EtG\G\3]cef 3 3F ;<AA)LVVWXZ[\${{6D4I4I4??3[acd

:q*d6K6KL&S ,E5#sCu4fll3B/44yy&%b9YYB7
&'6'='=j,X\XfXf'g$J'49I9IT__9\5543C3Cdoo3U/VWL(?(M(MKK,,.E)
 %8	%
  $}}C$2H2HLL	%
 	%
!\ (49I9IT__9\%aA/@/@&@AK!))*j"EPPRkk+.L((rJ   )NN)	rO   rP   rQ   rR   rz   Tensortupler   r   r@   rJ   rI   rg   rg      s~     /3(,>)||>) #5<<#=>>) t+	>)
 >) 
u||U\\D00	1>)rJ   rg   c                       e Zd Zdedef fdZ	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
eej                  ej                  f   dz  dee   dej                  fdZ xZS )MiniCPM3DecoderLayerr   r   c                     t         |   ||       |j                  t        j                  |j
                        z  | _        y rM   )rD   __init__r=   rB   rC   r+   residual_scale)rF   r   r   rH   s      rI   r   zMiniCPM3DecoderLayer.__init__   s6    + %00499V=U=U3VVrJ   Nrh   rj   position_idsrk   	use_cacheri   rG   rK   c           
          |}| j                  |      } | j                  d||||||d|\  }}	||| j                  z  z   }|}| j                  |      }| j	                  |      }||| j                  z  z   }|S )N)rh   rj   r   rk   r   ri   r@   )input_layernorm	self_attnr   post_attention_layernormmlp)
rF   rh   rj   r   rk   r   ri   rG   residual_s
             rI   r   zMiniCPM3DecoderLayer.forward   s     !,,];)4>> 
')%+ 3
 
q !=43F3F#FF 55mD/ =43F3F#FFrJ   )NNNFN)rO   rP   rQ   r"   rU   r   rz   r   
LongTensorr   rX   r   r   r   r   rZ   r[   s   @rI   r   r      s    W~ W# W /304(,!&HL|| t+ &&-	
  $; #5<<#=>E +, 
rJ   r   c                   :    e Zd Z ej                         d        Zy)MiniCPM3PreTrainedModelc                     t        j                  | |       t        |t              r+t	        j
                  |j                  |j                         y y rM   )r   _init_weights
isinstancer]   init	constant_embed_scalescalar_embed_scale)rF   modules     rI   r   z%MiniCPM3PreTrainedModel._init_weights
  s<    %%dF3f9:NN6--v/H/HI ;rJ   N)rO   rP   rQ   rz   no_gradr   r@   rJ   rI   r   r   	  s    U]]_J JrJ   r   c                   $     e Zd Zdef fdZ xZS )MiniCPM3Modelr   c                     t         |   |       t        |j                  |j                  | j
                  |j                        | _        y )N)r   )rD   r   r]   r'   r(   padding_idxr<   embed_tokens)rF   r   rH   s     rI   r   zMiniCPM3Model.__init__  s?     7v1143C3CQWQaQa
rJ   )rO   rP   rQ   r"   r   rZ   r[   s   @rI   r   r     s    
~ 
 
rJ   r   c                      e Zd Zee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  dej                  dz  de
dz  d	eej                  z  d
ee   defd              Zy)MiniCPM3ForCausalLMN	input_idsrj   r   rk   inputs_embedslabelsr   logits_to_keeprG   rK   c	           
          | j                   d||||||d|	}
|
j                  }|| j                  j                  z  }t	        |t
              rt        | d      n|}| j                  |dd|ddf         }d}|* | j                  d||| j                  j                  d|	}t        |||
j                  |
j                  |
j                        S )a  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, MiniCPM3ForCausalLM

        >>> model = MiniCPM3ForCausalLM.from_pretrained("openbmb/MiniCPM3-4B")
        >>> tokenizer = AutoTokenizer.from_pretrained("openbmb/MiniCPM3-4B")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)r   rj   r   rk   r   r   N)logitsr   r'   )lossr   rk   rh   
attentionsr@   )modellast_hidden_stater   rN   r   rU   slicelm_headloss_functionr'   r	   rk   rh   r   )rF   r   rj   r   rk   r   r   r   r   rG   outputsrh   slice_indicesr   r   s                  rI   r   zMiniCPM3ForCausalLM.forward  s    < ,64:: ,
)%+',
 ,
  11%(B(BB8B>SV8W~ot4]kmA}a,?@A%4%%pVFt{{OeOepiopD%#33!//))
 	
rJ   )NNNNNNNr   )rO   rP   rQ   r   r   rz   r   r   r   FloatTensorrX   rU   r   r   r	   r   r@   rJ   rI   r   r     s     .2.204(,26*.!%-.6
##d*6
 t+6
 &&-	6

 6
 ((4/6
   4'6
 $;6
 ell*6
 +,6
 
 6
  6
rJ   r   c                       e Zd Zy)!MiniCPM3ForSequenceClassificationNr^   r@   rJ   rI   r   r   X  r_   rJ   r   )r"   r   r   r   r   )>rB   collections.abcr   rz   torch.nn.functionalnn
functionalr   huggingface_hub.dataclassesr    r   r   cache_utilsr   modeling_outputsr   r	   modeling_utilsr
   r   processing_utilsr   utilsr   r   r   r   utils.genericr    deepseek_v2.modeling_deepseek_v2r   gemma3.modeling_gemma3r   llama.configuration_llamar   llama.modeling_llamar   r   r   r   r   r   r   r   r   r   
get_loggerrO   loggerr"   r]   ra   rc   re   rg   r   r   r   r   r   __all__r@   rJ   rI   <module>r      s5    $    . &   O F & R R 9 B B 3   
		H	% 01U6[ U6  2U6p	"? 		l 		2 		( 	F)+ F)R", "JJ2 J 
J 
 
 9
* 9
 9
x	(F 	rJ   