
    ^j                      z    d dl Z d dlmZ ddlmZ ddlmZ ddlmZ  ed      e G d	 d
e                    Z	d
gZ
y)    N)strict   )PreTrainedConfig)RopeParameters)auto_docstringzopenbmb/MiniCPM3-4B)
checkpointc            	           e Zd ZU dZdZdgZdddddddddZdgd	gfd
dgd
gfd
gd
gfdZdZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	dz  e
d<   dZee
d<   dZe	e
d<   dZee
d<   dZee
d <   d!Zee
d"<   dZe	dz  e
d#<   d$Ze	dz  e
d%<   d&Ze	ee	   z  dz  e
d'<   d$Ze	dz  e
d(<   d!Zee
d)<   dZee z  dz  e
d*<   d+Z!ee
d,<   d-Z"e	ez  dz  e
d.<   d+Z#ee
d/<   dZ$e	dz  e
d0<   d1Z%e	e
d2<   d3Z&e	dz  e
d4<   d5Z'e	e
d6<   d7Z(e	e
d8<   dZ)e	dz  e
d9<   d:Z*e	ez  e
d;<   d<Z+e	ez  dz  e
d=<   d1Z,e	dz  e
d><    fd?Z-d@ Z.e/dAefdB       Z0 xZ1S )CMiniCPM3Configa  
    kv_lora_rank (`int`, *optional*, defaults to 256):
        Rank of the low-rank KV projection in multi-head latent attention.
    q_lora_rank (`int`, *optional*, defaults to 768):
        Rank of the low-rank query projection in multi-head latent attention. If `None`, the query projection
        is a single dense projection rather than a low-rank one.
    qk_nope_head_dim (`int`, *optional*, defaults to 64):
        Dimension of the non-RoPE part of each query/key head.
    qk_rope_head_dim (`int`, *optional*, defaults to 32):
        Dimension of the RoPE part of each query/key head.
    v_head_dim (`int`, *optional*):
        Dimension of each value head. If `None`, defaults to `hidden_size // num_attention_heads`.
    scale_emb (`int` or `float`, *optional*, defaults to 12):
        Multiplier applied to input embeddings.
    scale_depth (`int` or `float`, *optional*, defaults to 1.4):
        Multiplier for residual connections; the effective scaling is `scale_depth / sqrt(num_hidden_layers)`.
        If `None`, defaults to `sqrt(num_hidden_layers)` (no-op scaling).
    dim_model_base (`int`, *optional*, defaults to 256):
        Base model dimension used to scale logits before the language model head. If `None`,
        defaults to `hidden_size` (no-op scaling).

    Example:

    ```python
    >>> from transformers import MiniCPM3Model, MiniCPM3Config
    >>> # Initializing a MiniCPM3 style configuration
    >>> configuration = MiniCPM3Config()
    >>> # Initializing a model from the configuration
    >>> model = MiniCPM3Model(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    minicpm3past_key_valuescolwisemla_kv_a_projrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.q_b_projz%layers.*.self_attn.kv_a_proj_with_mqazlayers.*.self_attn.kv_b_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proj	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnormi 
vocab_sizei 
  hidden_sizei   intermediate_size>   num_hidden_layers(   num_attention_headsNnum_key_value_headssilu
hidden_acti   max_position_embeddingsg?initializer_rangegh㈵>rms_norm_epsT	use_cachepad_token_id   bos_token_id   eos_token_idpretraining_tptie_word_embeddingsrope_parametersFattention_biasg        attention_dropoutmlp_biashead_dim   kv_lora_ranki   q_lora_rank@   qk_nope_head_dim    qk_rope_head_dim
v_head_dim   	scale_embgffffff?scale_depthdim_model_basec                    | j                   | _        | j                  | j                  | j                  z  | _        | j
                  $t        j                  | j                        | _        | j                  | j                  | _	        | j                  | j                  | j                  z  | _        | j                  | j                  | _
        t        | 0  di | y )N )r7   r0   r8   r   r   r;   mathsqrtr   r<   r   super__post_init__)selfkwargs	__class__s     ~/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/minicpm3/configuration_minicpm3.pyrB   zMiniCPM3Config.__post_init__v   s    -- ??""..$2J2JJDO##yy)?)?@D&"&"2"2D==  ,,0H0HHDM##+'+'?'?D$''    c                     | j                   | j                  z  dk7  r&t        d| j                    d| j                   d      y)zOPart of `@strict`-powered validation. Validates the architecture of the config.r   zThe hidden size (z6) is not a multiple of the number of attention heads (z).N)r   r   
ValueErrorrC   s    rF   validate_architecturez$MiniCPM3Config.validate_architecture   sS    d666!;#D$4$4#5 622327  <rG   returnc                 4    | j                   | j                  z  S )N)r   r<   rJ   s    rF   logits_scalingzMiniCPM3Config.logits_scaling   s     $"5"555rG   )2__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planr   int__annotations__r   r   r   r   r   r    strr!   r"   floatr#   r$   boolr%   r'   r)   listr*   r+   r,   r   dictr-   r.   r/   r0   r2   r3   r5   r7   r8   r:   r;   r<   rB   rK   propertyrN   __classcell__)rE   s   @rF   r
   r
      s    D J#4"5 &/'01@(1%."+ )"+	 &(9:#%568IJ!"_$56 JK!s!s!!&(t(J#(S("u"L%It#L#*# L#* +,L#S	/D(,!"NC$J" $$48O^d*T18 ND ,/sU{T)/HdHcDjL#!Kt!cc!Jd
!IsU{&)Kut#)!$NC$J$($ 6 6 6rG   r
   )r?   huggingface_hub.dataclassesr   configuration_utilsr   modeling_rope_utilsr   utilsr   r
   __all__r>   rG   rF   <module>re      sN   *  . 3 1 # 01s6% s6  2s6l 
rG   