Ë
    ÜÓ^j   ã                   óz   — d dl Z d dlmZ ddlmZ ddlmZ ddlmZ  ed¬«      e G d	„ d
e«      «       «       Z	d
gZ
y)é    N)Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringzopenbmb/MiniCPM3-4B)Ú
checkpointc            	       óˆ  ‡ — e Zd ZU dZdZdgZdddddddddœZdgd	gfd
dgd
gfd
gd
gfdœZdZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	dz  e
d<   dZee
d<   dZe	e
d<   dZee
d<   dZee
d <   d!Zee
d"<   dZe	dz  e
d#<   d$Ze	dz  e
d%<   d&Ze	ee	   z  dz  e
d'<   d$Ze	dz  e
d(<   d!Zee
d)<   dZee z  dz  e
d*<   d+Z!ee
d,<   d-Z"e	ez  dz  e
d.<   d+Z#ee
d/<   dZ$e	dz  e
d0<   d1Z%e	e
d2<   d3Z&e	dz  e
d4<   d5Z'e	e
d6<   d7Z(e	e
d8<   dZ)e	dz  e
d9<   d:Z*e	ez  e
d;<   d<Z+e	ez  dz  e
d=<   d1Z,e	dz  e
d><   ˆ fd?„Z-d@„ Z.e/dAefdB„«       Z0ˆ xZ1S )CÚMiniCPM3Configa‰  
    kv_lora_rank (`int`, *optional*, defaults to 256):
        Rank of the low-rank KV projection in multi-head latent attention.
    q_lora_rank (`int`, *optional*, defaults to 768):
        Rank of the low-rank query projection in multi-head latent attention. If `None`, the query projection
        is a single dense projection rather than a low-rank one.
    qk_nope_head_dim (`int`, *optional*, defaults to 64):
        Dimension of the non-RoPE part of each query/key head.
    qk_rope_head_dim (`int`, *optional*, defaults to 32):
        Dimension of the RoPE part of each query/key head.
    v_head_dim (`int`, *optional*):
        Dimension of each value head. If `None`, defaults to `hidden_size // num_attention_heads`.
    scale_emb (`int` or `float`, *optional*, defaults to 12):
        Multiplier applied to input embeddings.
    scale_depth (`int` or `float`, *optional*, defaults to 1.4):
        Multiplier for residual connections; the effective scaling is `scale_depth / sqrt(num_hidden_layers)`.
        If `None`, defaults to `sqrt(num_hidden_layers)` (no-op scaling).
    dim_model_base (`int`, *optional*, defaults to 256):
        Base model dimension used to scale logits before the language model head. If `None`,
        defaults to `hidden_size` (no-op scaling).

    Example:

    ```python
    >>> from transformers import MiniCPM3Model, MiniCPM3Config
    >>> # Initializing a MiniCPM3 style configuration
    >>> configuration = MiniCPM3Config()
    >>> # Initializing a model from the configuration
    >>> model = MiniCPM3Model(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    Úminicpm3Úpast_key_valuesÚcolwiseÚmla_kv_a_projÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.q_b_projz%layers.*.self_attn.kv_a_proj_with_mqazlayers.*.self_attn.kv_b_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormiè Ú
vocab_sizei 
  Úhidden_sizei   Úintermediate_sizeé>   Únum_hidden_layersé(   Únum_attention_headsNÚnum_key_value_headsÚsiluÚ
hidden_acti €  Úmax_position_embeddingsgš™™™™™¹?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsTÚ	use_cacheÚpad_token_idé   Úbos_token_idé   Úeos_token_idÚpretraining_tpÚtie_word_embeddingsÚrope_parametersFÚattention_biasg        Úattention_dropoutÚmlp_biasÚhead_dimé   Úkv_lora_ranki   Úq_lora_ranké@   Úqk_nope_head_dimé    Úqk_rope_head_dimÚ
v_head_dimé   Ú	scale_embgffffffö?Úscale_depthÚdim_model_basec                 óÂ  •— | j                   | _        | j                  €| j                  | j                  z  | _        | j
                  €$t        j                  | j                  «      | _        | j                  €| j                  | _	        | j                  €| j                  | j                  z  | _        | j                  €| j                  | _
        t        ‰| �0  di |¤Ž y )N© )r7   r0   r8   r   r   r;   ÚmathÚsqrtr   r<   r   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €ú~/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/minicpm3/configuration_minicpm3.pyrB   zMiniCPM3Config.__post_init__v   s½   ø€ à×-Ñ-ˆŒð �?‰?Ð"Ø"×.Ñ.°$×2JÑ2JÑJˆDŒOØ×ÑÐ#Ü#Ÿy™y¨×)?Ñ)?Ó@ˆDÔØ×ÑÐ&Ø"&×"2Ñ"2ˆDÔØ�=‰=Ð Ø ×,Ñ,°×0HÑ0HÑHˆDŒMØ×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$ä‰ÑÑ' Ó'ó    c                 óˆ   — | j                   | j                  z  dk7  r&t        d| j                   › d| j                  › d�«      ‚y)zOPart of `@strict`-powered validation. Validates the architecture of the config.r   zThe hidden size (z6) is not a multiple of the number of attention heads (z).N)r   r   Ú
ValueError©rC   s    rF   Úvalidate_architecturez$MiniCPM3Config.validate_architectureˆ   sS   € à×Ñ˜d×6Ñ6Ñ6¸!Ò;ÜØ# D×$4Ñ$4Ð#5ð 6Ø×2Ñ2Ð3°2ð7óð ð <rG   Úreturnc                 ó4   — | j                   | j                  z  S )N)r   r<   rJ   s    rF   Úlogits_scalingzMiniCPM3Config.logits_scaling�   s   € ð ×Ñ $×"5Ñ"5Ñ5Ð5rG   )2Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planr   ÚintÚ__annotations__r   r   r   r   r   r    Ústrr!   r"   Úfloatr#   r$   Úboolr%   r'   r)   Úlistr*   r+   r,   r   Údictr-   r.   r/   r0   r2   r3   r5   r7   r8   r:   r;   r<   rB   rK   ÚpropertyrN   Ú__classcell__)rE   s   @rF   r
   r
      s  ø… ñ ðD €JØ#4Ð"5Ðð &/Ø'0Ø1@Ø(1Ø%.Ø"+Ø )Ø"+ñ	Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð €J�ÓØ€K�ÓØ!Ð�sÓ!ØÐ�sÓØ!Ð˜Ó!Ø&(Ð˜˜t™Ó(Ø€J�ÓØ#(Ð˜SÓ(Ø"Ð�uÓ"Ø€L�%ÓØ€IˆtÓØ#€L�#˜‘*Ó#Ø €L�#˜‘*Ó Ø+,€L�#˜˜S™	‘/ DÑ(Ó,Ø!"€N�C˜$‘JÓ"Ø $Ð˜Ó$Ø48€O�^ dÑ*¨TÑ1Ó8Ø €N�DÓ Ø,/Ð�s˜U‘{ TÑ)Ó/Ø€HˆdÓØ€Hˆc�D‰jÓØ€L�#ÓØ!€K��t‘Ó!ØÐ�cÓØÐ�cÓØ!€J��d‘
Ó!Ø€Iˆs�U‰{ÓØ&)€K��u‘˜tÑ#Ó)Ø!$€N�C˜$‘JÓ$ô(ò$ð ð6 ò 6ó ô6rG   r
   )r?   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr   r
   Ú__all__r>   rG   rF   ú<module>re      sN   ðó* å .å 3Ý 1Ý #ñ Ð0Ô1Øôs6Ð%ó s6ó ó 2ðs6ðl Ð
�rG   