
    ^j                        d dl Z d dlmZ d dl mZ ddlmZ ddlmZ ddl	m
Z
mZ dd	lmZ dd
lmZ ddlmZmZmZmZmZmZ ddlmZ  ej2                  e      Z e
d      e G d de                    Z G d de      Z G d de      Z G d de      Z G d de      Z  G d de      Z! G d dee      Z" G d de      Z# G d d e      Z$g d!Z%y)"    N)strict)nn   )initialization)PreTrainedModel)auto_docstringlogging   )DeepseekV3Config)DeepseekV3Attention)LlamaDecoderLayerLlamaForCausalLM
LlamaModelLlamaPreTrainedModelLlamaRMSNormLlamaRotaryEmbedding)Qwen3MLPztencent/Youtu-LLM-2B)
checkpointc                       e Zd ZU dZdZddddZi ZdZee	d<   dZ
ee	d	<   d
Zee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZedz  e	d<   dZedz  e	d<   dZedz  e	d<   dZedz  e	d<   dZeee   z  dz  e	d<   dZee	d<    e       Z e       Z e       Z e       Z e       Z e       Z e       Z  e       Z! e       Z" e       Z# e       Z$ fdZ% xZ&S )YoutuConfiga   
    rope_interleave (`bool`, *optional*, defaults to `True`):
        Whether to interleave the rotary position embeddings.
    embedding_initializer_range (`float`, *optional*):
        The standard deviation of the truncated_normal_initializer for initializing all embedding matrices.

    ```python
    >>> from transformers import YoutuModel, YoutuConfig
    >>> # Initializing a Youtu-LLM-2B style configuration
    >>> configuration = YoutuConfig()
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```youtucolwiserowwise)zlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proji  
vocab_sizei   hidden_sizei   intermediate_size    num_hidden_layers   num_attention_headsnum_key_value_headsi   max_position_embeddingsNinitializer_rangeembedding_initializer_rangepad_token_idi  bos_token_idi eos_token_idTtie_word_embeddingsc                     | j                   1| j                  dk7  rdd| j                  z  dz  z  | _         nd| _         | j                  xs d| j                   z  | _        t        |   di | y )Nr   g       @g      @g      ?{Gz? )r#   r   r$   super__post_init__)selfkwargs	__class__s     r/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/youtu/modular_youtu.pyr-   zYoutuConfig.__post_init__`   sq    !!)1$),d6F6F0F3/N)N&)-&+/+K+K+ksUYUkUkOk(''    )'__name__
__module____qualname____doc__
model_typebase_model_tp_planattribute_mapr   int__annotations__r   r   r   r    r!   r"   r#   floatr$   r%   r&   r'   listr(   boolAttributeErrorn_shared_expertsn_routed_expertsrouted_scaling_factorn_group
topk_groupnum_experts_per_tokfirst_k_dense_replacenorm_topk_probpretraining_tpmoe_intermediate_sizenum_mtp_layersr-   __classcell__)r0   s   @r1   r   r   ,   s8    J"+ )"+
 MJK!s!s!!!!#)S)&*ut|*044#L#*#%L#*%+1L#S	/D(1 $$ &'%'*,G!J(**,#%N#%N*,#%N( (r2   r   c                       e Zd Zy)YoutuRMSNormNr3   r4   r5   r+   r2   r1   rM   rM   k       r2   rM   c                       e Zd Zy)YoutuRotaryEmbeddingNrN   r+   r2   r1   rQ   rQ   o   rO   r2   rQ   c                       e Zd Zy)YoutuMLPNrN   r+   r2   r1   rS   rS   s   rO   r2   rS   c                       e Zd Zy)YoutuAttentionNrN   r+   r2   r1   rU   rU   w   rO   r2   rU   c                       e Zd Zy)YoutuDecoderLayerNrN   r+   r2   r1   rW   rW   {   rO   r2   rW   c                   :    e Zd Z ej                         d        Zy)YoutuPreTrainedModelc                    t        j                  | |       t        | j                  dd      }t        | j                  dd|z        }t	        |t
        j                        rft        j                  |j                  d|       |j                  7t        j                  |j                  j                  |j                            y y y )Nr#   r*   r$   r
   g        )meanstd)r   _init_weightsgetattrconfig
isinstancer   	Embeddinginitnormal_weightpadding_idxzeros_data)r.   moduler\   	embed_stds       r1   r]   z"YoutuPreTrainedModel._init_weights   s    %%dF3dkk#6=DKK)FCP	fbll+LLSi@!!-FMM..v/A/ABC . ,r2   N)r3   r4   r5   torchno_gradr]   r+   r2   r1   rY   rY      s    U]]_D Dr2   rY   c                       e Zd Zy)
YoutuModelNrN   r+   r2   r1   rm   rm      rO   r2   rm   c                       e Zd Zy)YoutuForCausalLMNrN   r+   r2   r1   ro   ro      rO   r2   ro   )r   rY   rm   ro   )&rj   huggingface_hub.dataclassesr   r    r   rb   modeling_utilsr   utilsr   r	   %deepseek_v3.configuration_deepseek_v3r    deepseek_v3.modeling_deepseek_v3r   llama.modeling_llamar   r   r   r   r   r   qwen3.modeling_qwen3r   
get_loggerr3   loggerr   rM   rQ   rS   rU   rW   rY   rm   ro   __all__r+   r2   r1   <module>r{      s   *  .  & - , D B  , 
		H	% 12:(" :(  3:(z	< 		/ 		x 		( 		) 		D/ 	D	 		' 	r2   