
    ^j"                        d Z ddlZddlmZ ddlmZ ddlmZ ddlm	Z	 ddl
mZ ddlmZmZmZ d	d
lmZ d	dlmZmZmZmZmZmZmZ  ed      e G d de                    Z G d de      Z G d de      Z G d de      Z G d de      Ze G d de             Ze G d de             Z e G d de             Z!g dZ"y)z%HyperCLOVAX modular model definition.    N)strict   )Cache)CausalLMOutputWithPast)Unpack)TransformersKwargsauto_docstringcan_return_tuple   )GraniteConfig)GraniteAttentionGraniteDecoderLayerGraniteForCausalLMGraniteModelGranitePreTrainedModelGraniteRMSNormGraniteRotaryEmbeddingz,naver-hyperclovax/HyperCLOVAX-SEED-Think-14B)
checkpointc                   d     e Zd ZU dZdZdZedz  ed<   dZe	dz  ed<   dZ
eed<    fdZd	 Z xZS )
HyperCLOVAXConfiga@  
    embedding_multiplier (`float`, *optional*, defaults to `1.0`):
        Scaling factor applied to the token embedding outputs. Used in MuP to control the
        scale of the embedding activations.
    logits_scaling (`float`, *optional*, defaults to `1.0`):
        Scaling factor **multiplied** to the final logits before loss computation or sampling.
        Used in MuP to ensure consistent output scale across model sizes. Note: unlike
        [`GraniteConfig`], this is a multiplier, not a divisor.
    residual_multiplier (`float`, *optional*, defaults to `1.0`):
        Scaling factor applied to each sub-layer output before adding to the residual stream.
        Used in Maximal Update Parametrization (MuP) to stabilize training across model sizes.
    attention_multiplier (`float`, *optional*, defaults to `head_dim ** -0.5`):
        Scaling factor applied to attention logits before softmax, replacing the standard
        `1 / sqrt(head_dim)` scaling. Set explicitly for MuP-based training; when `None`,
        defaults to the standard value.
    use_post_norm (`bool`, *optional*, defaults to `True`):
        Whether to apply an extra RMSNorm after each sub-layer output (Peri-Layer Normalization).

    ```python
    >>> from transformers import HyperCLOVAXModel, HyperCLOVAXConfig

    >>> # Initializing a HyperCLOVAX style configuration
    >>> configuration = HyperCLOVAXConfig()

    >>> # Initializing a model from the configuration
    >>> model = HyperCLOVAXModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```hyperclovaxNhead_dimattention_multiplierTuse_post_normc                     | j                   | j                  | j                  z  | _         t        |   di | | j
                  | j                   dz  | _        y y )Ng      ࿩ )r   hidden_sizenum_attention_headssuper__post_init__r   )selfkwargs	__class__s     ~/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/hyperclovax/modular_hyperclovax.pyr    zHyperCLOVAXConfig.__post_init__P   s[     ==  ,,0H0HHDM'' $$,(,t(;D% -    c                     | j                   | j                  z  dk7  r&t        d| j                    d| j                   d      y)zCValidates that `hidden_size` is divisible by `num_attention_heads`.r   zThe hidden size (z6) is not a multiple of the number of attention heads (z).N)r   r   
ValueError)r!   s    r$   validate_architecturez'HyperCLOVAXConfig.validate_architecture]   sS    d666!;#D$4$4#5 622327  <r%   )__name__
__module____qualname____doc__
model_typer   int__annotations__r   floatr   boolr    r(   __classcell__r#   s   @r$   r   r   $   sD    > JHcDj *.%$,- M4<r%   r   c                       e Zd Zy)HyperCLOVAXRMSNormNr)   r*   r+   r   r%   r$   r5   r5   f       r%   r5   c                       e Zd Zy)HyperCLOVAXRotaryEmbeddingNr6   r   r%   r$   r9   r9   j   r7   r%   r9   c                       e Zd Zy)HyperCLOVAXAttentionNr6   r   r%   r$   r;   r;   n   r7   r%   r;   c                       e Zd Zdedef fdZ	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
eej                  ej                  f   dz  dee   dej                  fdZ xZS )HyperCLOVAXDecoderLayerconfig	layer_idxc                 J   t         |   ||       |j                  r!t        |j                  |j
                        nt        j                         | _        |j                  r't        |j                  |j
                        | _	        y t        j                         | _	        y )N)eps)
r   __init__r   r5   r   rms_norm_epsnnIdentity
post_norm1
post_norm2)r!   r>   r?   r#   s      r$   rB   z HyperCLOVAXDecoderLayer.__init__s   s    + PVOcOcv11v7J7JKikititiv 	 PVOcOcv11v7J7JK 	ikititiv 	r%   Nhidden_statesattention_maskposition_idspast_key_values	use_cacheposition_embeddingsr"   returnc           
      8   |}| j                  |      } | j                  d||||||d|\  }}	| j                  |      }||| j                  z  z   }|}| j	                  |      }| j                  |      }| j                  |      }||| j                  z  z   }|S )N)rH   rI   rJ   rK   rL   rM   r   )input_layernorm	self_attnrF   residual_multiplierpost_attention_layernormmlprG   )
r!   rH   rI   rJ   rK   rL   rM   r"   residual_s
             r$   forwardzHyperCLOVAXDecoderLayer.forward}   s     !,,];)4>> 
')%+ 3
 
q 6 =43K3K#KK !55mD/6 =43K3K#KKr%   )NNNFN)r)   r*   r+   r   r.   rB   torchTensor
LongTensorr   r1   tupler   r   rW   r2   r3   s   @r$   r=   r=   r   s    
0 
S 
 /304(,!&HL|| t+ &&-	
  $; #5<<#=>E +, 
r%   r=   c                       e Zd Zy)HyperCLOVAXPreTrainedModelNr6   r   r%   r$   r]   r]          r%   r]   c                       e Zd Zy)HyperCLOVAXModelNr6   r   r%   r$   r`   r`      r^   r%   r`   c                      e Zd Zee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  dej                  dz  de
dz  d	eej                  z  d
ee   defd              Zy)HyperCLOVAXForCausalLMN	input_idsrI   rJ   rK   inputs_embedslabelsrL   logits_to_keepr"   rN   c	           
          | j                   d||||||d|	}
|
j                  }t        |t              rt	        | d      n|}| j                  |dd|ddf         | j                  j                  z  }d}|* | j                  d||| j                  j                  d|	}t        |||
j                  |
j                  |
j                        S )a&  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, HyperCLOVAXForCausalLM

        >>> model = HyperCLOVAXForCausalLM.from_pretrained("naver-hyperclovax/HyperCLOVAX-SEED-Think-14B")
        >>> tokenizer = AutoTokenizer.from_pretrained("naver-hyperclovax/HyperCLOVAX-SEED-Think-14B")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me? Are you okay?" The man was confused and answered, "Yes." Then the woman asked.
        ```)rc   rI   rJ   rK   rd   rL   N)logitsre   
vocab_size)lossrh   rK   rH   
attentionsr   )modellast_hidden_state
isinstancer.   slicelm_headr>   logits_scalingloss_functionri   r   rK   rH   rk   )r!   rc   rI   rJ   rK   rd   re   rL   rf   r"   outputsrH   slice_indicesrh   rj   s                  r$   rW   zHyperCLOVAXForCausalLM.forward   s    > $** 
)%+'
 
  118B>SV8W~ot4]kmA}a,?@ADKKD^D^^%4%%pVFt{{OeOepiopD%#33!//))
 	
r%   )NNNNNNNr   )r)   r*   r+   r
   r	   rX   rZ   rY   r   FloatTensorr1   r.   r   r   r   rW   r   r%   r$   rb   rb      s     .2.204(,26*.!%-.6
##d*6
 t+6
 &&-	6

 6
 ((4/6
   4'6
 $;6
 ell*6
 +,6
 
 6
  6
r%   rb   )r   r]   r`   rb   )#r,   rX   torch.nnrD   huggingface_hub.dataclassesr   cache_utilsr   modeling_outputsr   processing_utilsr   utilsr   r	   r
   granite.configuration_graniter   granite.modeling_graniter   r   r   r   r   r   r   r   r5   r9   r;   r=   r]   r`   rb   __all__r   r%   r$   <module>r      s    ,   .   6 & I I 9   IJ= =  K=@	 		!7 		+ 	*1 *Z 	!7 	 	 	| 	 	 9
/ 9
 9
xr%   