Ë
    ÛÓ^j�"  ã                   óˆ  — d Z ddlZddlmZ ddlmZ ddlmZ ddlm	Z	 ddl
mZ ddlmZmZmZ d	d
lmZ d	dlmZmZmZmZmZmZmZ  ed¬«      e G d„ de«      «       «       Z G d„ de«      Z G d„ de«      Z G d„ de«      Z G d„ de«      Ze G d„ de«      «       Ze G d„ de«      «       Z e G d„ de«      «       Z!g d¢Z"y)z%HyperCLOVAX modular model definition.é    N)Ústricté   )ÚCache)ÚCausalLMOutputWithPast)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleé   )ÚGraniteConfig)ÚGraniteAttentionÚGraniteDecoderLayerÚGraniteForCausalLMÚGraniteModelÚGranitePreTrainedModelÚGraniteRMSNormÚGraniteRotaryEmbeddingz,naver-hyperclovax/HyperCLOVAX-SEED-Think-14B)Ú
checkpointc                   ód   ‡ — e Zd ZU dZdZdZedz  ed<   dZe	dz  ed<   dZ
eed<   ˆ fd„Zd	„ Zˆ xZS )
ÚHyperCLOVAXConfiga@  
    embedding_multiplier (`float`, *optional*, defaults to `1.0`):
        Scaling factor applied to the token embedding outputs. Used in MuP to control the
        scale of the embedding activations.
    logits_scaling (`float`, *optional*, defaults to `1.0`):
        Scaling factor **multiplied** to the final logits before loss computation or sampling.
        Used in MuP to ensure consistent output scale across model sizes. Note: unlike
        [`GraniteConfig`], this is a multiplier, not a divisor.
    residual_multiplier (`float`, *optional*, defaults to `1.0`):
        Scaling factor applied to each sub-layer output before adding to the residual stream.
        Used in Maximal Update Parametrization (MuP) to stabilize training across model sizes.
    attention_multiplier (`float`, *optional*, defaults to `head_dim ** -0.5`):
        Scaling factor applied to attention logits before softmax, replacing the standard
        `1 / sqrt(head_dim)` scaling. Set explicitly for MuP-based training; when `None`,
        defaults to the standard value.
    use_post_norm (`bool`, *optional*, defaults to `True`):
        Whether to apply an extra RMSNorm after each sub-layer output (Peri-Layer Normalization).

    ```python
    >>> from transformers import HyperCLOVAXModel, HyperCLOVAXConfig

    >>> # Initializing a HyperCLOVAX style configuration
    >>> configuration = HyperCLOVAXConfig()

    >>> # Initializing a model from the configuration
    >>> model = HyperCLOVAXModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```ÚhyperclovaxNÚhead_dimÚattention_multiplierTÚuse_post_normc                 óº   •— | j                   €| j                  | j                  z  | _         t        ‰| �  di |¤Ž | j
                  €| j                   dz  | _        y y )Ng      à¿© )r   Úhidden_sizeÚnum_attention_headsÚsuperÚ__post_init__r   )ÚselfÚkwargsÚ	__class__s     €ú~/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/hyperclovax/modular_hyperclovax.pyr    zHyperCLOVAXConfig.__post_init__P   s[   ø€ ð �=‰=Ð Ø ×,Ñ,°×0HÑ0HÑHˆDŒMä‰ÑÑ' Ò'ð ×$Ñ$Ð,Ø(,¯©°tÑ(;ˆDÕ%ð -ó    c                 óˆ   — | j                   | j                  z  dk7  r&t        d| j                   › d| j                  › d�«      ‚y)zCValidates that `hidden_size` is divisible by `num_attention_heads`.r   zThe hidden size (z6) is not a multiple of the number of attention heads (z).N)r   r   Ú
ValueError)r!   s    r$   Úvalidate_architecturez'HyperCLOVAXConfig.validate_architecture]   sS   € à×Ñ˜d×6Ñ6Ñ6¸!Ò;ÜØ# D×$4Ñ$4Ð#5ð 6Ø×2Ñ2Ð3°2ð7óð ð <r%   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typer   ÚintÚ__annotations__r   Úfloatr   Úboolr    r(   Ú__classcell__©r#   s   @r$   r   r   $   sD   ø… ñð> €Jà€Hˆc�D‰jÓð *.Ð˜% $™,Ó-ð €M�4Óô<ör%   r   c                   ó   — e Zd Zy)ÚHyperCLOVAXRMSNormN©r)   r*   r+   r   r%   r$   r5   r5   f   ó   „ Ør%   r5   c                   ó   — e Zd Zy)ÚHyperCLOVAXRotaryEmbeddingNr6   r   r%   r$   r9   r9   j   r7   r%   r9   c                   ó   — e Zd Zy)ÚHyperCLOVAXAttentionNr6   r   r%   r$   r;   r;   n   r7   r%   r;   c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
eej                  ej                  f   dz  dee   dej                  fd„Zˆ xZS )ÚHyperCLOVAXDecoderLayerÚconfigÚ	layer_idxc                 óJ  •— t         ‰| �  ||«       |j                  r!t        |j                  |j
                  ¬«      nt        j                  «       | _        |j                  r't        |j                  |j
                  ¬«      | _	        y t        j                  «       | _	        y )N)Úeps)
r   Ú__init__r   r5   r   Úrms_norm_epsÚnnÚIdentityÚ
post_norm1Ú
post_norm2)r!   r>   r?   r#   s      €r$   rB   z HyperCLOVAXDecoderLayer.__init__s   s…   ø€ Ü‰Ñ˜ Ô+ð PV×OcÒOcÔ˜v×1Ñ1°v×7JÑ7JÕKÔik×itÑitÓivð 	Œð PV×OcÒOcÔ˜v×1Ñ1°v×7JÑ7JÔKð 	�Üik×itÑitÓivð 	�r%   NÚhidden_statesÚattention_maskÚposition_idsÚpast_key_valuesÚ	use_cacheÚposition_embeddingsr"   Úreturnc           
      ó8  — |}| j                  |«      } | j                  d||||||dœ|¤Ž\  }}	| j                  |«      }||| j                  z  z   }|}| j	                  |«      }| j                  |«      }| j                  |«      }||| j                  z  z   }|S )N)rH   rI   rJ   rK   rL   rM   r   )Úinput_layernormÚ	self_attnrF   Úresidual_multiplierÚpost_attention_layernormÚmlprG   )
r!   rH   rI   rJ   rK   rL   rM   r"   ÚresidualÚ_s
             r$   ÚforwardzHyperCLOVAXDecoderLayer.forward}   sÂ   € ð !ˆØ×,Ñ,¨]Ó;ˆà)˜4Ÿ>™>ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
Ñˆ�qð Ÿ™¨Ó6ˆØ  =°4×3KÑ3KÑ#KÑKˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØŸ™¨Ó6ˆØ  =°4×3KÑ3KÑ#KÑKˆØÐr%   )NNNFN)r)   r*   r+   r   r.   rB   ÚtorchÚTensorÚ
LongTensorr   r1   Útupler   r   rW   r2   r3   s   @r$   r=   r=   r   s¼   ø„ ð
Ð0ð 
¸Sõ 
ð /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð+Ñ,ðð 
�‰÷r%   r=   c                   ó   — e Zd Zy)ÚHyperCLOVAXPreTrainedModelNr6   r   r%   r$   r]   r]   Ÿ   ó   „ àr%   r]   c                   ó   — e Zd Zy)ÚHyperCLOVAXModelNr6   r   r%   r$   r`   r`   ¤   r^   r%   r`   c                   ó  — e Zd Zee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  dej                  dz  de
dz  d	eej                  z  d
ee   defd„«       «       Zy)ÚHyperCLOVAXForCausalLMNÚ	input_idsrI   rJ   rK   Úinputs_embedsÚlabelsrL   Úlogits_to_keepr"   rN   c	           
      ó¦  —  | j                   d||||||dœ|	¤Ž}
|
j                  }t        |t        «      rt	        | d«      n|}| j                  |dd…|dd…f   «      | j                  j                  z  }d}|�* | j                  d||| j                  j                  dœ|	¤Ž}t        |||
j                  |
j                  |
j                  ¬«      S )a&  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, HyperCLOVAXForCausalLM

        >>> model = HyperCLOVAXForCausalLM.from_pretrained("naver-hyperclovax/HyperCLOVAX-SEED-Think-14B")
        >>> tokenizer = AutoTokenizer.from_pretrained("naver-hyperclovax/HyperCLOVAX-SEED-Think-14B")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me? Are you okay?" The man was confused and answered, "Yes." Then the woman asked.
        ```)rc   rI   rJ   rK   rd   rL   N)Úlogitsre   Ú
vocab_size)Úlossrh   rK   rH   Ú
attentionsr   )ÚmodelÚlast_hidden_stateÚ
isinstancer.   ÚsliceÚlm_headr>   Úlogits_scalingÚloss_functionri   r   rK   rH   rk   )r!   rc   rI   rJ   rK   rd   re   rL   rf   r"   ÚoutputsrH   Úslice_indicesrh   rj   s                  r$   rW   zHyperCLOVAXForCausalLM.forward«   sè   € ð> �$—*‘*ð 
ØØ)Ø%Ø+Ø'Øñ
ð ñ
ˆð  ×1Ñ1ˆÜ8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆà—‘˜mªA¨}ºaÐ,?Ñ@ÓAÀDÇKÁK×D^ÑD^Ñ^ˆàˆØÐØ%�4×%Ñ%Ðp¨V¸FÈtÏ{É{×OeÑOeÑpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r%   )NNNNNNNr   )r)   r*   r+   r
   r	   rX   rZ   rY   r   ÚFloatTensorr1   r.   r   r   r   rW   r   r%   r$   rb   rb   ©   sä   „ àØð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ6
à×#Ñ# dÑ*ð6
ð Ÿ™ tÑ+ð6
ð ×&Ñ&¨Ñ-ð	6
ð
  ™ð6
ð ×(Ñ(¨4Ñ/ð6
ð × Ñ  4Ñ'ð6
ð ˜$‘;ð6
ð ˜eŸl™lÑ*ð6
ð Ð+Ñ,ð6
ð 
 ò6
ó ó ñ6
r%   rb   )r   r]   r`   rb   )#r,   rX   Útorch.nnrD   Úhuggingface_hub.dataclassesr   Úcache_utilsr   Úmodeling_outputsr   Úprocessing_utilsr   Úutilsr   r	   r
   Úgranite.configuration_graniter   Úgranite.modeling_graniter   r   r   r   r   r   r   r   r5   r9   r;   r=   r]   r`   rb   Ú__all__r   r%   r$   ú<module>r      sî   ðñ ,ã Ý Ý .å  Ý 6Ý &ß IÑ IÝ 9÷÷ ñ ñ ÐIÔJØô=˜ó =ó ó Kð=ô@	˜ô 	ô	Ð!7ô 	ô	Ð+ô 	ô*Ð1ô *ðZ ô	Ð!7ó 	ó ð	ð ô	�|ó 	ó ð	ð ô9
Ð/ó 9
ó ð9
òx�r%   