Ë
    ÛÓ^jÕ  ã                   ó¶  — d dl Z d dlZd dlmZ ddlmZ ddlmZ ddlm	Z	m
Z
mZ ddlmZ ddlmZmZmZmZmZ d	d
lmZ  ej,                  e«      Z G d„ dej2                  «      Z G d„ de«      Z G d„ de«      Zd„ Zd"d„Ze G d„ de«      «       Z G d„ de«      Z  G d„ de«      Z! G d„ de!e«      Z" G d„ de	«      Z# G d„ de
«      Z$ G d„ d e«      Z%g d!¢Z&y)#é    Né   )Úlogging)Úno_inherit_decoratoré   )ÚGemmaForCausalLMÚGemmaForSequenceClassificationÚGemmaForTokenClassification)ÚGraniteAttention)ÚLlamaDecoderLayerÚLlamaMLPÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRotaryEmbeddingé   )ÚHeliumConfigc                   ó,   ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Zˆ xZS )ÚHeliumRMSNormc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y ©N)ÚsuperÚ__init__ÚnnÚ	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s      €út/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/helium/modular_helium.pyr   zHeliumRMSNorm.__init__    s/   ø€ Ü‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕó    c                 ó\  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  j                  t        j                  «      |z  j                  |«      S )Nr   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtor   Úfloat32ÚpowÚmeanÚrsqrtr   r   )r   Úhidden_statesÚinput_dtypeÚvariances       r"   ÚforwardzHeliumRMSNorm.forward%   s†   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ—‘—‘œuŸ}™}Ó-°Ñ=×AÑAÀ+ÓNÐNr#   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)Útupler   Úshaper   )r   s    r"   Ú
extra_reprzHeliumRMSNorm.extra_repr,   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr#   )g�íµ ÷Æ°>)Ú__name__Ú
__module__Ú__qualname__r   r0   r4   Ú__classcell__©r!   s   @r"   r   r      s   ø„ õ$ò
OöJr#   r   c                   ó   — e Zd Zy)ÚHeliumRotaryEmbeddingN©r5   r6   r7   © r#   r"   r;   r;   0   ó   „ Ør#   r;   c                   ó   — e Zd Zy)Ú	HeliumMLPNr<   r=   r#   r"   r@   r@   4   r>   r#   r@   c                 ó|   — | dddd…f   }| dddd…f   }t        j                  | |fd¬«      j                  d«      S )	z*Rotates half the hidden dims of the input..r   Nr   r   r%   ©Údiméþÿÿÿ)r   ÚstackÚflatten)ÚxÚx1Úx2s      r"   Úrotate_halfrJ   8   sJ   € à	
ˆ3���1�ˆ9‰€BØ	
ˆ3���1�ˆ9‰€BÜ�;‰;˜˜˜R�y bÔ)×1Ñ1°"Ó5Ð5r#   c                 óF  — |j                  |«      }|j                  |«      }|dd|j                  d   dz  …f   j                  dd¬«      }|dd|j                  d   dz  …f   j                  dd¬«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    .Nr%   r   rB   )Ú	unsqueezer3   Úrepeat_interleaverJ   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds          r"   Úapply_rotary_pos_embrU   ?   s¿   € ð$ �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cð ˆcÐ'�S—Y‘Y˜r‘] aÑ'Ð'Ð'Ñ
(×
:Ñ
:¸1À"Ð
:Ó
E€CØ
ˆcÐ'�S—Y‘Y˜r‘] aÑ'Ð'Ð'Ñ
(×
:Ñ
:¸1À"Ð
:Ó
E€Cà�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€Gà�GÐÐr#   c                   ó0   ‡ — e Zd Zddededz  fˆ fd„Zˆ xZS )ÚHeliumAttentionNÚconfigÚ	layer_idxc                 óÖ   •— t         ‰| �  ||«       t        j                  |j                  |j                  d¬«      | _        dt        j                  | j                  «      z  | _	        y )NF)Úbiasr   )
r   r   r   ÚLinearr   Úo_projÚmathÚsqrtÚhead_dimÚscaling©r   rX   rY   r!   s      €r"   r   zHeliumAttention.__init__`   sK   ø€ Ü‰Ñ˜ Ô+Ü—i‘i × 2Ñ 2°F×4FÑ4FÈUÔSˆŒØœ4Ÿ9™9 T§]¡]Ó3Ñ3ˆ�r#   r   ©r5   r6   r7   r   Úintr   r8   r9   s   @r"   rW   rW   ^   s   ø„ ñ4˜|ð 4¸¸d¹
÷ 4ñ 4r#   rW   c                   ó0   ‡ — e Zd Zddededz  fˆ fd„Zˆ xZS )ÚHeliumDecoderLayerNrX   rY   c                 óÞ   •— t         ‰| �  ||«       t        |«      | _        t	        |j
                  |j                  ¬«      | _        t	        |j
                  |j                  ¬«      | _        y )N©r    )	r   r   r@   Úmlpr   r   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormrb   s      €r"   r   zHeliumDecoderLayer.__init__g   sT   ø€ Ü‰Ñ˜ Ô+ä˜VÓ$ˆŒÜ,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔÜ(5°f×6HÑ6HÈf×NaÑNaÔ(bˆÕ%r#   r   rc   r9   s   @r"   rf   rf   f   s#   ø„ ñc˜|ð c¸¸d¹
÷ cñ cr#   rf   c                   ó   — e Zd Zy)ÚHeliumPreTrainedModelNr<   r=   r#   r"   rn   rn   o   r>   r#   rn   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚHeliumModelrX   c           	      ó2  •— t         ‰| �  |«       t        j                  t	        |j
                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                  ¬«      | _        d| _        | j                  «        y c c}w )Nrh   F)r   r   r   Ú
ModuleListÚrangeÚnum_hidden_layersrf   Úlayersr   r   rj   ÚnormÚgradient_checkpointingÚ	post_initrb   s      €r"   r   zHeliumModel.__init__t   sw   ø€ Ü‰Ñ˜Ô Ü—m‘mÜDIÈ&×JbÑJbÓDcÖd°yÔ ¨	Õ2Òdó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÔNˆŒ	Ø&+ˆÔ#ð 	�‰Õùò es   ·B)r5   r6   r7   r   r   r8   r9   s   @r"   rp   rp   s   s   ø„ ð	˜|÷ 	ñ 	r#   rp   c                   ó   — e Zd Zy)ÚHeliumForCausalLMNr<   r=   r#   r"   rz   rz   €   r>   r#   rz   c                   ó   — e Zd Zy)ÚHeliumForSequenceClassificationNr<   r=   r#   r"   r|   r|   „   r>   r#   r|   c                   ó   — e Zd Zy)ÚHeliumForTokenClassificationNr<   r=   r#   r"   r~   r~   ˆ   r>   r#   r~   )rn   rp   rz   r|   r~   )r   )'r^   r   Útorch.nnr   Úutilsr   Úutils.genericr   Úgemma.modeling_gemmar   r   r	   Úgranite.modeling_graniter
   Úllama.modeling_llamar   r   r   r   r   Úconfiguration_heliumr   Ú
get_loggerr5   ÚloggerÚModuler   r;   r@   rJ   rU   rW   rf   rn   rp   rz   r|   r~   Ú__all__r=   r#   r"   ú<module>rŠ      sæ   ðó ã Ý å Ý 1ß pÑ pÝ 7ß vÕ vÝ .ð 
ˆ×	Ñ	˜HÓ	%€ôJ�B—I‘Iô Jô"	Ð0ô 	ô	�ô 	ò6óð> ô4Ð&ó 4ó ð4ôcÐ*ô cô	Ð0ô 	ô
Ð'¨ô 
ô	Ð(ô 	ô	Ð&Dô 	ô	Ð#>ô 	ò�r#   