Ë
    ÜÓ^jÛ  ã                   óÄ  — d dl mZ d dlZd dlmZ d dlmc mZ ddlm	Z	 ddl
mZ ddlmZ ddlmZ ddlmZ dd	lmZ d
dlmZmZmZmZmZmZmZmZmZ ddlmZ  ej@                  e!«      Z" G d„ dejF                  «      Z$ G d„ de«      Z% G d„ de«      Z& ed«      d!d„«       Z' G d„ de«      Z( G d„ de«      Z) G d„ de«      Z* G d„ de«      Z+ G d„ de«      Z,g d ¢Z-y)"é    )ÚCallableNé   )ÚCache)Úuse_kernel_func_from_hub)Údynamic_rope_update)ÚALL_ATTENTION_FUNCTIONS)Úlogging)Úmaybe_autocasté   )	ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaForCausalLMÚLlamaForSequenceClassificationÚLlamaMLPÚ
LlamaModelÚLlamaRotaryEmbeddingÚeager_attention_forwardÚrotate_halfé   )Ú
OlmoConfigc                   ód   ‡ — e Zd ZdZdeddfˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚOlmoLayerNormz/LayerNorm but with no learnable weight or bias.Úhidden_sizeÚreturnNc                 ó2   •— t         ‰| �  «        |f| _        y ©N)ÚsuperÚ__init__Únormalized_shape)Úselfr   Ú	__class__s     €úp/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/olmo/modular_olmo.pyr   zOlmoLayerNorm.__init__4   s   ø€ Ü‰ÑÔØ!, ˆÕó    Úhidden_statesc                 ó¼   — |j                   }t        j                  |j                  t        j
                  ¬«      | j                  d d d¬«      j                  |«      S )N)Údtypegñhãˆµøä>)Úeps)r&   ÚFÚ
layer_normÚtoÚtorchÚfloat32r   )r    r$   Ú
orig_dtypes      r"   ÚforwardzOlmoLayerNorm.forward8   sO   € Ø"×(Ñ(ˆ
Ü�|‰|˜M×,Ñ,´5·=±=Ð,ÓAÀ4×CXÑCXÐZ^Ð`dÐjnÔo×rÑrØó
ð 	
r#   )
Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úintr   r+   ÚTensorr.   Ú__classcell__©r!   s   @r"   r   r   1   s4   ø„ Ù9ð/ Cð /¨Dõ /ð
 U§\¡\ð 
°e·l±l÷ 
r#   r   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚOlmoMLPc                 óJ  •— t         ‰| �  |«       t        j                  | j                  | j
                  d¬«      | _        t        j                  | j                  | j
                  d¬«      | _        t        j                  | j
                  | j                  d¬«      | _        y )NF)Úbias)	r   r   ÚnnÚLinearr   Úintermediate_sizeÚ	gate_projÚup_projÚ	down_proj)r    Úconfigr!   s     €r"   r   zOlmoMLP.__init__@   ss   ø€ Ü‰Ñ˜Ô ÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆ�r#   )r/   r0   r1   r   r5   r6   s   @r"   r8   r8   ?   s   ø„ ÷Yð Yr#   r8   c                   óD   — e Zd Z ej                  «       ed„ «       «       Zy)ÚOlmoRotaryEmbeddingc                 óì  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  ||fd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       ||	fS # 1 sw Y   	fS xY w)
Nr   éÿÿÿÿr   ÚmpsÚcpuF)Údevice_typeÚenabledr   )Údim)Úinv_freqÚfloatÚexpandÚshaper*   ÚdeviceÚ
isinstanceÚtypeÚstrr
   Ú	transposer+   ÚcatÚcosÚattention_scalingÚsin)
r    ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrH   ÚfreqsÚembrU   rW   s
             r"   r.   zOlmoRotaryEmbedding.forwardJ   s8  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð
 �Cˆxˆ÷	5ð
 �Cˆxˆús   ÃBE'Å'E3N)r/   r0   r1   r+   Úno_gradr   r.   © r#   r"   rC   rC   I   s$   „ Ø€U‡]�]ƒ_Øñ
ó ó ñ
r#   rC   Úrotary_pos_embc                 ó
  — | j                   |j                   }}|j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }|j                  |«      |j                  |«      fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )r&   Ú	unsqueezer   r*   )	ÚqÚkrU   rW   Úunsqueeze_dimÚq_typeÚk_typeÚq_embedÚk_embeds	            r"   Úapply_rotary_pos_embrj   Y   s|   € ð& —W‘W˜aŸg™gˆF€FØ
�-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�:‰:�fÓ˜wŸz™z¨&Ó1Ð1Ð1r#   c                   óÊ   — e Zd Z	 ddej                  deej                  ej                  f   dej                  dz  dedz  deej                  ej                  dz  f   f
d„Zy)	ÚOlmoAttentionNr$   Úposition_embeddingsÚattention_maskÚpast_key_valuesr   c                 óª  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      }| j                  |«      }	| j	                  |«      }
| j
                  j                  �´|j                  | j
                  j                   | j
                  j                  ¬«       |	j                  | j
                  j                   | j
                  j                  ¬«       |
j                  | j
                  j                   | j
                  j                  ¬«       |j                  |«      j                  dd«      }|	j                  |«      j                  dd«      }	|
j                  |«      j                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j
                  j                  t         «      } || ||	|
|f| j"                  sdn| j$                  | j&                  dœ|¤Ž\  }} |j(                  g |¢d‘­Ž j+                  «       }| j-                  |«      }||fS )NrE   )ÚminÚmaxr   r   g        )ÚdropoutÚscaling)rN   Úhead_dimÚq_projÚk_projÚv_projrA   Úclip_qkvÚclamp_ÚviewrS   rj   ÚupdateÚ	layer_idxr   Úget_interfaceÚ_attn_implementationr   ÚtrainingÚattention_dropoutrt   ÚreshapeÚ
contiguousÚo_proj)r    r$   rm   rn   ro   ÚkwargsÚinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesrU   rW   Úattention_interfaceÚattn_outputÚattn_weightss                   r"   r.   zOlmoAttention.forwardu   s)  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1ˆØ—[‘[ Ó/ˆ
Ø—{‘{ =Ó1ˆà�;‰;×ÑÐ+Ø×Ñ T§[¡[×%9Ñ%9Ð$9¸t¿{¹{×?SÑ?SÐÔTØ×Ñ 4§;¡;×#7Ñ#7Ð"7¸T¿[¹[×=QÑ=QÐÔRØ×Ñ T§[¡[×%9Ñ%9Ð$9¸t¿{¹{×?SÑ?SÐÔTà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r#   r   )r/   r0   r1   r+   r4   Útupler   r.   r_   r#   r"   rl   rl   t   sv   „ ð )-ñ/)à—|‘|ð/)ð # 5§<¡<°·±Ð#=Ñ>ð/)ð Ÿ™ tÑ+ð	/)ð
  ™ð/)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1ô/)r#   rl   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚOlmoDecoderLayerrA   r}   c                 ó²   •— t         ‰| �  ||«       t        |j                  «      | _        t        |j                  «      | _        t        ||¬«      | _        y )N)rA   r}   )r   r   r   r   Úinput_layernormÚpost_attention_layernormrl   Ú	self_attn©r    rA   r}   r!   s      €r"   r   zOlmoDecoderLayer.__init__¨   sF   ø€ Ü‰Ñ˜ Ô+Ü,¨V×-?Ñ-?Ó@ˆÔÜ(5°f×6HÑ6HÓ(IˆÔ%Ü&¨fÀ	ÔJˆ�r#   )r/   r0   r1   r   r3   r   r5   r6   s   @r"   r�   r�   §   s   ø„ ðK˜zð K°c÷ Kñ Kr#   r�   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )Ú	OlmoModelrA   c           	      óì   •— t         ‰| �  |«       t        j                  t	        |j
                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  «      | _
        y c c}w r   )r   r   r;   Ú
ModuleListÚrangeÚnum_hidden_layersr�   Úlayersr   r   Únormr•   s      €r"   r   zOlmoModel.__init__°   s[   ø€ Ü‰Ñ˜Ô Ü—m‘mÜBGÈ×H`ÑH`ÓBaÖb°YÔ˜f iÕ0Òbó
ˆŒô " &×"4Ñ"4Ó5ˆ�	ùò cs   ·A1)r/   r0   r1   r   r   r5   r6   s   @r"   r—   r—   ¯   s   ø„ ð6˜z÷ 6ñ 6r#   r—   c                   ó   — e Zd Zy)ÚOlmoForCausalLMN©r/   r0   r1   r_   r#   r"   rŸ   rŸ   ¸   ó   „ Ør#   rŸ   c                   ó   — e Zd Zy)ÚOlmoForSequenceClassificationNr    r_   r#   r"   r£   r£   ¼   r¡   r#   r£   )rŸ   r£   r—   ÚOlmoPreTrainedModel)r   ).Úcollections.abcr   r+   Útorch.nnr;   Útorch.nn.functionalÚ
functionalr(   Úcache_utilsr   Úintegrationsr   Úmodeling_rope_utilsr   Úmodeling_utilsr   Úutilsr	   Úutils.genericr
   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   Úconfiguration_olmor   Ú
get_loggerr/   ÚloggerÚModuler   r8   rC   rj   rl   r�   r—   rŸ   r£   Ú__all__r_   r#   r"   ú<module>rµ      sÞ   ðõ( %ã Ý ß Ð å  Ý 4Ý 6Ý 5Ý Ý +÷
÷ 
õ 
õ +ð 
ˆ×	Ñ	˜HÓ	%€ô
�B—I‘Iô 
ôYˆhô YôÐ.ô ñ  Ð*Ó+ò2ó ,ð2ô40)�Nô 0)ôfKÐ(ô Kô6�
ô 6ô	Ð&ô 	ô	Ð$Bô 	ò�r#   