
    ^j                        d dl mZ d dlZd dlmZ d dlmc mZ ddlm	Z	 ddl
mZ ddlmZ ddlmZ ddlmZ dd	lmZ d
dlmZmZmZmZmZmZmZmZmZ ddlmZ  ej@                  e!      Z" G d dejF                        Z$ G d de      Z% G d de      Z& ed      d!d       Z' G d de      Z( G d de      Z) G d de      Z* G d de      Z+ G d de      Z,g d Z-y)"    )CallableN   )Cache)use_kernel_func_from_hub)dynamic_rope_update)ALL_ATTENTION_FUNCTIONS)logging)maybe_autocast   )	LlamaAttentionLlamaDecoderLayerLlamaForCausalLMLlamaForSequenceClassificationLlamaMLP
LlamaModelLlamaRotaryEmbeddingeager_attention_forwardrotate_half   )
OlmoConfigc                   d     e Zd ZdZdeddf fdZdej                  dej                  fdZ xZ	S )OlmoLayerNormz/LayerNorm but with no learnable weight or bias.hidden_sizereturnNc                 2    t         |           |f| _        y N)super__init__normalized_shape)selfr   	__class__s     p/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/olmo/modular_olmo.pyr   zOlmoLayerNorm.__init__4   s    !,    hidden_statesc                     |j                   }t        j                  |j                  t        j
                        | j                  d d d      j                  |      S )N)dtypegh㈵>)eps)r&   F
layer_normtotorchfloat32r   )r    r$   
orig_dtypes      r"   forwardzOlmoLayerNorm.forward8   sO    "((
||M,,5==,A4CXCXZ^`djnorr
 	
r#   )
__name__
__module____qualname____doc__intr   r+   Tensorr.   __classcell__r!   s   @r"   r   r   1   s4    9/C /D /
U\\ 
ell 
r#   r   c                        e Zd Z fdZ xZS )OlmoMLPc                 J   t         |   |       t        j                  | j                  | j
                  d      | _        t        j                  | j                  | j
                  d      | _        t        j                  | j
                  | j                  d      | _        y )NF)bias)	r   r   nnLinearr   intermediate_size	gate_projup_proj	down_proj)r    configr!   s     r"   r   zOlmoMLP.__init__@   ss     4#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXr#   )r/   r0   r1   r   r5   r6   s   @r"   r8   r8   ?   s    Y Yr#   r8   c                   D    e Zd Z ej                         ed               Zy)OlmoRotaryEmbeddingc                    | j                   d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d      5  |j                         |j                         z  j                  dd      }t        j                  ||fd	      }|j                         | j                  z  }|j                         | j                  z  }	d d d        ||	fS # 1 sw Y   	fS xY w)
Nr   r   mpscpuF)device_typeenabledr   )dim)inv_freqfloatexpandshaper*   device
isinstancetypestrr
   	transposer+   catcosattention_scalingsin)
r    xposition_idsinv_freq_expandedposition_ids_expandedrH   freqsembrU   rW   s
             r"   r.   zOlmoRotaryEmbedding.forwardJ   s8    !MM$4-8>>@GGHZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfkUC 	5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C		5
 Cx	5
 Cxs   BE''E3N)r/   r0   r1   r+   no_gradr   r.    r#   r"   rC   rC   I   s$    U]]_
  
r#   rC   rotary_pos_embc                 
   | j                   |j                   }}|j                  |      }|j                  |      }| |z  t        |       |z  z   }||z  t        |      |z  z   }|j                  |      |j                  |      fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )r&   	unsqueezer   r*   )	qkrU   rW   unsqueeze_dimq_typek_typeq_embedk_embeds	            r"   apply_rotary_pos_embrj   Y   s|    & WWaggFF
--
&C
--
&C3w;q>C/0G3w;q>C/0G::fwzz&111r#   c                       e Zd Z	 ddej                  deej                  ej                  f   dej                  dz  dedz  deej                  ej                  dz  f   f
dZy)	OlmoAttentionNr$   position_embeddingsattention_maskpast_key_valuesr   c                    |j                   d d }g |d| j                  }| j                  |      }| j                  |      }	| j	                  |      }
| j
                  j                  |j                  | j
                  j                   | j
                  j                         |	j                  | j
                  j                   | j
                  j                         |
j                  | j
                  j                   | j
                  j                         |j                  |      j                  dd      }|	j                  |      j                  dd      }	|
j                  |      j                  dd      }
|\  }}t        ||	||      \  }}	| |j                  |	|
| j                        \  }	}
t        j                  | j
                  j                  t               } || ||	|
|f| j"                  sdn| j$                  | j&                  d|\  }} |j(                  g |d j+                         }| j-                  |      }||fS )NrE   )minmaxr   r   g        )dropoutscaling)rN   head_dimq_projk_projv_projrA   clip_qkvclamp_viewrS   rj   update	layer_idxr   get_interface_attn_implementationr   trainingattention_dropoutrt   reshape
contiguouso_proj)r    r$   rm   rn   ro   kwargsinput_shapehidden_shapequery_states
key_statesvalue_statesrU   rW   attention_interfaceattn_outputattn_weightss                   r"   r.   zOlmoAttention.forwardu   s)    $))#2.88b8$--8{{=1[[/
{{=1;;+T[[%9%9$9t{{?S?ST4;;#7#7"7T[[=Q=QRT[[%9%9$9t{{?S?ST#((6@@AF__\2<<QB
#((6@@AF&S#7jRUWZ#[ j&'6'='=j,X\XfXf'g$J(?(M(MKK,,.E)
 %8	%
  $}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((r#   r   )r/   r0   r1   r+   r4   tupler   r.   r_   r#   r"   rl   rl   t   sv     )-/)||/) #5<<#=>/) t+	/)
 /) 
u||U\\D00	1/)r#   rl   c                   (     e Zd Zdedef fdZ xZS )OlmoDecoderLayerrA   r}   c                     t         |   ||       t        |j                        | _        t        |j                        | _        t        ||      | _        y )N)rA   r}   )r   r   r   r   input_layernormpost_attention_layernormrl   	self_attnr    rA   r}   r!   s      r"   r   zOlmoDecoderLayer.__init__   sF    +,V-?-?@(5f6H6H(I%&f	Jr#   )r/   r0   r1   r   r3   r   r5   r6   s   @r"   r   r      s    Kz Kc K Kr#   r   c                   $     e Zd Zdef fdZ xZS )	OlmoModelrA   c           	          t         |   |       t        j                  t	        |j
                        D cg c]  }t        ||       c}      | _        t        |j                        | _
        y c c}w r   )r   r   r;   
ModuleListrangenum_hidden_layersr   layersr   r   normr   s      r"   r   zOlmoModel.__init__   s[     mmBGH`H`BabYfi0b
 "&"4"45	 cs   A1)r/   r0   r1   r   r   r5   r6   s   @r"   r   r      s    6z 6 6r#   r   c                       e Zd Zy)OlmoForCausalLMNr/   r0   r1   r_   r#   r"   r   r          r#   r   c                       e Zd Zy)OlmoForSequenceClassificationNr   r_   r#   r"   r   r      r   r#   r   )r   r   r   OlmoPreTrainedModel)r   ).collections.abcr   r+   torch.nnr;   torch.nn.functional
functionalr(   cache_utilsr   integrationsr   modeling_rope_utilsr   modeling_utilsr   utilsr	   utils.genericr
   llama.modeling_llamar   r   r   r   r   r   r   r   r   configuration_olmor   
get_loggerr/   loggerModuler   r8   rC   rj   rl   r   r   r   r   __all__r_   r#   r"   <module>r      s   ( %       4 6 5  +
 
 
 + 
		H	%
BII 
Yh Y.   *+2 ,240)N 0)fK( K6
 6	& 		$B 	r#   