
    ^jZ"                        d dl mZ d dlZd dlmZ d dlmZ d dlmZ ddl	m
Z
 ddlmZ ddlmZ dd	lmZmZ d
dlmZmZmZ d
dlmZ d
dlmZmZmZmZmZmZmZ  ej@                  e!      Z" ed      e G d de                    Z# G d de      Z$ G d de      Z%d Z& G d de      Z' G d de      Z( G d de      Z) G d de      Z* G d d e      Z+ G d! d"e      Z,g d#Z-y)$    )CallableN)strict)TransformersKwargs   )Cache)ALL_ATTENTION_FUNCTIONS)Unpack)auto_docstringlogging   )LlamaPreTrainedModelLlamaRMSNormeager_attention_forward)
OlmoConfig)OlmoAttentionOlmoDecoderLayerOlmoForCausalLMOlmoForSequenceClassification	OlmoModelOlmoRotaryEmbeddingapply_rotary_pos_embzallenai/Olmo2-7B-1124-hf)
checkpointc                   l    e Zd ZU dZdZddddddddZdgd	gfd
dgd
gfd
gd
gfdZdZee	d<    e
       Zy)Olmo2Configa  
    Example:

    ```python
    >>> from transformers import Olmo2Model, Olmo2Config

    >>> # Initializing a Olmo2 7B style configuration
    >>> configuration = Olmo2Config()

    >>> # Initializing a model from the Olmo2 7B style configuration
    >>> model = Olmo2Model(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    olmo2colwise_gather_outputrowwise_split_inputcolwiserowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proj	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnormgh㈵>rms_norm_epsN)__name__
__module____qualname____doc__
model_typebase_model_tp_planbase_model_pp_planr'   float__annotations__AttributeErrorclip_qkv     r/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/olmo2/modular_olmo2.pyr   r   0   su    " J%<%<%<%:"+ )"+ &(9:#%568IJ!"_$56 L%Hr4   r   c                       e Zd Zd Zy)Olmo2RMSNormc                 "   |j                   }|j                  t        j                        }|j	                  d      j                  dd      }|t        j                  || j                  z         z  }| j                  |z  j                  |      S )Nr   T)keepdim)	dtypetotorchfloat32powmeanrsqrtvariance_epsilonweight)selfr"   input_dtypevariances       r5   forwardzOlmo2RMSNorm.forward[   sy    #))%((7 $$Q',,R,>%Ht?T?T4T(UUm+//<<r4   N)r(   r)   r*   rG   r3   r4   r5   r7   r7   Z   s    =r4   r7   c                       e Zd Zy)Olmo2RotaryEmbeddingNr(   r)   r*   r3   r4   r5   rI   rI   c       r4   rI   c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..Nr9   r   )dim)shaper=   cat)xx1x2s      r5   rotate_halfrS   g   sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r4   c                        e Zd Zddededz  f fdZ	 ddej                  deej                  ej                  f   dej                  dz  de	dz  d	e
e   d
eej                  ej                  dz  f   fdZ xZS )Olmo2AttentionNconfig	layer_idxc                     t         |   ||       t        |j                  | j                  z  |j
                        | _        t        |j                  | j                  z  |j
                        | _        y )NrW   )	super__init__r7   num_attention_headshead_dimr'   q_normnum_key_value_headsk_normrD   rV   rW   	__class__s      r5   r[   zOlmo2Attention.__init__r   s[    95"6#=#=#MvObObc"6#=#=#MvObObcr4   r"   position_embeddingsr#   past_key_valueskwargsreturnc                 R   |j                   d d }g |d| j                  }| j                  | j                  |            }| j	                  | j                  |            }	| j                  |      }
|j                  |      j                  dd      }|	j                  |      j                  dd      }	|
j                  |      j                  dd      }
|\  }}t        ||	||      \  }}	| |j                  |	|
| j                        \  }	}
t        j                  | j                  j                  t               } || ||	|
|f| j"                  sdn| j$                  | j&                  d|\  }} |j(                  g |d j+                         }| j-                  |      }||fS )Nr9      r   g        )dropoutscaling)rN   r]   r^   q_projr`   k_projv_projview	transposer   updaterW   r   get_interfacerV   _attn_implementationr   trainingattention_dropoutrj   reshape
contiguouso_proj)rD   r"   rc   r#   rd   re   input_shapehidden_shapequery_states
key_statesvalue_statescossinattention_interfaceattn_outputattn_weightss                   r5   rG   zOlmo2Attention.forwardw   s    $))#2.88b8$--8{{4;;}#=>[[]!;<
{{=1#((6@@AF__\2<<QB
#((6@@AF&S#7jRUWZ#[ j&'6'='=j,X\XfXf'g$J(?(M(MKK,,.E)
 %8	%
  $}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((r4   )N)r(   r)   r*   r   intr[   r=   Tensortupler   r	   r   rG   __classcell__rb   s   @r5   rU   rU   q   s    d{ dsTz d )-*)||*) #5<<#=>*) t+	*)
 *) +,*) 
u||U\\D00	1*)r4   rU   c                       e Zd Zdedef fdZ	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
eej                  ej                  f   dz  dee   dej                  fdZ xZS )Olmo2DecoderLayerrV   rW   c                     t         |   ||       t        |j                  |j                        | _        t        |j                  |j                        | _        t        ||      | _        | `	y )NrY   eps)rV   rW   )
rZ   r[   r7   hidden_sizer'   post_attention_layernormpost_feedforward_layernormrU   	self_attninput_layernormra   s      r5   r[   zOlmo2DecoderLayer.__init__   s_    95(4V5G5GVM`M`(a%*6v7I7IvObOb*c''vK r4   Nr"   r#   position_idsrd   	use_cacherc   re   rf   c           
          |} | j                   d||||||d|\  }}	| j                  |      }||z   }|}| j                  |      }| j                  |      }||z   }|S )N)r"   r#   r   rd   r   rc   r3   )r   r   mlpr   )
rD   r"   r#   r   rd   r   rc   re   residual_s
             r5   rG   zOlmo2DecoderLayer.forward   s     !)4>> 
')%+ 3
 
q 55mD =0 !/77F =0r4   )NNNFN)r(   r)   r*   r   r   r[   r=   r   
LongTensorr   boolr   r	   r   rG   r   r   s   @r5   r   r      s    !{ !s ! /304(,!&HL|| t+ &&-	
  $; #5<<#=>E +, 
r4   r   c                       e Zd Zy)Olmo2PreTrainedModelNrJ   r3   r4   r5   r   r      rK   r4   r   c                   $     e Zd Zdef fdZ xZS )
Olmo2ModelrV   c           	         t         |   |       t        |j                  |j                        | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        y c c}w )Nr   )rZ   r[   r7   r   r'   r&   nn
ModuleListrangenum_hidden_layersr   r%   ra   s      r5   r[   zOlmo2Model.__init__   s^      !3!39L9LM	mmCHIaIaCbcivy1c
cs   A=)r(   r)   r*   r   r[   r   r   s   @r5   r   r      s    
{ 
 
r4   r   c                       e Zd Zy)Olmo2ForCausalLMNrJ   r3   r4   r5   r   r      rK   r4   r   c                       e Zd Zy)Olmo2ForSequenceClassificationNrJ   r3   r4   r5   r   r      rK   r4   r   )r   r   r   r   r   ).collections.abcr   r=   torch.nnr   huggingface_hub.dataclassesr   transformers.utils.genericr   cache_utilsr   modeling_utilsr   processing_utilsr	   utilsr
   r   llama.modeling_llamar   r   r   olmo.configuration_olmor   olmo.modeling_olmor   r   r   r   r   r   r   
get_loggerr(   loggerr   r7   rI   rS   rU   r   r   r   r   r   __all__r3   r4   r5   <module>r      s   ( %   . 9   5 & , ^ ^ 0   
		H	% 56# * #   7# P=< =	. 	(0)] 0)l$( $N	/ 	
 
	 		%B 	r4   