
    ^j2                        d dl mZ d dlmZ d dlZd dlmZ d dlmZ ddl	m
Z ddlmZmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZ ddlmZmZ ddl m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-  ed      e G d de#                    Z. G d de+      Z/ G d de%      Z0 G d de&      Z1 G d de!      Z2 G d d e*      Z3 G d! d"e)      Z4 G d# d$e'      Z5 G d% d&e(      Z6g d'Z7y)(    )Callable)OptionalN)strict   )initialization)CacheDynamicCache)create_causal_mask!create_sliding_window_causal_mask)BaseModelOutputWithPast)ROPE_INIT_FUNCTIONS)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)auto_docstring)TransformersKwargsmaybe_autocast   )Gemma3RotaryEmbedding)Olmo2Config)	Olmo2AttentionOlmo2DecoderLayerOlmo2ForCausalLMOlmo2ForSequenceClassification
Olmo2ModelOlmo2PreTrainedModelOlmo2RMSNormapply_rotary_pos_embeager_attention_forwardzallenai/Olmo-3-7B-Instruct)
checkpointc                        e Zd ZU dZdZdZdgZdddddddd	Zd
gdgfddgdgfdgdgfdZdZ	e
dz  ed<   dZee   dz  ed<    fdZd Z xZS )Olmo3Configa  
    Example:

    ```python
    >>> from transformers import Olmo3Model, Olmo3Config

    >>> # Initializing a Olmo3 7B style configuration
    >>> configuration = Olmo3Config()

    >>> # Initializing a model from the Olmo3 7B style configuration
    >>> model = Olmo3Model(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    olmo3g    Apast_key_valuescolwise_gather_outputrowwise_split_inputcolwiserowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proj	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnormi   Nsliding_windowlayer_typesc                     | j                   | j                  | _         | j                  5t        | j                        D cg c]  }|dz   dz  dk7  rdnd c}| _        t        |   di | y c c}w )N      r   sliding_attentionfull_attention )num_key_value_headsnum_attention_headsr1   rangenum_hidden_layerssuper__post_init__)selfkwargsi	__class__s      r/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/olmo3/modular_olmo3.pyr=   zOlmo3Config.__post_init__W   sz    ##+'+'?'?D$#W\]a]s]sWt RSA{a'7#=MM D 	''	 s   A0c                 n   |j                  dd       }ddiddid}| j                  | j                  n|| _        || j                  d   j                  |       | j                  j                  d      ddi| j                  d<   | j                  d   j	                  d|j                  d| j
                               | j                  j                  d      ddi| j                  d<   | j                  d   j	                  d|j                  d| j
                               | j                          |S )Nrope_scaling	rope_typedefault)r5   r6   r6   
rope_thetar5   )poprope_parametersupdateget
setdefaultdefault_thetastandardize_rope_params)r>   r?   rD   default_rope_paramss       rB   convert_rope_params_to_dictz'Olmo3Config.convert_rope_params_to_dictb   s:   zz.$7
 #.y!9*I6
 8<7K7K7Wt33]p#  !1299,G ##$45=6A95MD  !12-.99,

S_aeasasHtu##$78@9Di8PD  !4501<<&**\43E3EF	

 	$$&    )__name__
__module____qualname____doc__
model_typerM   keys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planr0   int__annotations__r1   liststrr=   rP   __classcell__rA   s   @rB   r"   r"   .   s    " JM#4"5%<%<%<%:"+ )"+ &(9:#%568IJ!"_$56 "&NC$J%$(KcT!(	(rQ   r"   c                       e Zd Zy)Olmo3RMSNormNrR   rS   rT   r7   rQ   rB   ra   ra   ~       rQ   ra   c                        e Zd Zdedef fdZ	 ddej                  deej                  ej                  f   dej                  dz  de	dz  d	e
e   d
eej                  ej                  dz  f   fdZ xZS )Olmo3Attentionconfig	layer_idxc                     t         |   ||       |j                  |   | _        | j                  dk(  r|j                  | _        y d | _        y )N)rg   r5   )r<   __init__r1   attention_typer0   r>   rf   rg   rA   s      rB   ri   zOlmo3Attention.__init__   sL    95$00;7;7J7JNa7af33gkrQ   Nr+   position_embeddingsr,   r$   r?   returnc                 h   |j                   d d }g |d| j                  }| j                  | j                  |            }| j	                  | j                  |            }	| j                  |      }
|j                  |      j                  dd      }|	j                  |      j                  dd      }	|
j                  |      j                  dd      }
|\  }}t        ||	||      \  }}	| |j                  |	|
| j                        \  }	}
t        j                  | j                  j                  t               } || ||	|
|f| j"                  sdn| j$                  | j&                  | j(                  d|\  }} |j*                  g |d j-                         }| j/                  |      }||fS )Nr3   r   g        )dropoutscalingr0   )shapehead_dimq_normq_projk_normk_projv_projview	transposer   rJ   rg   r   get_interfacerf   _attn_implementationr   trainingattention_dropoutrq   r0   reshape
contiguouso_proj)r>   r+   rl   r,   r$   r?   input_shapehidden_shapequery_states
key_statesvalue_statescossinattention_interfaceattn_outputattn_weightss                   rB   forwardzOlmo3Attention.forward   s    $))#2.88b8$--8{{4;;}#=>[[]!;<
{{=1#((6@@AF__\2<<QB
#((6@@AF&S#7jRUWZ#[ j&'6'='=j,X\XfXf'g$J(?(M(MKK,,.E)
 %8
%
  $}}C$2H2HLL..
%
 
%
!\ *k));;;;FFHkk+.L((rQ   N)rR   rS   rT   r"   rZ   ri   torchTensortupler   r   r   r   r^   r_   s   @rB   re   re      s    l{ ls l )-+)||+) #5<<#=>+) t+	+)
 +) +,+) 
u||U\\D00	1+)rQ   re   c                       e Zd Zy)Olmo3DecoderLayerNrb   r7   rQ   rB   r   r      rc   rQ   r   c                        e Zd Zddef fdZe	 	 	 	 ddedz  ded   dedz  dedz  de	d	e
f   f
 fd
       ZddZ xZS )Olmo3RotaryEmbeddingNrf   c                 (    t         |   ||       y )Ndevice)r<   ri   )r>   rf   r   rA   s      rB   ri   zOlmo3RotaryEmbedding.__init__   s    /rQ   r   ztorch.deviceseq_len
layer_typerm   ztorch.Tensorc                 (    t         |   | |||      S r   )r<   compute_default_rope_parameters)rf   r   r   r   rA   s       rB   r   z4Olmo3RotaryEmbedding.compute_default_rope_parameters   s     w6vvwPZ[[rQ   c                    t        | | d      }t        | | d      }|d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d	      5  |j                         |j                         z  j                  dd
      }	t        j                  |	|	fd      }
|
j                         |z  }|
j                         |z  }d d d        ||fS # 1 sw Y   fS xY w)N	_inv_freq_attention_scalingr   ro   r3   mpscpuF)device_typeenabledr   )dim)getattrfloatexpandrr   tor   
isinstancetyper]   r   rz   r   catr   r   )r>   xposition_idsr   inv_freqattention_scalinginv_freq_expandedposition_ids_expandedr   freqsembr   r   s                rB   r   zOlmo3RotaryEmbedding.forward   sP   4J<y!9:#DZL8J*KL$T1d]399;BB<CUCUVWCXZ\^_`ccdedldlm ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfkUC 	0&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')//C'')//C		0 Cx	0 Cxs   *A1E''E3r   )NNNN)rR   rS   rT   r"   ri   staticmethodr   rZ   r]   r   r   r   r   r^   r_   s   @rB   r   r      s    0{ 0 %)+/"!%	\d"\(\ t\ $J	\
 
~u$	%\ \rQ   r   c                       e Zd Zd Zy)Olmo3PreTrainedModelc                    t        j                  |       t        |t              r|j                  D ]  }|j
                  }|j                  |   dk7  rt        |j                  |      } ||j                  |      \  }}t        j                  t        || d      |       t        j                  t        || d      |        y y )NrF   )r   r   _original_inv_freq)r   _init_weightsr   r   r1   r   rE   r   rf   initcopy_r   )r>   moduler   rope_init_fncurr_inv_freq_s         rB   r   z"Olmo3PreTrainedModel._init_weights   s    %%f-f23$00 ^
%EE##J/9<#6v7G7G
7S#TL#/*#U q

76j\+CDmT

76j\9K+LM}]^ 4rQ   N)rR   rS   rT   r   r7   rQ   rB   r   r      s    	^rQ   r   c                        e Zd Zdef fdZ	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	e
dz  d
ee   defdZ xZS )
Olmo3Modelrf   c           	      &   t         |   |       t        |j                  |j                        | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        |      | _        y c c}w )N)epsrf   )r<   ri   ra   hidden_sizerms_norm_epsr/   nn
ModuleListr:   r;   r   r.   r   
rotary_embrk   s      rB   ri   zOlmo3Model.__init__   so      !3!39L9LM	mmCHIaIaCbcivy1c
 /f= ds   BNr)   r,   r   r$   r*   	use_cacher?   rm   c           
      N   |d u |d uz  rt        d      || j                  |      }|r|t        | j                        }|V||j	                         nd}t        j                  |j                  d   |j                        |z   }|j                  d      }t        |x}	t              s)| j                  ||||d}
t        d
i |
t        d
i |
d}	|}i }t        | j                  j                        D ]  }| j!                  |||      ||<    t#        | j$                  d | j                  j&                         D ]G  \  }} ||f|	| j                  j                  |      |||| j                  j                  |      d|}I | j)                  |      }t+        ||	      S )Nz:You must specify exactly one of input_ids or inputs_embedsr   r   r3   r   )rf   r*   r,   r$   r   )r6   r5   )r,   r   r$   rl   )last_hidden_stater$   r7   )
ValueErrorr-   r	   rf   get_seq_lengthr   arangerr   r   	unsqueezer   dictr
   r   setr1   r   	enumerater.   r;   r/   r   )r>   r)   r,   r   r$   r*   r   r?   past_seen_tokenscausal_mask_mappingmask_kwargsr+   rl   r   r@   decoder_layers                   rB   r   zOlmo3Model.forward   s    -t";<YZZ *.*;*;I*FM0*$++>OCRC^==?de <<(;(;A(>}G[G[\_ooL'11!4L ?-F ++!."0#2 ,K #5"C{"C%F%U%U#
 & dkk556 	gJ.2oom\[e.f
+	g !*$++6U8U8U*V W 	A})24;;3J3J13MN) /$78O8OPQ8R$S M	 		-0&++
 	
rQ   )NNNNNN)rR   rS   rT   r"   ri   r   
LongTensorr   r   FloatTensorboolr   r   r   r   r^   r_   s   @rB   r   r      s    >{ > .2.204(,26!%;
##d*;
 t+;
 &&-	;

 ;
 ((4/;
 $;;
 +,;
 
!;
rQ   r   c                       e Zd Zy)Olmo3ForCausalLMNrb   r7   rQ   rB   r   r   2  rc   rQ   r   c                       e Zd Zy)Olmo3ForSequenceClassificationNrb   r7   rQ   rB   r   r   6  rc   rQ   r   )r"   r   r   r   r   )8collections.abcr   typingr   r   torch.nnr   huggingface_hub.dataclassesr    r   r   cache_utilsr   r	   masking_utilsr
   r   modeling_outputsr   modeling_rope_utilsr   modeling_utilsr   r   processing_utilsr   utilsr   utils.genericr   r   gemma3.modeling_gemma3r   olmo2.configuration_olmo2r   olmo2.modeling_olmo2r   r   r   r   r   r   r   r   r   r"   ra   re   r   r   r   r   r   r   __all__r7   rQ   rB   <module>r      s    %    . & . R 7 6 F & # ? : 3
 
 
 78K+ K  9K\	< 	1)^ 1)h	) 	0 >
^/ 
^ D
 D
N	' 		%C 	rQ   