Ë
    ÜÓ^jÐ2  ã                   óì  — d dl mZ d dlmZ d dlZd dlmZ d dlmZ ddl	m
Z ddlmZmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZ ddlmZmZ ddl m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-  ed¬«      e G d„ de#«      «       «       Z. G d„ de+«      Z/ G d„ de%«      Z0 G d„ de&«      Z1 G d„ de!«      Z2 G d„ d e*«      Z3 G d!„ d"e)«      Z4 G d#„ d$e'«      Z5 G d%„ d&e(«      Z6g d'¢Z7y)(é    )ÚCallable)ÚOptionalN)Ústricté   )Úinitialization)ÚCacheÚDynamicCache)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚBaseModelOutputWithPast)ÚROPE_INIT_FUNCTIONS)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úauto_docstring)ÚTransformersKwargsÚmaybe_autocasté   )ÚGemma3RotaryEmbedding)ÚOlmo2Config)	ÚOlmo2AttentionÚOlmo2DecoderLayerÚOlmo2ForCausalLMÚOlmo2ForSequenceClassificationÚ
Olmo2ModelÚOlmo2PreTrainedModelÚOlmo2RMSNormÚapply_rotary_pos_embÚeager_attention_forwardzallenai/Olmo-3-7B-Instruct)Ú
checkpointc                   ó    ‡ — e Zd ZU dZdZdZdgZdddddddd	œZd
gdgfddgdgfdgdgfdœZdZ	e
dz  ed<   dZee   dz  ed<   ˆ fd„Zd„ Zˆ xZS )ÚOlmo3Configa�  
    Example:

    ```python
    >>> from transformers import Olmo3Model, Olmo3Config

    >>> # Initializing a Olmo3 7B style configuration
    >>> configuration = Olmo3Config()

    >>> # Initializing a model from the Olmo3 7B style configuration
    >>> model = Olmo3Model(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    Úolmo3g    €„AÚpast_key_valuesÚcolwise_gather_outputÚrowwise_split_inputÚcolwiseÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormi   NÚsliding_windowÚlayer_typesc                 óê   •— | j                   €| j                  | _         | j                  €5t        | j                  «      D �cg c]  }|dz   dz  dk7  rdnd‘Œ c}| _        t        ‰| �  di |¤Ž y c c}w )Né   é   r   Úsliding_attentionÚfull_attention© )Únum_key_value_headsÚnum_attention_headsr1   ÚrangeÚnum_hidden_layersÚsuperÚ__post_init__)ÚselfÚkwargsÚiÚ	__class__s      €úr/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/olmo3/modular_olmo3.pyr=   zOlmo3Config.__post_init__W   sz   ø€ Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$à×ÑÐ#äW\Ð]a×]sÑ]sÓWtö ØRS¨¨A©°¡{°aÒ'7Ñ#Ð=MÑMò ˆDÔô 	‰ÑÑ' Ó'ùò	 s   ÁA0c                 ón  — |j                  dd «      }ddiddidœ}| j                  �| j                  n|| _        |�| j                  d   j                  |«       | j                  j                  d«      €ddi| j                  d<   | j                  d   j	                  d|j                  d| j
                  «      «       | j                  j                  d«      €ddi| j                  d<   | j                  d   j	                  d|j                  d| j
                  «      «       | j                  «        |S )NÚrope_scalingÚ	rope_typeÚdefault)r5   r6   r6   Ú
rope_thetar5   )ÚpopÚrope_parametersÚupdateÚgetÚ
setdefaultÚdefault_thetaÚstandardize_rope_params)r>   r?   rD   Údefault_rope_paramss       rB   Úconvert_rope_params_to_dictz'Olmo3Config.convert_rope_params_to_dictb   s:  € Ø—z‘z .°$Ó7ˆð
 #.¨yÐ!9Ø*¨IÐ6ñ
Ðð 8<×7KÑ7KÐ7W˜t×3Ò3Ð]pˆÔØÐ#Ø× Ñ Ð!1Ñ2×9Ñ9¸,ÔGð ×Ñ×#Ñ#Ð$4Ó5Ð=Ø6AÀ9Ð5MˆD× Ñ Ð!1Ñ2Ø×ÑÐ-Ñ.×9Ñ9¸,ÈÏ
É
ÐS_Ðae×asÑasÓHtÔuØ×Ñ×#Ñ#Ð$7Ó8Ð@Ø9DÀiÐ8PˆD× Ñ Ð!4Ñ5Ø×ÑÐ0Ñ1×<Ñ<Ø˜&Ÿ*™* \°4×3EÑ3EÓFô	
ð
 	×$Ñ$Ô&Øˆó    )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typerM   Úkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planr0   ÚintÚ__annotations__r1   ÚlistÚstrr=   rP   Ú__classcell__©rA   s   @rB   r"   r"   .   s�   ø… ñð" €JØ€MØ#4Ð"5Ðà%<Ø%<Ø%<Ø%:Ø"+Ø )Ø"+ñÐð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð "&€N�C˜$‘JÓ%Ø$(€K��c‘˜TÑ!Ó(ô	(örQ   r"   c                   ó   — e Zd Zy)ÚOlmo3RMSNormN©rR   rS   rT   r7   rQ   rB   ra   ra   ~   ó   „ ØrQ   ra   c                   óð   ‡ — e Zd Zdedefˆ fd„Z	 ddej                  deej                  ej                  f   dej                  dz  de	dz  d	e
e   d
eej                  ej                  dz  f   fd„Zˆ xZS )ÚOlmo3AttentionÚconfigÚ	layer_idxc                 ó    •— t         ‰| �  ||¬«       |j                  |   | _        | j                  dk(  r|j                  | _        y d | _        y )N)rg   r5   )r<   Ú__init__r1   Úattention_typer0   ©r>   rf   rg   rA   s      €rB   ri   zOlmo3Attention.__init__…   sL   ø€ Ü‰Ñ˜¨9ÐÔ5Ø$×0Ñ0°Ñ;ˆÔØ7;×7JÑ7JÐNaÒ7a˜f×3Ñ3ˆÕÐgkˆÕrQ   Nr+   Úposition_embeddingsr,   r$   r?   Úreturnc                 óh  — |j                   d d }g |¢d‘| j                  ‘­}| j                  | j                  |«      «      }| j	                  | j                  |«      «      }	| j                  |«      }
|j                  |«      j                  dd«      }|	j                  |«      j                  dd«      }	|
j                  |«      j                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t         «      } || ||	|
|f| j"                  sdn| j$                  | j&                  | j(                  dœ|¤Ž\  }} |j*                  g |¢d‘­Ž j-                  «       }| j/                  |«      }||fS )Néÿÿÿÿr3   r   g        )ÚdropoutÚscalingr0   )ÚshapeÚhead_dimÚq_normÚq_projÚk_normÚk_projÚv_projÚviewÚ	transposer   rJ   rg   r   Úget_interfacerf   Ú_attn_implementationr   ÚtrainingÚattention_dropoutrq   r0   ÚreshapeÚ
contiguousÚo_proj)r>   r+   rl   r,   r$   r?   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                   rB   ÚforwardzOlmo3Attention.forwardŠ   s¶  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ 4§;¡;¨}Ó#=Ó>ˆØ—[‘[ §¡¨]Ó!;Ó<ˆ
Ø—{‘{ =Ó1ˆà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(rQ   ©N)rR   rS   rT   r"   rZ   ri   ÚtorchÚTensorÚtupler   r   r   rŒ   r^   r_   s   @rB   re   re   „   s�   ø„ ðl˜{ð l°sõ lð )-ñ+)à—|‘|ð+)ð # 5§<¡<°·±Ð#=Ñ>ð+)ð Ÿ™ tÑ+ð	+)ð
  ™ð+)ð Ð+Ñ,ð+)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷+)rQ   re   c                   ó   — e Zd Zy)ÚOlmo3DecoderLayerNrb   r7   rQ   rB   r’   r’   ¸   rc   rQ   r’   c                   ó„   ‡ — e Zd Zddefˆ fd„Ze	 	 	 	 ddedz  ded   dedz  dedz  de	d	e
f   f
ˆ fd
„«       Zdd„Zˆ xZS )ÚOlmo3RotaryEmbeddingNrf   c                 ó(   •— t         ‰| �  ||¬«       y )N©Údevice)r<   ri   )r>   rf   r—   rA   s      €rB   ri   zOlmo3RotaryEmbedding.__init__½   s   ø€ Ü‰Ñ˜¨ÐÕ/rQ   r—   ztorch.deviceÚseq_lenÚ
layer_typerm   ztorch.Tensorc                 ó(   •— t         ‰| �  | |||«      S r�   )r<   Úcompute_default_rope_parameters)rf   r—   r˜   r™   rA   s       €rB   r›   z4Olmo3RotaryEmbedding.compute_default_rope_parametersÀ   s   ø€ ô ‰wÑ6°v¸vÀwÐPZÓ[Ð[rQ   c                 óì  — t        | |› d�«      }t        | |› d�«      }|d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬	«      5  |j                  «       |j                  «       z  j                  dd
«      }	t        j                  |	|	fd¬«      }
|
j                  «       |z  }|
j                  «       |z  }d d d «       ||fS # 1 sw Y   fS xY w)NÚ	_inv_freqÚ_attention_scalingr   ro   r3   ÚmpsÚcpuF)Údevice_typeÚenabledr   )Údim)ÚgetattrÚfloatÚexpandrr   Útor—   Ú
isinstanceÚtyper]   r   rz   rŽ   Úcatr‡   rˆ   )r>   ÚxÚposition_idsr™   Úinv_freqÚattention_scalingÚinv_freq_expandedÚposition_ids_expandedr¡   ÚfreqsÚembr‡   rˆ   s                rB   rŒ   zOlmo3RotaryEmbedding.forwardÉ   sP  € ä˜4 J <¨yÐ!9Ó:ˆÜ# D¨Z¨LÐ8JÐ*KÓLÐà$ Tª1¨d ]Ñ3×9Ñ9Ó;×BÑBÀ<×CUÑCUÐVWÑCXÐZ\Ð^_Ó`×cÑcÐde×dlÑdlÓmÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	0Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)Ð/Ñ/ˆCØ—'‘'“)Ð/Ñ/ˆC÷		0ð �Cˆxˆ÷	0ð �Cˆxˆús   Ã*A1E'Å'E3r�   )NNNN)rR   rS   rT   r"   ri   Ústaticmethodr   rZ   r]   r�   r¥   r›   rŒ   r^   r_   s   @rB   r”   r”   ¼   s…   ø„ ñ0˜{õ 0ð à%)Ø+/Ø"Ø!%ñ	\Ø˜dÑ"ð\à˜Ñ(ð\ð �t‘ð\ð ˜$‘Jð	\ð
 
ˆ~˜uÐ$Ñ	%ô\ó ð\÷rQ   r”   c                   ó   — e Zd Zd„ Zy)ÚOlmo3PreTrainedModelc                 ó”  — t        j                  |«       t        |t        «      r£|j                  D ]“  }|j
                  }|j                  |   dk7  rt        |j                  |      } ||j                  |¬«      \  }}t        j                  t        ||› d�«      |«       t        j                  t        ||› d�«      |«       Œ• y y )NrF   )r™   r�   Ú_original_inv_freq)r   Ú_init_weightsr¨   r”   r1   r›   rE   r   rf   ÚinitÚcopy_r¤   )r>   Úmoduler™   Úrope_init_fnÚcurr_inv_freqÚ_s         rB   r¸   z"Olmo3PreTrainedModel._init_weightsÜ   s¸   € Ü×%Ñ% fÔ-Ü�fÔ2Ô3Ø$×0Ñ0ò ^�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ò<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ô#UÑ �˜qÜ—
‘
œ7 6¨j¨\¸Ð+CÓDÀmÔTÜ—
‘
œ7 6¨j¨\Ð9KÐ+LÓMÈ}Õ]ñ^ð 4rQ   N)rR   rS   rT   r¸   r7   rQ   rB   rµ   rµ   Û   s   „ ó	^rQ   rµ   c                   óÔ   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	e
dz  d
ee   defd„Zˆ xZS )Ú
Olmo3Modelrf   c           	      ó&  •— t         ‰| �  |«       t        |j                  |j                  ¬«      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |¬«      | _        y c c}w )N)Úeps©rf   )r<   ri   ra   Úhidden_sizeÚrms_norm_epsr/   ÚnnÚ
ModuleListr:   r;   r’   r.   r”   Ú
rotary_embrk   s      €rB   ri   zOlmo3Model.__init__ì   so   ø€ Ü‰Ñ˜Ô Ü  ×!3Ñ!3¸×9LÑ9LÔMˆŒ	Ü—m‘mÜCHÈ×IaÑIaÓCbÖc°iÔ˜v yÕ1Òcó
ˆŒô /°fÔ=ˆ�ùò ds   ÁBNr)   r,   r¬   r$   r*   Ú	use_cacher?   rm   c           
      óN  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      s)| j                  ||||dœ}
t        d
i |
¤Žt        d
i |
¤Ždœ}	|}i }t        | j                  j                  «      D ]  }| j!                  |||«      ||<   Œ t#        | j$                  d | j                  j&                   «      D ]G  \  }} ||f|	| j                  j                  |      |||| j                  j                  |      dœ|¤Ž}ŒI | j)                  |«      }t+        ||¬	«      S )Nz:You must specify exactly one of input_ids or inputs_embedsrÃ   r   r3   r–   )rf   r*   r,   r$   r¬   )r6   r5   )r,   r¬   r$   rl   )Úlast_hidden_stater$   r7   )Ú
ValueErrorr-   r	   rf   Úget_seq_lengthrŽ   Úarangerr   r—   Ú	unsqueezer¨   Údictr
   r   Úsetr1   rÈ   Ú	enumerater.   r;   r/   r   )r>   r)   r,   r¬   r$   r*   rÉ   r?   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr+   rl   r™   r@   Údecoder_layers                   rB   rŒ   zOlmo3Model.forwardô   sÛ  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø*.×*;Ñ*;¸IÓ*FˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-ÄÔFð Ÿ+™+Ø!.Ø"0Ø#2Ø ,ñˆKô #5Ñ"C°{Ñ"CÜ%FÑ%UÈÑ%Uñ#Ðð
 &ˆØ ÐÜ˜dŸk™k×5Ñ5Ó6ò 	gˆJØ.2¯o©o¸mÈ\Ð[eÓ.fÐ 
Ò+ð	gô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÓ Wò 	ÑˆAˆ}Ù)Øðà2°4·;±;×3JÑ3JÈ1Ñ3MÑNØ)Ø /Ø$7¸¿¹×8OÑ8OÐPQÑ8RÑ$Sñð ñ‰Mð	ð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ô
ð 	
rQ   )NNNNNN)rR   rS   rT   r"   ri   rŽ   Ú
LongTensorr�   r   ÚFloatTensorÚboolr   r   r   rŒ   r^   r_   s   @rB   rÀ   rÀ   ë   s®   ø„ ð>˜{õ >ð .2Ø.2Ø04Ø(,Ø26Ø!%ñ;
à×#Ñ# dÑ*ð;
ð Ÿ™ tÑ+ð;
ð ×&Ñ&¨Ñ-ð	;
ð
  ™ð;
ð ×(Ñ(¨4Ñ/ð;
ð ˜$‘;ð;
ð Ð+Ñ,ð;
ð 
!÷;
rQ   rÀ   c                   ó   — e Zd Zy)ÚOlmo3ForCausalLMNrb   r7   rQ   rB   rÛ   rÛ   2  rc   rQ   rÛ   c                   ó   — e Zd Zy)ÚOlmo3ForSequenceClassificationNrb   r7   rQ   rB   rÝ   rÝ   6  rc   rQ   rÝ   )r"   rÛ   rÝ   rÀ   rµ   )8Úcollections.abcr   Útypingr   rŽ   Útorch.nnrÆ   Úhuggingface_hub.dataclassesr   Ú r   r¹   Úcache_utilsr   r	   Úmasking_utilsr
   r   Úmodeling_outputsr   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   Úutils.genericr   r   Úgemma3.modeling_gemma3r   Úolmo2.configuration_olmo2r   Úolmo2.modeling_olmo2r   r   r   r   r   r   r   r   r   r"   ra   re   r’   r”   rµ   rÀ   rÛ   rÝ   Ú__all__r7   rQ   rB   ú<module>rï      så   ðõ %Ý ã Ý Ý .å &ß .ß RÝ 7Ý 6ß FÝ &Ý #ß ?Ý :Ý 3÷
÷ 
õ 
ñ Ð7Ô8ØôK�+ó Kó ó 9ðKô\	�<ô 	ô1)�^ô 1)ôh	Ð)ô 	ôÐ0ô ô>
^Ð/ô 
^ô D
�ô D
ôN	Ð'ô 	ô	Ð%Cô 	ò�rQ   