Ë
    ÜÓ^jZ"  ã                   óÎ  — d dl mZ d dlZd dlmZ d dlmZ d dlmZ ddl	m
Z
 ddlmZ ddlmZ dd	lmZmZ d
dlmZmZmZ d
dlmZ d
dlmZmZmZmZmZmZmZ  ej@                  e!«      Z" ed¬«      e G d„ de«      «       «       Z# G d„ de«      Z$ G d„ de«      Z%d„ Z& G d„ de«      Z' G d„ de«      Z( G d„ de«      Z) G d„ de«      Z* G d„ d e«      Z+ G d!„ d"e«      Z,g d#¢Z-y)$é    )ÚCallableN)Ústrict)ÚTransformersKwargsé   )ÚCache)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)Úauto_docstringÚloggingé   )ÚLlamaPreTrainedModelÚLlamaRMSNormÚeager_attention_forward)Ú
OlmoConfig)ÚOlmoAttentionÚOlmoDecoderLayerÚOlmoForCausalLMÚOlmoForSequenceClassificationÚ	OlmoModelÚOlmoRotaryEmbeddingÚapply_rotary_pos_embzallenai/Olmo2-7B-1124-hf)Ú
checkpointc                   ól   — e Zd ZU dZdZddddddddœZdgd	gfd
dgd
gfd
gd
gfdœZdZee	d<    e
«       Zy)ÚOlmo2Configa�  
    Example:

    ```python
    >>> from transformers import Olmo2Model, Olmo2Config

    >>> # Initializing a Olmo2 7B style configuration
    >>> configuration = Olmo2Config()

    >>> # Initializing a model from the Olmo2 7B style configuration
    >>> model = Olmo2Model(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    Úolmo2Úcolwise_gather_outputÚrowwise_split_inputÚcolwiseÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormgñhãˆµøä>Úrms_norm_epsN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚbase_model_tp_planÚbase_model_pp_planr'   ÚfloatÚ__annotations__ÚAttributeErrorÚclip_qkv© ó    úr/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/olmo2/modular_olmo2.pyr   r   0   su   … ñð" €Jà%<Ø%<Ø%<Ø%:Ø"+Ø )Ø"+ñÐð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð €L�%ÓÙÓ�Hr4   r   c                   ó   — e Zd Zd„ Zy)ÚOlmo2RMSNormc                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |z  j                  |«      S )Nr   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtoÚtorchÚfloat32ÚpowÚmeanÚrsqrtÚvariance_epsilonÚweight)Úselfr"   Úinput_dtypeÚvariances       r5   ÚforwardzOlmo2RMSNorm.forward[   sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ—‘˜mÑ+×/Ñ/°Ó<Ð<r4   N)r(   r)   r*   rG   r3   r4   r5   r7   r7   Z   s   „ ó=r4   r7   c                   ó   — e Zd Zy)ÚOlmo2RotaryEmbeddingN©r(   r)   r*   r3   r4   r5   rI   rI   c   ó   „ Ør4   rI   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Nr9   r   )Údim)Úshaper=   Úcat)ÚxÚx1Úx2s      r5   Úrotate_halfrS   g   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r4   c                   óø   ‡ — e Zd Zddededz  fˆ fd„Z	 ddej                  deej                  ej                  f   dej                  dz  de	dz  d	e
e   d
eej                  ej                  dz  f   fd„Zˆ xZS )ÚOlmo2AttentionNÚconfigÚ	layer_idxc                 óð   •— t         ‰| �  ||¬«       t        |j                  | j                  z  |j
                  «      | _        t        |j                  | j                  z  |j
                  «      | _        y )N©rW   )	ÚsuperÚ__init__r7   Únum_attention_headsÚhead_dimr'   Úq_normÚnum_key_value_headsÚk_norm©rD   rV   rW   Ú	__class__s      €r5   r[   zOlmo2Attention.__init__r   s[   ø€ Ü‰Ñ˜¨9ÐÔ5Ü" 6×#=Ñ#=ÀÇÁÑ#MÈv×ObÑObÓcˆŒÜ" 6×#=Ñ#=ÀÇÁÑ#MÈv×ObÑObÓcˆ�r4   r"   Úposition_embeddingsr#   Úpast_key_valuesÚkwargsÚreturnc                 óR  — |j                   d d }g |¢d‘| j                  ‘­}| j                  | j                  |«      «      }| j	                  | j                  |«      «      }	| j                  |«      }
|j                  |«      j                  dd«      }|	j                  |«      j                  dd«      }	|
j                  |«      j                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t         «      } || ||	|
|f| j"                  sdn| j$                  | j&                  dœ|¤Ž\  }} |j(                  g |¢d‘­Ž j+                  «       }| j-                  |«      }||fS )Nr9   é   r   g        )ÚdropoutÚscaling)rN   r]   r^   Úq_projr`   Úk_projÚv_projÚviewÚ	transposer   ÚupdaterW   r   Úget_interfacerV   Ú_attn_implementationr   ÚtrainingÚattention_dropoutrj   ÚreshapeÚ
contiguousÚo_proj)rD   r"   rc   r#   rd   re   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                   r5   rG   zOlmo2Attention.forwardw   s­  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ 4§;¡;¨}Ó#=Ó>ˆØ—[‘[ §¡¨]Ó!;Ó<ˆ
Ø—{‘{ =Ó1ˆà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r4   )N)r(   r)   r*   r   Úintr[   r=   ÚTensorÚtupler   r	   r   rG   Ú__classcell__©rb   s   @r5   rU   rU   q   s¡   ø„ ñd˜{ð d°s¸T±zõ dð )-ñ*)à—|‘|ð*)ð # 5§<¡<°·±Ð#=Ñ>ð*)ð Ÿ™ tÑ+ð	*)ð
  ™ð*)ð Ð+Ñ,ð*)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷*)r4   rU   c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
eej                  ej                  f   dz  dee   dej                  fd„Zˆ xZS )ÚOlmo2DecoderLayerrV   rW   c                 óè   •— t         ‰| �  ||¬«       t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        ||¬«      | _        | `	y )NrY   ©Úeps)rV   rW   )
rZ   r[   r7   Úhidden_sizer'   Úpost_attention_layernormÚpost_feedforward_layernormrU   Ú	self_attnÚinput_layernormra   s      €r5   r[   zOlmo2DecoderLayer.__init__¨   s_   ø€ Ü‰Ñ˜¨9ÐÔ5Ü(4°V×5GÑ5GÈV×M`ÑM`Ô(aˆÔ%Ü*6°v×7IÑ7IÈv×ObÑObÔ*cˆÔ'Ü'¨vÀÔKˆŒØÑ r4   Nr"   r#   Úposition_idsrd   Ú	use_cacherc   re   rf   c           
      óÀ   — |} | j                   d||||||dœ|¤Ž\  }}	| j                  |«      }||z   }|}| j                  |«      }| j                  |«      }||z   }|S )N)r"   r#   r‘   rd   r’   rc   r3   )r�   r�   ÚmlprŽ   )
rD   r"   r#   r‘   rd   r’   rc   re   ÚresidualÚ_s
             r5   rG   zOlmo2DecoderLayer.forward¯   s�   € ð !ˆØ)˜4Ÿ>™>ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
Ñˆ�qð ×5Ñ5°mÓDˆØ  =Ñ0ˆð !ˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ  =Ñ0ˆØÐr4   )NNNFN)r(   r)   r*   r   r‚   r[   r=   rƒ   Ú
LongTensorr   Úboolr„   r	   r   rG   r…   r†   s   @r5   rˆ   rˆ   §   s»   ø„ ð!˜{ð !°sõ !ð /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð+Ñ,ðð 
�‰÷r4   rˆ   c                   ó   — e Zd Zy)ÚOlmo2PreTrainedModelNrJ   r3   r4   r5   rš   rš   Î   rK   r4   rš   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )Ú
Olmo2ModelrV   c           	      ó  •— t         ‰| �  |«       t        |j                  |j                  ¬«      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        y c c}w )NrŠ   )rZ   r[   r7   rŒ   r'   r&   ÚnnÚ
ModuleListÚrangeÚnum_hidden_layersrˆ   r%   ra   s      €r5   r[   zOlmo2Model.__init__Õ   s^   ø€ Ü‰Ñ˜Ô Ü  ×!3Ñ!3¸×9LÑ9LÔMˆŒ	Ü—m‘mÜCHÈ×IaÑIaÓCbÖc°iÔ˜v yÕ1Òcó
ˆ�ùÚcs   ÁA=)r(   r)   r*   r   r[   r…   r†   s   @r5   rœ   rœ   Ô   s   ø„ ð
˜{÷ 
ñ 
r4   rœ   c                   ó   — e Zd Zy)ÚOlmo2ForCausalLMNrJ   r3   r4   r5   r£   r£   Þ   rK   r4   r£   c                   ó   — e Zd Zy)ÚOlmo2ForSequenceClassificationNrJ   r3   r4   r5   r¥   r¥   â   rK   r4   r¥   )r   r£   r¥   rœ   rš   ).Úcollections.abcr   r=   Útorch.nnrž   Úhuggingface_hub.dataclassesr   Útransformers.utils.genericr   Úcache_utilsr   Úmodeling_utilsr   Úprocessing_utilsr	   Úutilsr
   r   Úllama.modeling_llamar   r   r   Úolmo.configuration_olmor   Úolmo.modeling_olmor   r   r   r   r   r   r   Ú
get_loggerr(   Úloggerr   r7   rI   rS   rU   rˆ   rš   rœ   r£   r¥   Ú__all__r3   r4   r5   ú<module>r´      sê   ðõ( %ã Ý Ý .å 9å  Ý 5Ý &ß ,ß ^Ñ ^Ý 0÷÷ ñ ð 
ˆ×	Ñ	˜HÓ	%€ñ Ð5Ô6Øô# �*ó # ó ó 7ð# ôP=�<ô =ô	Ð.ô 	ò(ô0)�]ô 0)ôl$Ð(ô $ôN	Ð/ô 	ô
�ô 
ô	�ô 	ô	Ð%Bô 	ò�r4   