Ë
    ÜÓ^j•  ã                   ó  — d dl Z d dlmZ ddlmZ ddlmZ ddlmZ ddl	m
Z
mZmZmZmZ  ed	¬
«      e G d„ de«      «       «       Z G d„ de«      Z G d„ de«      Z G d„ de
«      Z G d„ de«      Z G d„ de«      Zg d¢Zy)é    N)Ústricté   )ÚCache)Úauto_docstringé   )ÚGemma2Config)ÚGemma2AttentionÚGemma2DecoderLayerÚGemma2ForCausalLMÚ	Gemma2MLPÚGemma2RMSNormzgoogle/vaultgemma-1b)Ú
checkpointc                   ó   — e Zd ZdZ e«       Zy)ÚVaultGemmaConfiga(  
    query_pre_attn_scalar (`float`, *optional*, defaults to 256):
        scaling factor used on the attention scores
    final_logit_softcapping (`float`, *optional*, defaults to 30.0):
        scaling factor when applying tanh softcapping on the logits.
    attn_logit_softcapping (`float`, *optional*, defaults to 50.0):
        scaling factor when applying tanh softcapping on the attention scores.

    ```python
    >>> from transformers import VaultGemmaModel, VaultGemmaConfig
    >>> # Initializing a VaultGemma vaultgemma-7b style configuration
    >>> configuration = VaultGemmaConfig()
    >>> # Initializing a model from the vaultgemma-7b style configuration
    >>> model = VaultGemmaModel(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚAttributeErrorÚuse_bidirectional_attention© ó    ú|/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/vaultgemma/modular_vaultgemma.pyr   r      s   „ ññ$ #1Ó"2Ñr   r   c                   ó   — e Zd Zy)ÚVaultGemmaRMSNormN©r   r   r   r   r   r   r   r   1   ó   „ Ør   r   c                   ó   — e Zd Zy)ÚVaultGemmaMLPNr   r   r   r   r   r   5   r   r   r   c                   ó,   ‡ — e Zd ZdZdedefˆ fd„Zˆ xZS )ÚVaultGemmaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperÚconfigÚ	layer_idxc                 ó0   •— t         ‰| �  «        d| _        y )NT)ÚsuperÚ__init__Ú	is_causal)Úselfr"   r#   Ú	__class__s      €r   r&   zVaultGemmaAttention.__init__<   s   ø€ Ü‰ÑÔØˆ�r   )r   r   r   r   r   Úintr&   Ú__classcell__©r)   s   @r   r!   r!   9   s   ø„ ÙGðÐ/ð ¸C÷ ñ r   r!   c                   ó  ‡ — e Zd Zˆ fd„Z	 	 	 d
dej
                  deej
                  ej
                  f   dej
                  dz  dej                  dz  dedz  deej                  eej                  ej                  f   dz  f   fd	„Z
ˆ xZS )ÚVaultGemmaDecoderLayerc                 ó,   •— t        ‰| �  di |¤Ž | `| `y )Nr   )r%   r&   Úpost_attention_layernormÚpost_feedforward_layernorm)r(   Úsuper_kwargsr)   s     €r   r&   zVaultGemmaDecoderLayer.__init__B   s   ø€ Ü‰ÑÑ(˜<Ò(ØÐ)ØÑ+r   NÚhidden_statesÚposition_embeddingsÚattention_maskÚposition_idsÚpast_key_valuesÚreturnc           	      ó¾   — |}| j                  |«      } | j                  d|||||dœ|¤Ž\  }}||z   }|}| j                  |«      }| j                  |«      }||z   }|S )N)r3   r4   r5   r6   r7   r   )Úinput_layernormÚ	self_attnÚpre_feedforward_layernormÚmlp)	r(   r3   r4   r5   r6   r7   ÚkwargsÚresidualÚ_s	            r   ÚforwardzVaultGemmaDecoderLayer.forwardG   s‹   € ð !ˆØ×,Ñ,¨]Ó;ˆà)˜4Ÿ>™>ð 
Ø'Ø 3Ø)Ø%Ø+ñ
ð ñ
Ñˆ�qð ! =Ñ0ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ  =Ñ0ˆàÐr   )NNN)r   r   r   r&   ÚtorchÚTensorÚtupleÚ
LongTensorr   ÚFloatTensorrA   r+   r,   s   @r   r.   r.   A   s¬   ø„ ô,ð /3Ø04Ø(,ñà—|‘|ðð # 5§<¡<°·±Ð#=Ñ>ðð Ÿ™ tÑ+ð	ð
 ×&Ñ&¨Ñ-ðð  ™ðð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷r   r.   c                   ó   — e Zd Zy)ÚVaultGemmaForCausalLMNr   r   r   r   rH   rH   e   r   r   rH   )r   rH   ÚVaultGemmaModelÚVaultGemmaPreTrainedModel)rB   Úhuggingface_hub.dataclassesr   Úcache_utilsr   Úutilsr   Úgemma2.configuration_gemma2r   Úgemma2.modeling_gemma2r	   r
   r   r   r   r   r   r   r!   r.   rH   Ú__all__r   r   r   ú<module>rQ      s‹   ðó  Ý .å  Ý #Ý 6ß uÕ uñ Ð1Ô2Øô3�|ó 3ó ó 3ð3ô,	˜ô 	ô	�Iô 	ô˜/ô ô!Ð/ô !ôH	Ð-ô 	ò�r   