Ë
    ÛÓ^j’%  ã                   ó8  — d dl Z d dlmZ d dl mZ ddlmZ ddlmZm	Z	 ddl
mZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZ ddlmZmZmZmZmZm Z m!Z!m"Z" ddiZ#dZ$ ejJ                  e&«      Z' ed¬«      e G d„ de«      «       «       Z( G d„ dejR                  «      Z* G d„ dejV                  «      Z, G d„ de«      Z- G d„ de"«      Z. G d„ d e«      Z/ G d!„ d"e!«      Z0 G d#„ d$e «      Z1 G d%„ d&e«      Z2 G d'„ d(e«      Z3 G d)„ d*e«      Z4g d+¢Z5y),é    N)Ústrict)Únné   )Úinitialization)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_mask)ÚBaseModelOutputWithPast)ÚRopeParameters)ÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚloggingé   )ÚLlamaAttentionÚLlamaForCausalLMÚLlamaForSequenceClassificationÚLlamaForTokenClassificationÚLlamaMLPÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRotaryEmbeddingÚ
vocab_fileztokenizer.modelu   â–�zgoogle/gemma-7b)Ú
checkpointc                   ó¤  — e Zd ZU dZdZdgZddddddddœZdgdgfd	d
gd	gfd	gd	gfdœZdZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZee
d<   dZe	e
d<   dZee
d<   dZee
d <   d!Zee
d"<   d#Ze	d$z  e
d%<   d&Ze	ee	   z  d$z  e
d'<   d(Ze	d$z  e
d)<   d!Zee
d*<   d$Zee z  d$z  e
d+<   d,Z!ee
d-<   d.Z"ee	z  e
d/<   d$Z#ed$z  e
d0<   y$)1ÚGemmaConfiga  
    use_bidirectional_attention (`bool`, *optional*):
        If True, the model will attend to all text tokens instead of using a causal mask.

    ```python
    >>> from transformers import GemmaModel, GemmaConfig
    >>> # Initializing a Gemma gemma-7b style configuration
    >>> configuration = GemmaConfig()
    >>> # Initializing a model from the gemma-7b style configuration
    >>> model = GemmaModel(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```ÚgemmaÚpast_key_valuesÚcolwiseÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormi è Ú
vocab_sizei   Úhidden_sizei `  Úintermediate_sizeé   Únum_hidden_layersé   Únum_attention_headsÚnum_key_value_headsé   Úhead_dimÚgelu_pytorch_tanhÚ
hidden_acti    Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeç�íµ ÷Æ°>Úrms_norm_epsTÚ	use_cacher   NÚpad_token_idé   Úeos_token_idr   Úbos_token_idÚtie_word_embeddingsÚrope_parametersFÚattention_biasg        Úattention_dropoutÚuse_bidirectional_attention)$Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planr*   ÚintÚ__annotations__r+   r,   r.   r0   r1   r3   r5   Ústrr6   r7   Úfloatr9   r:   Úboolr;   r=   Úlistr>   r?   r@   r   ÚdictrA   rB   rC   © ó    úr/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/gemma/modular_gemma.pyr   r   1   sd  … ñð €JØ#4Ð"5Ðà%.Ø%.Ø%.Ø%.Ø"+Ø )Ø"+ñÐð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð €J�ÓØ€K�ÓØ"Ð�sÓ"ØÐ�sÓØ!Ð˜Ó!Ø!Ð˜Ó!Ø€HˆcÓØ)€J�Ó)Ø#'Ð˜SÓ'Ø#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ €L�#˜‘*Ó Ø+,€L�#˜˜S™	‘/ DÑ(Ó,Ø €L�#˜‘*Ó Ø $Ð˜Ó$Ø48€O�^ dÑ*¨TÑ1Ó8Ø €N�DÓ Ø%(Ð�u˜s‘{Ó(Ø/3Ð ¨¡Ô3rT   r   c            	       óZ   ‡ — e Zd ZdZd	dedededefˆ fd„Zdej                  fˆ fd„Z	ˆ xZ
S )
ÚGemmaTextScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scalec                 ó„   •— t         ‰| �  |||«       || _        | j                  dt	        j
                  |«      d¬«       y )Nr[   F)Ú
persistent)ÚsuperÚ__init__Úscalar_embed_scaleÚregister_bufferÚtorchÚtensor)ÚselfrX   rY   rZ   r[   Ú	__class__s        €rU   r_   z%GemmaTextScaledWordEmbedding.__init__n   s;   ø€ Ü‰Ñ˜¨¸ÔDØ"-ˆÔØ×Ñ˜]¬E¯L©L¸Ó,EÐRWÐÕXrT   r#   c                 ó‚   •— t         ‰| �  |«      | j                  j                  | j                  j
                  «      z  S ©N)r^   Úforwardr[   ÚtoÚweightÚdtype)rd   r#   re   s     €rU   rh   z$GemmaTextScaledWordEmbedding.forwards   s2   ø€ Ü‰w‰˜yÓ)¨D×,<Ñ,<×,?Ñ,?ÀÇÁ×@QÑ@QÓ,RÑRÐRrT   )ç      ð?)rD   rE   rF   rG   rL   rO   r_   rb   ÚTensorrh   Ú__classcell__©re   s   @rU   rW   rW   i   sG   ø„ ññY sð Y¸3ð YÈSð YÐ_dõ Yð
S §¡÷ Sñ SrT   rW   c                   ó<   ‡ — e Zd Zddedefˆ fd„Zd„ Zd„ Zd„ Zˆ xZ	S )ÚGemmaRMSNormÚdimÚepsc                 óŠ   •— t         ‰| �  «        || _        t        j                  t        j                  |«      «      | _        y rg   )r^   r_   rs   r   Ú	Parameterrb   Úzerosrj   )rd   rr   rs   re   s      €rU   r_   zGemmaRMSNorm.__init__x   s.   ø€ Ü‰ÑÔØˆŒÜ—l‘l¤5§;¡;¨sÓ#3Ó4ˆ�rT   c                 óŒ   — |t        j                  |j                  d«      j                  dd¬«      | j                  z   «      z  S )Nr   éÿÿÿÿT)Úkeepdim)rb   ÚrsqrtÚpowÚmeanrs   )rd   Úxs     rU   Ú_normzGemmaRMSNorm._norm}   s4   € Ø”5—;‘;˜qŸu™u Q›xŸ}™}¨R¸˜}Ó>ÀÇÁÑIÓJÑJÐJrT   c                 ó¢   — | j                  |j                  «       «      }|d| j                  j                  «       z   z  }|j                  |«      S )Nrl   )r~   rO   rj   Útype_as)rd   r}   Úoutputs      rU   rh   zGemmaRMSNorm.forward€   sC   € Ø—‘˜AŸG™G›IÓ&ˆð ˜3 §¡×!2Ñ!2Ó!4Ñ4Ñ5ˆØ�~‰~˜aÓ Ð rT   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)Útuplerj   Úshapers   )rd   s    rU   Ú
extra_reprzGemmaRMSNorm.extra_repr‡   s'   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$·(±(°Ð<Ð<rT   )r8   )
rD   rE   rF   rL   rO   r_   r~   rh   r…   rn   ro   s   @rU   rq   rq   w   s&   ø„ ñ5˜Cð 5 eõ 5ò
Kò!ö=rT   rq   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚGemmaMLPc                 óJ  •— t         ‰| �  |«       t        j                  | j                  | j
                  d¬«      | _        t        j                  | j                  | j
                  d¬«      | _        t        j                  | j
                  | j                  d¬«      | _        y )NF)Úbias)	r^   r_   r   ÚLinearr+   r,   Ú	gate_projÚup_projÚ	down_proj©rd   Úconfigre   s     €rU   r_   zGemmaMLP.__init__Œ   ss   ø€ Ü‰Ñ˜Ô ÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆ�rT   )rD   rE   rF   r_   rn   ro   s   @rU   r‡   r‡   ‹   s   ø„ ÷Yð YrT   r‡   c                   ó   — e Zd Zy)ÚGemmaRotaryEmbeddingN©rD   rE   rF   rS   rT   rU   r‘   r‘   “   ó   „ ØrT   r‘   c                   ó,   ‡ — e Zd ZdZdedefˆ fd„Zˆ xZS )ÚGemmaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr�   Ú	layer_idxc                 óH   •— t         ‰| �  «        t        |dd«       | _        y )NrC   F)r^   r_   ÚgetattrÚ	is_causal)rd   r�   r–   re   s      €rU   r_   zGemmaAttention.__init__š   s"   ø€ Ü‰ÑÔÜ$ VÐ-JÈEÓRÐRˆ�rT   )rD   rE   rF   rG   r   rL   r_   rn   ro   s   @rU   r•   r•   —   s"   ø„ ÙGðS˜{ð S°s÷ Sñ SrT   r•   c                   ó:   — e Zd Z ej                  «       d„ «       Zy)ÚGemmaPreTrainedModelc                 ó  — t        j                  | |«       d|j                  j                  v r t	        j
                  |j                  «       y t        |t        «      r+t	        j                  |j                  |j                  «       y y )NÚRMSNorm)r   Ú_init_weightsre   rD   ÚinitÚzeros_rj   Ú
isinstancerW   Ú	constant_r[   r`   )rd   Úmodules     rU   rž   z"GemmaPreTrainedModel._init_weights    s`   € ä×%Ñ% d¨FÔ3à˜×(Ñ(×1Ñ1Ñ1Ü�K‰K˜Ÿ™Õ&Ü˜Ô <Ô=Ü�N‰N˜6×-Ñ-¨v×/HÑ/HÕIð >rT   N)rD   rE   rF   rb   Úno_gradrž   rS   rT   rU   r›   r›   Ÿ   s   „ Ø€U‡]�]ƒ_ñJó ñJrT   r›   c                   óÔ   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	e
dz  d
ee   defd„Zˆ xZS )Ú
GemmaModelr�   c                 ó¶   •— t         ‰| �  |«       t        |j                  |j                  | j
                  | j                  j                  dz  ¬«      | _        y )Ng      à?)r[   )r^   r_   rW   r*   r+   rZ   r�   r'   rŽ   s     €rU   r_   zGemmaModel.__init__«   sK   ø€ Ü‰Ñ˜Ô ä8Ø×Ñ˜v×1Ñ1°4×3CÑ3CÐQU×Q\ÑQ\×QhÑQhÐjmÑQmô
ˆÕrT   Nr#   r&   Úposition_idsr    r$   r:   ÚkwargsÚreturnc           
      óN  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        | j                  ||||¬«      }	|}
| j                  |
|¬«      }| j                  d | j                  j                   D ]  } ||
f|	||||dœ|¤Ž}
Œ | j                  |
«      }
t        |
|r|¬	«      S d ¬	«      S )
Nz:You must specify exactly one of input_ids or inputs_embeds)r�   r   r<   )Údevice)r�   r$   r&   r    r¨   )r¨   )r&   r¨   r    r:   Úposition_embeddings)Úlast_hidden_stater    )Ú
ValueErrorr'   r   r�   Úget_seq_lengthrb   Úaranger„   r¬   Ú	unsqueezer
   Ú
rotary_embr(   r.   r)   r   )rd   r#   r&   r¨   r    r$   r:   r©   Úpast_seen_tokensÚcausal_maskr%   r­   Údecoder_layers                rU   rh   zGemmaModel.forward²   s[  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆð &ˆØ"Ÿo™o¨mÈ,˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÐIò 		ˆMÙ)Øðà*Ø)Ø /Ø#Ø$7ñð ñ‰Mð		ð Ÿ	™	 -Ó0ˆÜ&Ø+Ù/8˜Oô
ð 	
à>Bô
ð 	
rT   )NNNNNN)rD   rE   rF   r   r_   rb   Ú
LongTensorrm   r   ÚFloatTensorrP   r   r   r   rh   rn   ro   s   @rU   r¦   r¦   ª   s®   ø„ ð
˜{õ 
ð .2Ø.2Ø04Ø(,Ø26Ø!%ñ2
à×#Ñ# dÑ*ð2
ð Ÿ™ tÑ+ð2
ð ×&Ñ&¨Ñ-ð	2
ð
  ™ð2
ð ×(Ñ(¨4Ñ/ð2
ð ˜$‘;ð2
ð Ð+Ñ,ð2
ð 
!÷2
rT   r¦   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚGemmaForCausalLMc                  ó8   •—  t        «       j                  di | ¤ŽS )a|  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, GemmaForCausalLM

        >>> model = GemmaForCausalLM.from_pretrained("google/gemma-7b")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-7b")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```rS   )r^   rh   )Úsuper_kwargsre   s    €rU   rh   zGemmaForCausalLM.forwardè   s   ø€ ð$ Œu‹w�‰Ñ. Ñ.Ð.rT   )rD   rE   rF   rh   rn   ro   s   @rU   rº   rº   ç   s   ø„ ÷/ð /rT   rº   c                   ó   — e Zd Zy)ÚGemmaForSequenceClassificationNr’   rS   rT   rU   r¾   r¾   ý   r“   rT   r¾   c                   ó   — e Zd Zy)ÚGemmaForTokenClassificationNr’   rS   rT   rU   rÀ   rÀ     r“   rT   rÀ   )r   r¦   rº   r¾   rÀ   r›   )6rb   Úhuggingface_hub.dataclassesr   r   Ú r   rŸ   Úcache_utilsr   r   Úconfiguration_utilsr	   Úmasking_utilsr
   Úmodeling_outputsr   Úmodeling_rope_utilsr   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úllama.modeling_llamar   r   r   r   r   r   r   r   ÚVOCAB_FILES_NAMESÚSPIECE_UNDERLINEÚ
get_loggerrD   Úloggerr   Ú	EmbeddingrW   ÚModulerq   r‡   r‘   r•   r›   r¦   rº   r¾   rÀ   Ú__all__rS   rT   rU   ú<module>rÓ      s#  ðó" Ý .Ý å &ß .Ý 3Ý /Ý 7Ý 1Ý -Ý &ß @Ñ @÷	÷ 	ó 	ð "Ð#4Ð5Ð àÐ à	ˆ×	Ñ	˜HÓ	%€ñ Ð,Ô-Øô34Ð"ó 34ó ó .ð34ôlS 2§<¡<ô Sô=�2—9‘9ô =ô(Yˆxô Yô	Ð/ô 	ôS�^ô SôJÐ/ô Jô:
�ô :
ôz/Ð'ô /ô,	Ð%Cô 	ô	Ð"=ô 	ò�rT   