Ë
    ÛÓ^jH  ã                   ó2  — d dl mZ d dlZd dlmZ d dlmZ ddlmZ ddl	m
Z
mZ ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZmZmZ ddlmZ ddlmZ ddl m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1  e#jd                  e3«      Z4 e"d¬«      e G d„ de«      «       «       Z5 G d„ de.«      Z6 G d„ de+«      Z7 G d„ de/«      Z8	 	 	 d6dejr                  dejt                  dejt                  d ejt                  d!ejt                  dz  d"e;e<z  d#e;dz  d$e;dz  d%e=ejt                  ejt                  f   fd&„Z> G d'„ d(e'«      Z? G d)„ d*e«      Z@ G d+„ d,e-«      ZA G d-„ d.e,«      ZB G d/„ d0e(«      ZC G d1„ d2e)«      ZD G d3„ d4e*«      ZEg d5¢ZFy)7é    )ÚCallableN)Ústricté   )ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚRopeParametersÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úmaybe_autocasté   )ÚGemmaAttentionÚGemmaForCausalLMÚGemmaForSequenceClassificationÚGemmaForTokenClassificationÚGemmaMLPÚ
GemmaModelÚGemmaPreTrainedModelÚGemmaRMSNormÚGemmaRotaryEmbeddingÚapply_rotary_pos_embÚ	repeat_kvzgoogle/gemma2-7b)Ú
checkpointc                   ó&  ‡ — e Zd ZU dZdZdgZddddddddœZdgdgfd	d
gd	gfd	gd	gfdœZdZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZe	e
d<   dZee
d<   dZe	e
d<   dZee
d<   d Zee
d!<   d"Zee
d#<   d$Ze	d%z  e
d&<   d'Ze	ee	   z  d%z  e
d(<   d)Ze	d%z  e
d*<   d"Zee
d+<   d%Zee z  d%z  e
d,<   d-Z!ee
d.<   d/Z"e	ez  d%z  e
d0<   dZ#e	e
d1<   d2Z$e	d%z  e
d3<   d%Z%ee   d%z  e
d4<   d5Z&ed%z  e
d6<   d7Z'ed%z  e
d8<   d%Z(ed%z  e
d9<   ˆ fd:„Z)d;„ Z*ˆ xZ+S )<ÚGemma2Configaœ  
    query_pre_attn_scalar (`float`, *optional*, defaults to 256):
        scaling factor used on the attention scores
    final_logit_softcapping (`float`, *optional*, defaults to 30.0):
        scaling factor when applying tanh softcapping on the logits.
    attn_logit_softcapping (`float`, *optional*, defaults to 50.0):
        scaling factor when applying tanh softcapping on the attention scores.
    use_bidirectional_attention (`bool`, *optional*):
        If True, the model will attend to all text tokens instead of using a causal mask.

    ```python
    >>> from transformers import Gemma2Model, Gemma2Config
    >>> # Initializing a Gemma2 gemma2-7b style configuration
    >>> configuration = Gemma2Config()
    >>> # Initializing a model from the gemma2-7b style configuration
    >>> model = Gemma2Model(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Úgemma2Úpast_key_valuesÚcolwiseÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormi è Ú
vocab_sizei 	  Úhidden_sizei $  Úintermediate_sizeé   Únum_hidden_layersé   Únum_attention_headsé   Únum_key_value_headsé   Úhead_dimÚgelu_pytorch_tanhÚhidden_activationi    Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeg�íµ ÷Æ°>Úrms_norm_epsTÚ	use_cacher   NÚpad_token_idé   Úeos_token_idr   Úbos_token_idÚtie_word_embeddingsÚrope_parametersFÚattention_biasç        Úattention_dropoutÚquery_pre_attn_scalari   Úsliding_windowÚlayer_typesg      >@Úfinal_logit_softcappingg      I@Úattn_logit_softcappingÚuse_bidirectional_attentionc                 ó¼   •— | j                   €;t        | j                  «      D �cg c]  }t        |dz   dz  «      rdnd‘Œ c}| _         t	        ‰| �  di |¤Ž y c c}w )NrE   r   Úsliding_attentionÚfull_attention© )rO   Úranger7   ÚboolÚsuperÚ__post_init__)ÚselfÚkwargsÚiÚ	__class__s      €út/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/gemma2/modular_gemma2.pyrZ   zGemma2Config.__post_init__y   s`   ø€ Ø×ÑÐ#äX]Ð^b×^tÑ^tÓXuö ØST¤t¨Q°©U°a©KÔ'8Ñ#Ð>NÑNò ˆDÔô 	‰ÑÑ' Ó'ùò	 s   ¥Ac                 óˆ   — | j                   | j                  z  dk7  r&t        d| j                   › d| j                  › d�«      ‚y)zOPart of `@strict`-powered validation. Validates the architecture of the config.r   zThe hidden size (z6) is not a multiple of the number of attention heads (z).N)r4   r9   Ú
ValueError)r[   s    r_   Úvalidate_architecturez"Gemma2Config.validate_architecture�   sS   € à×Ñ˜d×6Ñ6Ñ6¸!Ò;ÜØ# D×$4Ñ$4Ð#5ð 6Ø×2Ñ2Ð3°2ð7óð ð <ó    ),Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planr3   ÚintÚ__annotations__r4   r5   r7   r9   r;   r=   r?   Ústrr@   rA   ÚfloatrB   rC   rX   rD   rF   ÚlistrG   rH   rI   r   ÚdictrJ   rL   rM   rN   rO   rP   rQ   rR   rZ   rb   Ú__classcell__©r^   s   @r_   r'   r'   7   s¿  ø… ñð( €JØ#4Ð"5Ðà%.Ø%.Ø%.Ø%.Ø"+Ø )Ø"+ñÐð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð €J�ÓØ€K�ÓØ!Ð�sÓ!ØÐ�sÓØ Ð˜Ó Ø Ð˜Ó Ø€HˆcÓØ0Ð�sÓ0Ø#'Ð˜SÓ'Ø#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ €L�#˜‘*Ó Ø+,€L�#˜˜S™	‘/ DÑ(Ó,Ø €L�#˜‘*Ó Ø $Ð˜Ó$Ø48€O�^ dÑ*¨TÑ1Ó8Ø €N�DÓ Ø,/Ð�s˜U‘{ TÑ)Ó/Ø!$Ð˜3Ó$Ø!%€N�C˜$‘JÓ%Ø$(€K��c‘˜TÑ!Ó(Ø,0Ð˜U T™\Ó0Ø+/Ð˜E D™LÓ/Ø/3Ð ¨¡Ó3ô(örc   r'   c                   ó   — e Zd Zy)ÚGemma2RMSNormN©rd   re   rf   rV   rc   r_   ru   ru   Š   ó   „ Ørc   ru   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )Ú	Gemma2MLPc                 óT   •— t         ‰| �  |«       t        |j                     | _        y ©N)rY   Ú__init__r   r?   Úact_fn©r[   Úconfigr^   s     €r_   r|   zGemma2MLP.__init__�   s"   ø€ Ü‰Ñ˜Ô Ü˜V×5Ñ5Ñ6ˆ�rc   )rd   re   rf   r|   rr   rs   s   @r_   ry   ry   Ž   s   ø„ ÷7ð 7rc   ry   c                   óR   — e Zd Zddefd„Z ej                  «       ed„ «       «       Zy)ÚGemma2RotaryEmbeddingNr   c                 óÌ  — t         j                  j                  «        |j                  | _        |j                  | _        || _        | j                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j                  |«      \  }| _        | j                  d|d¬«       | j                  d|j                  «       d¬«       y )NÚ	rope_typeÚdefaultÚinv_freqF)Ú
persistentÚoriginal_inv_freq)ÚnnÚModuler|   r@   Úmax_seq_len_cachedÚoriginal_max_seq_lenr   rI   rƒ   Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)r[   r   ÚdeviceÚrope_init_fnr…   s        r_   r|   zGemma2RotaryEmbedding.__init__•   sº   € Ü
�	‰	×ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àˆŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÒ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆ�$Ô(à×Ñ˜Z¨¸eÐÔDØ×ÑÐ0°(·.±.Ó2BÈuÐÕUrc   c                 óN  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  ||fd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j	                  |j                   ¬
«      	j	                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   éÿÿÿÿrE   ÚmpsÚcpuF)Údevice_typeÚenabledr   )Údim)Údtype)r…   ro   ÚexpandÚshapeÚtor�   Ú
isinstanceÚtypern   r   Ú	transposeÚtorchÚcatÚcosr�   Úsinr™   )
r[   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr–   ÚfreqsÚembr¢   r£   s
             r_   ÚforwardzGemma2RotaryEmbedding.forward¥   sR  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   ÃBFÆF$r{   )	rd   re   rf   r'   r|   r    Úno_gradr   rª   rV   rc   r_   r�   r�   ”   s4   „ ñV˜|ó Vð  €U‡]�]ƒ_Øñ<ó ó ñ<rc   r�   ÚmoduleÚqueryÚkeyÚvaluer/   ÚdropoutÚscalingÚsoftcapÚreturnc                 ó|  — |€| j                   dz  }t        || j                  «      }	t        || j                  «      }
t        j                  ||	j                  dd«      «      |z  }|�||z  }t        j                  |«      }||z  }|�||z   }t        j                  j                  |dt        j                  ¬«      j                  |j                  «      }t        j                  j                  ||| j                  ¬«      }t        j                  ||
«      }|j                  dd«      j                  «       }||fS )Nç      à¿r   r   r“   )r˜   r™   )ÚpÚtrainingrE   )r=   r$   Únum_key_value_groupsr    ÚmatmulrŸ   Útanhrˆ   Ú
functionalÚsoftmaxÚfloat32rœ   r™   r°   r·   Ú
contiguous)r¬   r­   r®   r¯   r/   r°   r±   r²   r\   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs                r_   Úeager_attention_forwardrÃ   µ   s  € ð €Ø—/‘/ 4Ñ'ˆä˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LàÐØ# gÑ-ˆÜ—z‘z ,Ó/ˆØ# gÑ-ˆØÐ!Ø# nÑ4ˆô —=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØ˜Ð$Ð$rc   c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  de	dz  d	e
e   d
eej                  ej                  dz  eej                     dz  f   fd„Zˆ xZS )ÚGemma2Attentionr   Ú	layer_idxc                 ót  •— t        |d«      r|j                  |   nd | _        t        ‰| �  ||«       | j
                  j                  | _        | j
                  j                  | _        t        |dd«       | _	        |j                  dz  | _        | j                  dk(  r|j                  | _        y d | _        y )NrO   rR   Frµ   rT   )ÚhasattrrO   Ú
layer_typerY   r|   r   rQ   rL   ÚgetattrÚ	is_causalrM   r±   rN   ©r[   r   rÆ   r^   s      €r_   r|   zGemma2Attention.__init__Ø   sœ   ø€ Ü;BÀ6È=Ô;Y˜&×,Ñ,¨YÒ7Ð_cˆŒä‰Ñ˜ Ô+Ø&*§k¡k×&HÑ&HˆÔ#Ø!%§¡×!>Ñ!>ˆÔÜ$ VÐ-JÈEÓRÐRˆŒØ×3Ñ3°TÑ9ˆŒØ7;·±ÐJ]Ò7]˜f×3Ñ3ˆÕÐcgˆÕrc   Nr.   Úposition_embeddingsr/   r)   r\   r³   c                 ó6  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t        «      } || ||	|
|f| j                  r| j                   nd| j"                  | j$                  | j&                  dœ|¤Ž\  }} |j(                  g |¢d‘­Ž j+                  «       }| j-                  |«      }||fS )Nr“   rE   r   rK   )r°   r±   rN   r²   )r›   r=   Úq_projÚviewrŸ   Úk_projÚv_projr#   ÚupdaterÆ   r   Úget_interfacer   Ú_attn_implementationrÃ   r·   rL   r±   rN   rQ   Úreshaper¾   Úo_proj)r[   r.   rÍ   r/   r)   r\   Úinput_shapeÚhidden_shapeÚquery_statesr¿   rÀ   r¢   r£   Úattention_interfacerÂ   rÁ   s                   r_   rª   zGemma2Attention.forwardâ   s£  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð%
ð /3¯mªm�D×*Ò*ÀØ—L‘LØ×.Ñ.Ø×/Ñ/ñ%
ð ñ%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(rc   )NNN)rd   re   rf   r'   rl   r|   r    ÚTensorÚtupler   r   r   rª   rr   rs   s   @r_   rÅ   rÅ   ×   sº   ø„ ðh˜|ð h¸õ hð IMØ.2Ø(,ñ()à—|‘|ð()ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð()ð Ÿ™ tÑ+ð	()ð
  ™ð()ð Ð-Ñ.ð()ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷()rc   rÅ   c                   ó.  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	e
dz  d
eej                  eej                  ej                  f   dz  f   fd„Zˆ xZS )ÚGemma2DecoderLayerr   rÆ   c                 óÆ  •— t         ‰| �  «        |j                  | _        || _        t	        ||¬«      | _        t        |«      | _        t        |j                  |j                  ¬«      | _
        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        y )N)r   rÆ   )Úeps)rY   r|   r4   r   rÅ   Ú	self_attnry   Úmlpru   rB   Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormrÌ   s      €r_   r|   zGemma2DecoderLayer.__init__  sª   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØˆŒÜ(°À)ÔLˆŒÜ˜VÓ$ˆŒÜ,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔÜ(5°f×6HÑ6HÈf×NaÑNaÔ(bˆÔ%ä)6°v×7IÑ7IÈv×ObÑObÔ)cˆÔ&Ü*7¸×8JÑ8JÐPV×PcÑPcÔ*dˆÕ'rc   Nr.   rÍ   r/   r¥   r)   r³   c           	      ó  — |}| j                  |«      } | j                  d|||||dœ|¤Ž\  }}| j                  |«      }||z   }|}| j                  |«      }| j	                  |«      }| j                  |«      }||z   }|S )N)r.   rÍ   r/   r¥   r)   rV   )rä   râ   rå   ræ   rã   rç   )	r[   r.   rÍ   r/   r¥   r)   r\   ÚresidualÚ_s	            r_   rª   zGemma2DecoderLayer.forward  s­   € ð !ˆà×,Ñ,¨]Ó;ˆð *˜4Ÿ>™>ð 
Ø'Ø 3Ø)Ø%Ø+ñ
ð ñ
Ñˆ�qð ×5Ñ5°mÓDˆØ  =Ñ0ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ  =Ñ0ˆàÐrc   )NNNN)rd   re   rf   r'   rl   r|   r    rÜ   rÝ   Ú
LongTensorr   ÚFloatTensorrª   rr   rs   s   @r_   rß   rß     sÇ   ø„ ð
e˜|ð 
e¸õ 
eð IMØ.2Ø04Ø(,ñà—|‘|ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ÿ™ tÑ+ð	ð
 ×&Ñ&¨Ñ-ðð  ™ðð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷rc   rß   c                   ó   — e Zd Zy)ÚGemma2PreTrainedModelNrv   rV   rc   r_   rî   rî   <  rw   rc   rî   c                   óÔ   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	e
dz  d
ee   defd„Zˆ xZS )ÚGemma2Modelr   c           	      óØ   •— t         ‰| �  |«       t        j                  t	        |j
                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |«      | _	        y c c}w r{   )
rY   r|   rˆ   Ú
ModuleListrW   r7   rß   r1   r�   Ú
rotary_embrÌ   s      €r_   r|   zGemma2Model.__init__A  sU   ø€ Ü‰Ñ˜Ô Ü—m‘mÜDIÈ&×JbÑJbÓDcÖd°yÔ ¨	Õ2Òdó
ˆŒô 0°Ó7ˆ�ùò es   ·A'Nr,   r/   r¥   r)   r-   rC   r\   r³   c           	      óÆ  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      s)| j                  ||||dœ}
t        d
i |
¤Žt        d
i |
¤Ždœ}	|}| j                  ||«      }t        | j                   d | j                  j"                   «      D ]-  \  }} ||f|	| j                  j$                  |      |||dœ|¤Ž}Œ/ | j'                  |«      }t)        ||¬	«      S )Nz:You must specify exactly one of input_ids or inputs_embeds)r   r   rE   )r�   )r   r-   r/   r)   r¥   )rU   rT   )r/   rÍ   r¥   r)   )Úlast_hidden_stater)   rV   )ra   r0   r   r   Úget_seq_lengthr    Úaranger›   r�   Ú	unsqueezer�   rq   r
   r   ró   Ú	enumerater1   r7   rO   r2   r   )r[   r,   r/   r¥   r)   r-   rC   r\   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr.   rÍ   r]   Údecoder_layers                  r_   rª   zGemma2Model.forwardH  s–  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø*.×*;Ñ*;¸IÓ*FˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-ÄÔFð Ÿ+™+Ø!.Ø"0Ø#2Ø ,ñˆKô #5Ñ"C°{Ñ"CÜ%FÑ%UÈÑ%Uñ#Ðð &ˆØ"Ÿo™o¨m¸\ÓJÐä )¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÓ Wò 	ÑˆAˆ}Ù)Øðà2°4·;±;×3JÑ3JÈ1Ñ3MÑNØ$7Ø)Ø /ñð ñ‰Mð	ð Ÿ	™	 -Ó0ˆä&Ø+Ø+ô
ð 	
rc   )NNNNNN)rd   re   rf   r'   r|   r    rë   rÜ   r   rì   rX   r   r   r   rª   rr   rs   s   @r_   rð   rð   @  s®   ø„ ð8˜|õ 8ð .2Ø.2Ø04Ø(,Ø26Ø!%ñ;
à×#Ñ# dÑ*ð;
ð Ÿ™ tÑ+ð;
ð ×&Ñ&¨Ñ-ð	;
ð
  ™ð;
ð ×(Ñ(¨4Ñ/ð;
ð ˜$‘;ð;
ð Ð+Ñ,ð;
ð 
!÷;
rc   rð   c                   ó  ‡ — e Zd Zˆ fd„Z	 	 	 	 	 	 	 	 ddej
                  dz  dej                  dz  dej
                  dz  dedz  dej                  dz  dej
                  dz  d	e	dz  d
e
ej                  z  dee   defd„Zˆ xZS )ÚGemma2ForCausalLMc                 ód   •— t         ‰| �  |«       t        |«      | _        | j	                  «        y r{   )rY   r|   rð   ÚmodelÚ	post_initr~   s     €r_   r|   zGemma2ForCausalLM.__init__‡  s&   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø�‰Õrc   Nr,   r/   r¥   r)   r-   ÚlabelsrC   Úlogits_to_keepr\   r³   c	           
      ó  —  | j                   d||||||dœ|	¤Ž}
|
j                  }t        |t        «      rt	        | d«      n|}| j                  |dd…|dd…f   «      }| j                  j                  �G|| j                  j                  z  }t        j                  |«      }|| j                  j                  z  }d}|� | j                  ||| j                  fi |	¤Ž}t        |||
j                  |
j                  |
j                  ¬«      S )a‚  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, Gemma2ForCausalLM

        >>> model = Gemma2ForCausalLM.from_pretrained("google/gemma-2-9b")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-9b")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```)r,   r/   r¥   r)   r-   rC   N)ÚlossÚlogitsr)   r.   Ú
attentionsrV   )r  rõ   r�   rl   ÚsliceÚlm_headr   rP   r    rº   Úloss_functionr3   r   r)   r.   r  )r[   r,   r/   r¥   r)   r-   r  rC   r  r\   Úoutputsr.   Úslice_indicesr  r  s                  r_   rª   zGemma2ForCausalLM.forwardŒ  s  € ð< ,6¨4¯:©:ð ,
ØØ)Ø%Ø+Ø'Øñ,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆØ�;‰;×.Ñ.Ð:Ø˜dŸk™k×AÑAÑAˆFÜ—Z‘Z Ó'ˆFØ˜dŸk™k×AÑAÑAˆFàˆØÐØ%�4×%Ñ% f¨f°d·o±oÑPÈÑPˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
rc   )NNNNNNNr   )rd   re   rf   r|   r    rë   rÜ   r   rì   rX   rl   r   r   r   rª   rr   rs   s   @r_   rÿ   rÿ   †  sÕ   ø„ ôð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ;
à×#Ñ# dÑ*ð;
ð Ÿ™ tÑ+ð;
ð ×&Ñ&¨Ñ-ð	;
ð
  ™ð;
ð ×(Ñ(¨4Ñ/ð;
ð × Ñ  4Ñ'ð;
ð ˜$‘;ð;
ð ˜eŸl™lÑ*ð;
ð Ð+Ñ,ð;
ð 
 ÷;
rc   rÿ   c                   ó   — e Zd Zy)ÚGemma2ForSequenceClassificationNrv   rV   rc   r_   r  r  Ê  rw   rc   r  c                   ó   — e Zd Zy)ÚGemma2ForTokenClassificationNrv   rV   rc   r_   r  r  Î  rw   rc   r  )r'   rÿ   rð   rî   r  r  )rK   NN)GÚcollections.abcr   r    Útorch.nnrˆ   Úhuggingface_hub.dataclassesr   Úactivationsr   Úcache_utilsr   r   Úconfiguration_utilsr	   Úmasking_utilsr
   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úgemma.modeling_gemmar   r   r   r   r   r   r    r!   r"   r#   r$   Ú
get_loggerrd   Úloggerr'   ru   ry   r�   r‰   rÜ   ro   rl   rÝ   rÃ   rÅ   rß   rî   rð   rÿ   r  r  Ú__all__rV   rc   r_   ú<module>r%     sÅ  ðõ %ã Ý Ý .å !ß .Ý 3ß RÝ BÝ 9ß O÷ñ õ
 6Ý &ß @Ñ @Ý +÷÷ ÷ ñ ð 
ˆ×	Ñ	˜HÓ	%€ñ Ð-Ô.ØôNÐ#ó Nó ó /ðNôb	�Lô 	ô7�ô 7ô<Ð0ô <ðN Ø Ø ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �S‰[ð%ð �T‰\ð%ð �T‰\ð%ð ˆ5�<‰<˜Ÿ™Ð%Ñ&ó%ôD3)�nô 3)ôl,Ð3ô ,ô^	Ð0ô 	ôC
�*ô C
ôLA
Ð(ô A
ôH	Ð&Dô 	ô	Ð#>ô 	ò�rc   