Ë
    ÜÓ^jsí  ã                   ó€  — d dl Z d dlmZ d dlmZ d dlZd dlmZ ddlm	Z
 ddlmZ ddlmZmZmZmZ ddlmZmZmZ dd	lmZmZ dd
lmZmZmZmZ ddlmZ ddl m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0 ddl1m2Z2m3Z3m4Z4m5Z5 ddl6m7Z7m8Z8 ddl9m:Z:m;Z; ddl<m=Z=m>Z>  e5j~                  e@«      ZA G d„ dej„                  «      ZC G d„ dej„                  «      ZD G d„ dej„                  «      ZEd„ ZF ed«      dLd„«       ZGdej�                  d eId!ej�                  fd"„ZJ	 	 	 dMd#ej„                  d$ej�                  d%ej�                  d&ej�                  d'ej�                  dz  d(eKeIz  d)eKdz  d*eKdz  d!eLej�                  ej�                  f   fd+„ZM eeG«       G d,„ d-ej„                  «      «       ZN eeG«       G d.„ d/ej„                  «      «       ZO G d0„ d1e!«      ZP G d2„ d3e!«      ZQ G d4„ d5ej„                  «      ZR G d6„ d7ej„                  «      ZSe3 G d8„ d9e.«      «       ZTd:ejª                  dz  dej�                  d;eIdz  d!ej�                  fd<„ZV G d=„ d>eT«      ZW G d?„ d@eT«      ZXe3 G dA„ dBeT«      «       ZYe3 G dC„ dDeT«      «       ZZ G dE„ dFeTe«      Z[e3 G dG„ dHeT«      «       Z\e3 G dI„ dJeT«      «       Z]g dK¢Z^y)Né    N)ÚCallable)ÚOptionalé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCacheÚStaticCache)ÚGenerationConfigÚGenerationMixinÚGenerationMode)Úuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚ(create_bidirectional_sliding_window_maskÚcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚT5GemmaConfigÚT5GemmaModuleConfigc                   ó<   ‡ — e Zd Zddedefˆ fd„Zd„ Zd„ Zd„ Zˆ xZ	S )ÚT5GemmaRMSNormÚdimÚepsc                 óŠ   •— t         ‰| �  «        || _        t        j                  t        j                  |«      «      | _        y ©N)ÚsuperÚ__init__r0   ÚnnÚ	ParameterÚtorchÚzerosÚweight)Úselfr/   r0   Ú	__class__s      €úw/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/t5gemma/modeling_t5gemma.pyr4   zT5GemmaRMSNorm.__init__>   s.   ø€ Ü‰ÑÔØˆŒÜ—l‘l¤5§;¡;¨sÓ#3Ó4ˆ�ó    c                 óŒ   — |t        j                  |j                  d«      j                  dd¬«      | j                  z   «      z  S )Né   éÿÿÿÿT)Úkeepdim)r7   ÚrsqrtÚpowÚmeanr0   )r:   Úxs     r<   Ú_normzT5GemmaRMSNorm._normC   s4   € Ø”5—;‘;˜qŸu™u Q›xŸ}™}¨R¸˜}Ó>ÀÇÁÑIÓJÑJÐJr=   c                 ó¢   — | j                  |j                  «       «      }|d| j                  j                  «       z   z  }|j                  |«      S )Nç      ð?)rF   Úfloatr9   Útype_as)r:   rE   Úoutputs      r<   ÚforwardzT5GemmaRMSNorm.forwardF   sC   € Ø—‘˜AŸG™G›IÓ&ˆð ˜3 §¡×!2Ñ!2Ó!4Ñ4Ñ5ˆØ�~‰~˜aÓ Ð r=   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)Útupler9   Úshaper0   ©r:   s    r<   Ú
extra_reprzT5GemmaRMSNorm.extra_reprM   s'   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$·(±(°Ð<Ð<r=   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__ÚintrI   r4   rF   rL   rQ   Ú__classcell__©r;   s   @r<   r.   r.   =   s&   ø„ ñ5˜Cð 5 eõ 5ò
Kò!ö=r=   r.   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú
T5GemmaMLPc                 ó  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _	        t        |j                     | _        t        j                  |j                  «      | _        y )NF©Úbias)r3   r4   ÚconfigÚhidden_sizeÚintermediate_sizer5   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Úhidden_activationÚact_fnÚDropoutÚdropout_rateÚdropout©r:   r]   r;   s     €r<   r4   zT5GemmaMLP.__init__R   s¼   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×5Ñ5Ñ6ˆŒÜ—z‘z &×"5Ñ"5Ó6ˆ�r=   c                 ó®   — | j                  | j                  |«      «      | j                  |«      z  }| j                  |«      }| j	                  |«      }|S r2   )re   ra   rb   rh   rc   )r:   rE   Úhidden_statesrc   s       r<   rL   zT5GemmaMLP.forward]   sH   € ØŸ™ D§N¡N°1Ó$5Ó6¸¿¹Àa»ÑHˆØŸ™ ]Ó3ˆØ—N‘N =Ó1ˆ	ØÐr=   )rR   rS   rT   r4   rL   rV   rW   s   @r<   rY   rY   Q   s   ø„ ô	7ör=   rY   c                   óÄ   ‡ — e Zd ZU ej                  ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d   de
dz  ded	ef   fd
„«       Z ej                  «       ed„ «       «       Zˆ xZS )ÚT5GemmaRotaryEmbeddingÚinv_freqNr]   c                 ó®  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |«      \  }| _
        | j                  d|d¬«       | j                  d|j                  «       d¬«       y )NÚ	rope_typeÚdefaultrn   F)Ú
persistentÚoriginal_inv_freq)r3   r4   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr]   Úrope_parametersrp   Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)r:   r]   ÚdeviceÚrope_init_fnrn   r;   s        €r<   r4   zT5GemmaRotaryEmbedding.__init__g   s¶   ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àˆŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÒ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆ�$Ô(à×Ñ˜Z¨¸eÐÔDØ×ÑÐ0°(·.±.Ó2BÈuÐÕUr=   r|   ztorch.deviceÚseq_lenÚreturnztorch.Tensorc                 ó  — | j                   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNrH   r   r?   ©Údtype©r|   r„   )	rw   Úgetattrr^   Únum_attention_headsr7   ÚarangeÚint64ÚtorI   )r]   r|   r~   Úbaser/   Úattention_factorrn   s          r<   rx   z6T5GemmaRotaryEmbedding.compute_default_rope_parametersw   s�   € ð& ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r=   c                 óN  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  ||fd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j	                  |j                   ¬
«      	j	                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   r@   r*   ÚmpsÚcpuF)Údevice_typeÚenabledr?   ©r/   rƒ   )rn   rI   ÚexpandrO   rŠ   r|   Ú
isinstanceÚtypeÚstrr&   Ú	transposer7   ÚcatÚcosry   Úsinr„   )
r:   rE   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedr�   ÚfreqsÚembr™   rš   s
             r<   rL   zT5GemmaRotaryEmbedding.forward•   sR  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   ÃBFÆF$r2   ©NNN)rR   rS   rT   r7   ÚTensorÚ__annotations__r+   r4   Ústaticmethodr   rU   rN   rI   rx   Úno_gradr   rL   rV   rW   s   @r<   rm   rm   d   s˜   ø… Ø�l‰lÓñV˜}õ Vð  à'+Ø+/Ø"ñ*Ø Ñ$ð*à˜Ñ(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ñ	%ò	*ó ð*ð: €U‡]�]ƒ_Øñ<ó ó ô<r=   rm   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Nr@   r?   r’   )rO   r7   r˜   )rE   Úx1Úx2s      r<   Úrotate_halfr¨   ¥   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r=   Úrotary_pos_embc                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer¨   )ÚqÚkr™   rš   Úunsqueeze_dimÚq_embedÚk_embeds          r<   Úapply_rotary_pos_embr±   ¬   sY   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐr=   rk   Ún_repr   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r*   N)rO   r“   Úreshape)rk   r²   ÚbatchÚnum_key_value_headsÚslenr‚   s         r<   Ú	repeat_kvr¸   Æ   so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr=   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskrh   ÚscalingÚsoftcapc                 ó|  — |€| j                   dz  }t        || j                  «      }	t        || j                  «      }
t        j                  ||	j                  dd«      «      |z  }|�||z  }t        j                  |«      }||z  }|�||z   }t        j                  j                  |dt        j                  ¬«      j                  |j                  «      }t        j                  j                  ||| j                  ¬«      }t        j                  ||
«      }|j                  dd«      j                  «       }||fS )Nç      à¿r?   r   r@   )r/   r„   )ÚpÚtrainingr*   )r‚   r¸   Únum_key_value_groupsr7   Úmatmulr—   Útanhr5   Ú
functionalÚsoftmaxÚfloat32rŠ   r„   rh   rÃ   Ú
contiguous)r¹   rº   r»   r¼   r½   rh   r¾   r¿   ÚkwargsÚ
key_statesÚvalue_statesÚattn_weightsÚattn_outputs                r<   Úeager_attention_forwardrÐ   Ò   s  € ð €Ø—/‘/ 4Ñ'ˆä˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LàÐØ# gÑ-ˆÜ—z‘z ,Ó/ˆØ# gÑ-ˆØÐ!Ø# nÑ4ˆô —=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØ˜Ð$Ð$r=   c                   ó   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej                  de	ej                  ej                  f   dz  dej                  dz  d	e
dz  d
ee   de	ej                  ej                  dz  e	ej                     dz  f   fd„Zˆ xZS )ÚT5GemmaSelfAttentionú=Multi-headed attention from 'Attention Is All You Need' paperr]   Ú	layer_idxc                 óV  •— t         ‰| �  «        t        |d«      r|j                  |   nd | _        || _        || _        t        |d|j                  |j                  z  «      | _
        |j                  |j                  z  | _        |j                  dz  | _        | j
                  j                  | _        |j                   | _        t%        j&                  |j                  |j                  | j                  z  |j(                  ¬«      | _        t%        j&                  |j                  |j                  | j                  z  |j(                  ¬«      | _        t%        j&                  |j                  |j                  | j                  z  |j(                  ¬«      | _        t%        j&                  |j                  | j                  z  |j                  |j(                  ¬«      | _        | j
                  j2                  | _        | j                  dk(  r|j4                  | _        y d | _        y )NÚlayer_typesr‚   rÁ   r[   Úsliding_attention)r3   r4   ÚhasattrrÖ   Ú
layer_typer]   rÔ   r†   r^   r‡   r‚   r¶   rÄ   Úquery_pre_attn_scalarr¾   Úattention_dropoutÚ
is_decoderÚ	is_causalr5   r`   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚattn_logit_softcappingÚsliding_window©r:   r]   rÔ   r;   s      €r<   r4   zT5GemmaSelfAttention.__init__ø   s­  ø€ Ü‰ÑÔÜ;BÀ6È=Ô;Y˜&×,Ñ,¨YÒ7Ð_cˆŒØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø×3Ñ3°TÑ9ˆŒØ!%§¡×!>Ñ!>ˆÔà×*Ñ*ˆŒä—i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒð '+§k¡k×&HÑ&HˆÔ#Ø7;·±ÐJ]Ò7]˜f×3Ñ3ˆÕÐcgˆÕr=   Nrk   Úposition_embeddingsr½   Úpast_key_valuesrË   r   c                 ó6  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t        «      } || ||	|
|f| j                  r| j                   nd| j"                  | j$                  | j&                  dœ|¤Ž\  }} |j(                  g |¢d‘­Ž j+                  «       }| j-                  |«      }||fS )Nr@   r*   r?   ç        ©rh   r¾   rä   r¿   )rO   r‚   rß   Úviewr—   rà   rá   r±   ÚupdaterÔ   r   Úget_interfacer]   Ú_attn_implementationrÐ   rÃ   rÛ   r¾   rä   rã   r´   rÊ   râ   )r:   rk   ræ   r½   rç   rË   Úinput_shapeÚhidden_shapeÚquery_statesrÌ   rÍ   r™   rš   Úattention_interfacerÏ   rÎ   s                   r<   rL   zT5GemmaSelfAttention.forward  s£  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð%
ð /3¯mªm�D×*Ò*ÀØ—L‘LØ×.Ñ.Ø×/Ñ/ñ%
ð ñ%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r=   r    )rR   rS   rT   Ú__doc__r,   rU   r4   r7   r¡   rN   r   r!   r   rL   rV   rW   s   @r<   rÒ   rÒ   ô   s¾   ø„ áGðhÐ2ð h¸sõ hð< IMØ.2Ø(,ñ()à—|‘|ð()ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð()ð Ÿ™ tÑ+ð	()ð
  ™ð()ð Ð-Ñ.ð()ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷()r=   rÒ   c                   óþ   ‡ — e Zd ZdZdedefˆ fd„Z	 ddej                  dej                  dz  dej                  dz  d	e	dz  d
e
e   deej                  ej                  dz  eej                     dz  f   fd„Zˆ xZS )ÚT5GemmaCrossAttentionrÓ   r]   rÔ   c                 óÜ  •— t         ‰| �  «        || _        || _        t	        |d|j
                  |j                  z  «      | _        |j                  |j                  z  | _	        |j                  dz  | _        | j                  j                  | _        d| _        t        j                  |j
                  |j                  | j                  z  |j                   ¬«      | _        t        j                  |j$                  |j                  | j                  z  |j                   ¬«      | _        t        j                  |j$                  |j                  | j                  z  |j                   ¬«      | _        t        j                  |j                  | j                  z  |j
                  |j                   ¬«      | _        | j                  j,                  | _        |j$                  €t/        d«      ‚y )Nr‚   rÁ   Fr[   zBCross-attention needs cross_attention_hidden_size to be specified.)r3   r4   r]   rÔ   r†   r^   r‡   r‚   r¶   rÄ   rÚ   r¾   rÛ   rÝ   r5   r`   rÞ   rß   Úcross_attention_hidden_sizerà   rá   râ   rã   Ú
ValueErrorrå   s      €r<   r4   zT5GemmaCrossAttention.__init__B  s„  ø€ Ü‰ÑÔØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø×3Ñ3°TÑ9ˆŒØ!%§¡×!>Ñ!>ˆÔØˆŒä—i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×.Ñ.°×0JÑ0JÈTÏ]É]Ñ0ZÐag×avÑavô
ˆŒô —i‘iØ×.Ñ.°×0JÑ0JÈTÏ]É]Ñ0ZÐag×avÑavô
ˆŒô —i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒð '+§k¡k×&HÑ&HˆÔ#à×-Ñ-Ð5ÜÐaÓbÐbð 6r=   Nrk   r½   Úencoder_hidden_statesrç   rË   r   c                 ó‚  — |€t        d«      ‚|j                  d d }g |¢d‘| j                  ‘­}| j                  |«      j	                  |«      j                  dd«      }|�1|j                  j                  | j                  «      }	|j                  }
|�	s¾|j                  d d }g |¢d‘| j                  ‘­}| j                  |«      j	                  |«      j                  dd«      }| j                  |«      j	                  |«      j                  dd«      }|�€
j                  ||| j                  «      \  }}d|j                  | j                  <   nF
j                  | j                     j                  }|
j                  | j                     j                  }t!        j"                  | j$                  j&                  t(        «      } || ||||f| j*                  r| j,                  nd| j.                  d | j0                  dœ|¤Ž\  }} |j2                  g |¢d‘­Ž j5                  «       }| j7                  |«      }||fS )Nz5Encoder hidden state is required for cross attention.r@   r*   r?   Tré   rê   )rø   rO   r‚   rß   rë   r—   Ú
is_updatedÚgetrÔ   Úcross_attention_cacherà   rá   rì   ÚlayersÚkeysÚvaluesr   rí   r]   rî   rÐ   rÃ   rÛ   r¾   rã   r´   rÊ   râ   )r:   rk   r½   rù   rç   rË   rï   rð   rñ   rû   Úcurr_past_key_valuesÚencoder_input_shapeÚencoder_hidden_shaperÌ   rÍ   rò   rÏ   rÎ   s                     r<   rL   zT5GemmaCrossAttention.forward^  s?  € ð !Ð(ÜÐTÓUÐUà#×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆØ—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆàÐ&Ø(×3Ñ3×7Ñ7¸¿¹ÓGˆJØ#2×#HÑ#HÐ àÐ"©*Ø"7×"=Ñ"=¸c¸rÐ"BÐØ#LÐ%8Ð#L¸"Ð#L¸d¿m¹mÑ#LÐ ØŸ™Ð%:Ó;×@Ñ@ÐAUÓV×`Ñ`ÐabÐdeÓfˆJØŸ;™;Ð'<Ó=×BÑBÐCWÓX×bÑbÐcdÐfgÓhˆLàÐ*Ø+?×+FÑ+FÀzÐS_Ðae×aoÑaoÓ+pÑ(�
˜LØ=A�×*Ñ*¨4¯>©>Ò:à-×4Ñ4°T·^±^ÑD×IÑIˆJØ/×6Ñ6°t·~±~ÑF×MÑMˆLä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð%
ð /3¯mªm�D×*Ò*ÀØ—L‘LØØ×/Ñ/ñ%
ð ñ%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r=   r2   )rR   rS   rT   ró   r,   rU   r4   r7   r¡   r   r!   r   rN   rL   rV   rW   s   @r<   rõ   rõ   >  s©   ø„ áGðcÐ2ð c¸sõ cðB )-ñ3)à—|‘|ð3)ð Ÿ™ tÑ+ð3)ð  %Ÿ|™|¨dÑ2ð	3)ð
  ™ð3)ð Ð-Ñ.ð3)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷3)r=   rõ   c                   óè   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	eej                  f   f
d
„Zˆ xZS )ÚT5GemmaEncoderLayerzEncoder sub-layer.rÔ   c                 óD  •— t         ‰| �  «        |j                  | _        || _        || _        |j
                  |   | _        t        ||¬«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        |«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t#        j$                  |j&                  «      | _        y ©N)r]   rÔ   ©r0   )r3   r4   r^   r]   rÔ   rÖ   Úattention_typerÒ   Ú	self_attnr.   Úrms_norm_epsÚpre_self_attn_layernormÚpost_self_attn_layernormrY   ÚmlpÚpre_feedforward_layernormÚpost_feedforward_layernormr5   rf   rg   rh   rå   s      €r<   r4   zT5GemmaEncoderLayer.__init__—  sá   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØˆŒØ"ˆŒØ$×0Ñ0°Ñ;ˆÔä-ØØô
ˆŒô (6°f×6HÑ6HÈf×NaÑNaÔ'bˆÔ$Ü(6°v×7IÑ7IÈv×ObÑObÔ(cˆÔ%ä˜fÓ%ˆŒÜ)7¸×8JÑ8JÐPV×PcÑPcÔ)dˆÔ&Ü*8¸×9KÑ9KÐQW×QdÑQdÔ*eˆÔ'ä—z‘z &×"5Ñ"5Ó6ˆ�r=   Nrk   ræ   r½   r›   r   c           	      ó>  — |}| j                  |«      } | j                  d||||d dœ|¤Ž\  }}| j                  |«      }|| j                  |«      z   }|}| j	                  |«      }| j                  |«      }| j                  |«      }|| j                  |«      z   }|S )N)rk   ræ   r½   r›   rç   © )r  r
  r  rh   r  r  r  )r:   rk   ræ   r½   r›   rË   ÚresidualÚ_s           r<   rL   zT5GemmaEncoderLayer.forward«  s½   € ð !ˆØ×4Ñ4°]ÓCˆØ)˜4Ÿ>™>ð 
Ø'Ø 3Ø)Ø%Ø ñ
ð ñ
Ñˆ�qð ×5Ñ5°mÓDˆØ  4§<¡<°Ó#>Ñ>ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ  4§<¡<°Ó#>Ñ>ˆØÐr=   r    )rR   rS   rT   ró   rU   r4   r7   r¡   rN   Ú
LongTensorÚFloatTensorrL   rV   rW   s   @r<   r  r  ”  s‘   ø„ Ùð7¨#õ 7ð. IMØ.2Ø04ñà—|‘|ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ÿ™ tÑ+ð	ð
 ×&Ñ&¨Ñ-ðð 
ˆu× Ñ Ð!Ñ	"÷r=   r  c                   ó8  ‡ — e Zd ZdZdefˆ fd„Z	 	 	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	e
dz  d
edz  dej                  dz  dej                  dz  dej                  fd„Zˆ xZS )ÚT5GemmaDecoderLayerz2Decoder sub-layer: an extra cross-attention layer.rÔ   c                 ó   •— t         ‰| �  «        |j                  | _        || _        || _        |j
                  |   | _        t        ||¬«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        |«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t#        j$                  |j&                  «      | _        t+        ||¬«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        y r  )r3   r4   r^   r]   rÔ   rÖ   r	  rÒ   r
  r.   r  r  r  rY   r  r  r  r5   rf   rg   rh   rõ   Ú
cross_attnÚpre_cross_attn_layernormÚpost_cross_attn_layernormrå   s      €r<   r4   zT5GemmaDecoderLayer.__init__Ë  s&  ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØˆŒØ"ˆŒØ$×0Ñ0°Ñ;ˆÔä-ØØô
ˆŒô (6°f×6HÑ6HÈf×NaÑNaÔ'bˆÔ$Ü(6°v×7IÑ7IÈv×ObÑObÔ(cˆÔ%ä˜fÓ%ˆŒÜ)7¸×8JÑ8JÐPV×PcÑPcÔ)dˆÔ&Ü*8¸×9KÑ9KÐQW×QdÑQdÔ*eˆÔ'ä—z‘z &×"5Ñ"5Ó6ˆŒÜ/°vÈÔSˆŒÜ(6°v×7IÑ7IÈv×ObÑObÔ(cˆÔ%Ü)7¸×8JÑ8JÐPV×PcÑPcÔ)dˆÕ&r=   Nrk   ræ   r½   r›   rç   Ú	use_cacherù   Úencoder_attention_maskr   c	           
      ó  — |}
| j                  |«      } | j                  d|||||�|j                  nd |dœ|	¤Ž\  }}| j                  |«      }|
| j	                  |«      z   }|}
| j                  |«      } | j                  d|||||dœ|	¤Ž\  }}| j                  |«      }|
| j	                  |«      z   }|}
| j                  |«      }| j                  |«      }| j                  |«      }|
| j	                  |«      z   }|S )N)rk   ræ   r½   r›   rç   r  )rk   rù   r½   rç   r  r  )r  r
  Úself_attention_cacher  rh   r  r  r  r  r  r  )r:   rk   ræ   r½   r›   rç   r  rù   r  rË   r  r  s               r<   rL   zT5GemmaDecoderLayer.forwardâ  s;  € ð !ˆØ×4Ñ4°]ÓCˆØ)˜4Ÿ>™>ð 
Ø'Ø 3Ø)Ø%ØDSÐD_˜O×@Ò@ÐeiØñ
ð ñ
Ñˆ�qð ×5Ñ5°mÓDˆØ  4§<¡<°Ó#>Ñ>ˆà ˆØ×5Ñ5°mÓDˆØ*˜4Ÿ?™?ð 
Ø'Ø"7Ø1Ø+Øñ
ð ñ
Ñˆ�qð ×6Ñ6°}ÓEˆØ  4§<¡<°Ó#>Ñ>ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ  4§<¡<°Ó#>Ñ>ˆØÐr=   )NNNNFNN)rR   rS   rT   ró   rU   r4   r7   r¡   rN   r  r
   Úboolr  rL   rV   rW   s   @r<   r  r  È  sÚ   ø„ Ù<ðe¨#õ eð4 IMØ.2Ø04Ø6:Ø!&Ø59Ø6:ñ,à—|‘|ð,ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð,ð Ÿ™ tÑ+ð	,ð
 ×&Ñ&¨Ñ-ð,ð -¨tÑ3ð,ð ˜$‘;ð,ð  %Ÿ|™|¨dÑ2ð,ð !&§¡¨tÑ 3ð,ð 
×	Ñ	÷,r=   r  c                   ój   ‡ — e Zd ZdZd	dededefˆ fd„Zdej                  dej                  fd„Z	ˆ xZ
S )
ÚT5GemmaClassificationHeadz-Head for sentence-level classification tasks.r^   Ú
num_labelsÚclassifier_dropout_ratec                 óŽ   •— t         ‰| �  «        t        j                  |¬«      | _        t        j
                  ||«      | _        y )N)rÂ   )r3   r4   r5   rf   rh   r`   Úout_proj)r:   r^   r$  r%  r;   s       €r<   r4   z"T5GemmaClassificationHead.__init__  s1   ø€ Ü‰ÑÔÜ—z‘zÐ$;Ô<ˆŒÜŸ	™	 +¨zÓ:ˆ�r=   rk   r   c                 óJ   — | j                  |«      }| j                  |«      }|S r2   )rh   r'  )r:   rk   s     r<   rL   z!T5GemmaClassificationHead.forward  s$   € ØŸ™ ]Ó3ˆØŸ™ mÓ4ˆØÐr=   )ré   )rR   rS   rT   ró   rU   rI   r4   r7   r¡   rL   rV   rW   s   @r<   r#  r#    s<   ø„ Ù7ñ; Cð ;°Sð ;ÐSXõ ;ð
 U§\¡\ð °e·l±l÷ r=   r#  c                   ój   ‡ — e Zd ZdZd	dededefˆ fd„Zdej                  dej                  fd„Z	ˆ xZ
S )
ÚT5GemmaLMHeadz.Head for language modeling (generation) tasks.r^   Ú
vocab_sizer\   c                 ó\   •— t         ‰| �  «        t        j                  |||¬«      | _        y )Nr[   )r3   r4   r5   r`   r'  )r:   r^   r+  r\   r;   s       €r<   r4   zT5GemmaLMHead.__init__"  s"   ø€ Ü‰ÑÔÜŸ	™	 +¨zÀÔEˆ�r=   rk   r   c                 ó(   — | j                  |«      }|S r2   )r'  )r:   rk   Úlogitss      r<   rL   zT5GemmaLMHead.forward&  s   € Ø—‘˜}Ó-ˆØˆr=   )F)rR   rS   rT   ró   rU   r!  r4   r7   r¡   rL   rV   rW   s   @r<   r*  r*    s?   ø„ Ù8ñF Cð F°Sð FÀõ Fð U§\¡\ð °e·l±l÷ r=   r*  c                   ó†   ‡ — e Zd ZU eed<   dZdZddgZdgZdZ	dZ
dZdZdZdZ ej                   «       ˆ fd„«       Zd	„ Zˆ xZS )
ÚT5GemmaPreTrainedModelr]   ÚmodelTr  r  rç   Nc                 ó4  •— t         ‰| �  |«       | j                  j                  }t	        |t
        «      r­|j                  j                  j                  d   dz  }t        j                  |j                  j                  d||z  ¬«       t        |j                  d«      rA|j                  j                  �*t        j                  |j                  j                  «       y y y t	        |t        «      rm| j                  j                  sV|j                  j                  j                  d   dz  }t        j                  |j                  j                  d||z  ¬«       y y d|j                   j"                  v r t        j                  |j                  «       y y )Nr   rÁ   ré   )rD   Ústdr\   ÚRMSNorm)r3   Ú_init_weightsr]   Úinitializer_ranger”   r#  r'  r9   rO   ÚinitÚnormal_rØ   r\   Úzeros_r*  Útie_word_embeddingsr;   rR   )r:   r¹   r3  Úscaler;   s       €r<   r5  z$T5GemmaPreTrainedModel._init_weights;  s)  ø€ ô 	‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fÔ7Ô8Ø—O‘O×*Ñ*×0Ñ0°Ñ3°tÑ;ˆEÜ�L‰L˜Ÿ™×/Ñ/°c¸sÀU¹{ÕKÜ�v—‘¨Ô/°F·O±O×4HÑ4HÐ4TÜ—‘˜FŸO™O×0Ñ0Õ1ð 5UÐ/ä˜¤Ô.Ø—;‘;×2Ò2ØŸ™×.Ñ.×4Ñ4°QÑ7¸4Ñ?�Ü—‘˜VŸ_™_×3Ñ3¸#À3ÈÁ;ÖOð 3ð ˜&×*Ñ*×3Ñ3Ñ3Ü�K‰K˜Ÿ™Õ&ð 4r=   c                 ó`  — | j                   j                  j                  }| j                   j                  j                  }|€t	        d«      ‚|j                  |j                  «      }|ddd…f   j                  «       |ddd…f<   ||d<   |€t	        d«      ‚|j                  |dk(  |«       |S )	zú
        Shifts input_ids to the right, prepends the decoder_start_token_id, and handles
        pad_token_id replacement for labels that were -100.
        This is a common preparation step for decoder inputs in sequence-to-sequence models.
        Nz:self.model.config.decoder.bos_token_id has to be defined. .r@   r*   ).r   z9self.model.config.decoder.pad_token_id has to be defined.iœÿÿÿ)	r]   ÚdecoderÚbos_token_idÚpad_token_idrø   Ú	new_zerosrO   r{   Úmasked_fill_)r:   Ú	input_idsÚdecoder_start_token_idr?  Úshifted_input_idss        r<   Ú_shift_rightz#T5GemmaPreTrainedModel._shift_rightM  sº   € ð "&§¡×!4Ñ!4×!AÑ!AÐØ—{‘{×*Ñ*×7Ñ7ˆà!Ð)ÜÐYÓZÐZð &×/Ñ/°	·±Ó@ÐØ%.¨s°C°R°C¨xÑ%8×%>Ñ%>Ó%@Ð˜#˜q™r˜'Ñ"Ø$:Ð˜&Ñ!àÐÜÐXÓYÐYð 	×&Ñ&Ð'8¸DÑ'@À,ÔOà Ð r=   )rR   rS   rT   r+   r¢   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendÚ_can_record_outputsr7   r¤   r5  rE  rV   rW   s   @r<   r0  r0  +  sl   ø… àÓØÐØ&*Ð#Ø.Ð0EÐFÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&ÐàÐà€U‡]�]ƒ_ó'ó ð'ö"!r=   r0  Ú	token_idsr?  c                 ó  — | �<|€t        d«      ‚| |k7  j                  |j                  t        j                  «      }|S t        j
                  |j                  d   |j                  d   f|j                  t        j                  ¬«      }|S )z%Construct the default attention mask.z3`pad_token_id` is required for padding information.r   r*   r…   )rø   rŠ   r|   r7   ÚlongÚonesrO   )rP  rk   r?  r½   s       r<   Úmake_default_2d_attention_maskrT  h  sŒ   € ð ÐØÐÜÐRÓSÐSØ# |Ñ3×7Ñ7¸×8LÑ8LÌeÏjÉjÓYˆð
 Ðô Ÿ™Ø× Ñ  Ñ# ]×%8Ñ%8¸Ñ%;Ð<À]×EYÑEYÔaf×akÑakô
ˆð Ðr=   c                   óÚ   ‡ — e Zd ZeedœZˆ fd„Zee	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  dee   d	eez  fd
„«       «       Zˆ xZS )ÚT5GemmaEncoder)Ú
attentionsrk   c           	      óT  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        |j                  |j                  ¬«      | _        d| _        t        j                  t        |j                  «      D �cg c]  }t!        ||«      ‘Œ c}«      | _        t        j$                  |j&                  «      | _        t+        |¬«      | _        | j/                  «        y c c}w ©Nr  F©r]   )r3   r4   r?  Úpadding_idxr+  r5   Ú	Embeddingr^   Úembed_tokensr.   r  ÚnormÚgradient_checkpointingÚ
ModuleListÚrangeÚnum_hidden_layersr  rþ   rf   rg   rh   rm   Ú
rotary_embÚ	post_initrå   s      €r<   r4   zT5GemmaEncoder.__init__  óÜ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ" 6×#5Ñ#5¸6×;NÑ;NÔOˆŒ	Ø&+ˆÔ#ä—m‘mÜEJÈ6×KcÑKcÓEdÖe¸	Ô  ¨Õ3Òeó
ˆŒô —z‘z &×"5Ñ"5Ó6ˆŒÜ0¸Ô?ˆŒð 	�‰Õùò fó   Ã D%NrB  r½   r›   Úinputs_embedsrË   r   c                 ó‚  — |d u |d uz  rt        d«      ‚|j                  dd «       |€| j                  |«      }|€?t        j                  |j
                  d   |j                  ¬«      }|j                  d«      }|€!t        ||| j                  j                  «      }t        |x}t        «      s'| j                  ||dœ}t        di |¤Žt        di |¤Ždœ}|}t        j                  | j                  j                   dz  |j"                  ¬	«      }	||	z  }| j%                  |«      }| j'                  ||«      }
t)        | j*                  d | j                  j,                   «      D ]+  \  }} |||
|| j                  j.                  |      |fi |¤Ž}Œ- | j1                  |«      }| j%                  |«      }t3        |¬
«      S )Nú:You must specify exactly one of input_ids or inputs_embedsrç   r*   ©r|   r   )r]   rg  r½   ©Úfull_attentionr×   ç      à?rƒ   )Úlast_hidden_stater  )rø   Úpopr]  r7   rˆ   rO   r|   r«   rT  r]   r?  r”   Údictr   r   Útensorr^   r„   rh   rc  Ú	enumeraterþ   rb  rÖ   r^  r   )r:   rB  r½   r›   rg  rË   Úself_attn_mask_mappingÚmask_kwargsrk   Ú
normalizerræ   ÚiÚlayer_modules                r<   rL   zT5GemmaEncoder.forward‘  sË  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZð 	�
‰
Ð$ dÔ+àÐ Ø ×-Ñ-¨iÓ8ˆMàÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\ˆLØ'×1Ñ1°!Ó4ˆLàÐ!Ü;¸IÀ}ÐVZ×VaÑVa×VnÑVnÓoˆNä°NÐBÐ0ÄDÔIàŸ+™+Ø!.Ø"0ñˆKô #<Ñ"J¸kÑ"JÜ%MÑ%\ÐP[Ñ%\ñ&Ð"ð
 &ˆÜ—\‘\ $§+¡+×"9Ñ"9¸3Ñ">Àm×FYÑFYÔZˆ
Ø%¨
Ñ2ˆØŸ™ ]Ó3ˆà"Ÿo™o¨m¸\ÓJÐä(¨¯©Ð5T°t·{±{×7TÑ7TÐ)UÓVò 	‰OˆAˆ|Ù(ØØ#Ø& t§{¡{×'>Ñ'>¸qÑ'AÑBØñ	ð
 ñ‰Mð	ð Ÿ	™	 -Ó0ˆØŸ™ ]Ó3ˆÜØ+ô
ð 	
r=   ©NNNN)rR   rS   rT   rÒ   r  rO  r4   r'   r)   r7   r  r¡   r  r!   r"   rN   r   rL   rV   rW   s   @r<   rV  rV  y  s¯   ø„ à*Ø,ñÐô
ð$  Øð .2Ø.2Ø04Ø26ñ6
à×#Ñ# dÑ*ð6
ð Ÿ™ tÑ+ð6
ð ×&Ñ&¨Ñ-ð	6
ð
 ×(Ñ(¨4Ñ/ð6
ð Ð+Ñ,ð6
ð 
�Ñ	 ò6
ó ó  ô6
r=   rV  c                   óT  ‡ — e Zd Z eed¬«       eed¬«      edœZˆ fd„Ze	e
	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  d	edz  d
ej                  dz  dedz  dej                  dz  dej                  dz  dee   deez  fd„«       «       Zˆ xZS )ÚT5GemmaDecoderr*   )Úindex)rW  Úcross_attentionsrk   c           	      óT  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        |j                  |j                  ¬«      | _        d| _        t        j                  t        |j                  «      D �cg c]  }t!        ||«      ‘Œ c}«      | _        t        j$                  |j&                  «      | _        t+        |¬«      | _        | j/                  «        y c c}w rY  )r3   r4   r?  r[  r+  r5   r\  r^   r]  r.   r  r^  r_  r`  ra  rb  r  rþ   rf   rg   rh   rm   rc  rd  rå   s      €r<   r4   zT5GemmaDecoder.__init__Ó  re  rf  NrB  r½   r›   rç   rg  r  rù   r  rË   r   c	                 ó¤  — |d u |d uz  rt        d«      ‚|€t        d«      ‚|€| j                  |«      }| j                  s,|r*|€(t        t	        | j
                  ¬«      t	        «       «      }|€V|�|j                  «       nd}
t        j                  |j                  d   |j                  ¬«      |
z   }|j                  d«      }|€#|€!t        ||| j
                  j                  «      }t        |x}t        «      s7| j
                  |||�|j                   nd |dœ}t#        di |¤Žt%        di |¤Ždœ}t        |x}t        «      sd	t'        | j
                  |||¬
«      i}|}t        j(                  | j
                  j*                  dz  |j,                  ¬«      }||z  }| j/                  |«      }| j1                  ||«      }t3        | j4                  d | j
                  j6                   «      D ]2  \  }} ||||| j
                  j8                  |      |||||d	   fi |	¤Ž}Œ4 | j;                  |«      }| j/                  |«      }t=        ||¬«      S )Nri  z0`encoder_hidden_states` must be given in decoderrZ  r   r*   rj  )r]   rg  r½   rç   r›   rk  rl  )r]   rg  r½   rù   rm  rƒ   )rn  rç   r  )rø   r]  rÃ   r
   r	   r]   Úget_seq_lengthr7   rˆ   rO   r|   r«   rT  r?  r”   rp  r   r   r   r   rq  r^   r„   rh   rc  rr  rþ   rb  rÖ   r^  r   )r:   rB  r½   r›   rç   rg  r  rù   r  rË   Úpast_seen_tokensrs  rt  Úcross_attn_mask_mappingrk   ru  ræ   rv  rw  s                      r<   rL   zT5GemmaDecoder.forwardå  su  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZØ Ð(ÜÐOÓPÐPàÐ Ø ×-Ñ-¨iÓ8ˆMà�}Š}¡¨Ð/Fô 2´,ÀdÇkÁkÔ2RÔT`ÓTbÓcˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLàÐ! oÐ&=Ü;¸IÀ}ÐVZ×VaÑVa×VnÑVnÓoˆNä°NÐBÐ0ÄDÔIàŸ+™+Ø!.Ø"0ØKZÐKf ?×#GÒ#GÐlpØ ,ñˆKô #5Ñ"C°{Ñ"CÜ%FÑ%UÈÑ%Uñ&Ð"ô
 Ð5KÐKÐ1ÌTÔRà Ô";ØŸ;™;Ø"/Ø#9Ø*?ô	#ð'Ð#ð &ˆÜ—\‘\ $§+¡+×"9Ñ"9¸3Ñ">Àm×FYÑFYÔZˆ
Ø%¨
Ñ2ˆØŸ™ ]Ó3ˆà"Ÿo™o¨m¸\ÓJÐä(¨¯©Ð5T°t·{±{×7TÑ7TÐ)UÓVò 	‰OˆAˆ|Ù(ØØ#Ø& t§{¡{×'>Ñ'>¸qÑ'AÑBØØØØ%Ø'Ð(8Ñ9ñ
ð ñ
‰Mð	ð Ÿ	™	 -Ó0ˆØŸ™ ]Ó3ˆÜ8Ø+Ø+ô
ð 	
r=   )NNNNNNNN)rR   rS   rT   r(   rÒ   rõ   r  rO  r4   r'   r)   r7   r  r¡   r
   r  r!  r!   r"   rN   r   rL   rV   rW   s   @r<   rz  rz  Ì  s  ø„ á$Ð%9ÀÔCÙ*Ð+@ÈÔJØ,ñÐôð$  Øð .2Ø.2Ø04Ø6:Ø26Ø!%Ø59Ø6:ñP
à×#Ñ# dÑ*ðP
ð Ÿ™ tÑ+ðP
ð ×&Ñ&¨Ñ-ð	P
ð
 -¨tÑ3ðP
ð ×(Ñ(¨4Ñ/ðP
ð ˜$‘;ðP
ð  %Ÿ|™|¨dÑ2ðP
ð !&§¡¨tÑ 3ðP
ð Ð+Ñ,ðP
ð 
Ð:Ñ	:òP
ó ó  ôP
r=   rz  c                   ó€  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  dedz  dedz  de	j                  dz  de	j                  dz  dedz  dee   defd„«       «       Zˆ xZS )ÚT5GemmaModelr]   c                 óÚ   •— t         ‰| �  |«       |j                  st        d«      ‚t	        |j
                  «      | _        t        |j                  «      | _        | j                  «        y )NzVT5GemmaModel only support encoder-decoder modeling. Use `T5GemmaEncoderModel` instead.)	r3   r4   Úis_encoder_decoderrø   rV  Úencoderrz  r=  rd  ri   s     €r<   r4   zT5GemmaModel.__init__<  sO   ø€ Ü‰Ñ˜Ô à×(Ò(ÜÐuÓvÐvä% f§n¡nÓ5ˆŒÜ% f§n¡nÓ5ˆŒà�‰Õr=   c                 ó6   — | j                   j                  «       S r2   ©r†  Úget_input_embeddingsrP   s    r<   r‰  z!T5GemmaModel.get_input_embeddingsG  ó   € Ø�|‰|×0Ñ0Ó2Ð2r=   c                 ó8   — | j                   j                  |«      S r2   ©r†  Úset_input_embeddings©r:   Únew_embeddingss     r<   r�  z!T5GemmaModel.set_input_embeddingsJ  ó   € Ø�|‰|×0Ñ0°Ó@Ð@r=   NrB  r½   r›   Údecoder_input_idsÚdecoder_attention_maskÚdecoder_position_idsÚencoder_outputsrç   rg  Údecoder_inputs_embedsr  rË   r   c                 ó†  — |€ | j                   d||||	dœ|¤Ž}|j                  } | j                  d||||
||||dœ|¤Ž}t        |j                  |j                  |j                  dd«      r|j                  n|j                  f|j                  |j                  |j                  |j                  |j                  ¬«      S )aX  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        ©rB  r½   r›   rg  )rB  r½   r›   rg  rç   rù   r  r  Úoutput_hidden_statesF)rn  rç   Údecoder_hidden_statesÚdecoder_attentionsr|  Úencoder_last_hidden_staterù   Úencoder_attentionsr  )	r†  rn  r=  r   rç   rü   rk   rW  r|  )r:   rB  r½   r›   r‘  r’  r“  r”  rç   rg  r•  r  rË   rù   Údecoder_outputss                  r<   rL   zT5GemmaModel.forwardM  sî   € ð, Ð"Ø*˜dŸl™lð Ø#Ø-Ø)Ø+ñ	ð
 ñˆOð !0× AÑ AÐà&˜$Ÿ,™,ð 

Ø'Ø1Ø-Ø/Ø+Ø"7Ø#1Øñ

ð ñ

ˆô "Ø-×?Ñ?Ø+×;Ñ;à�z‰zÐ0°%Ô8ð #2×"?Ò"?à!×3Ñ3Ð5Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô
ð 	
r=   )NNNNNNNNNNN)rR   rS   rT   r+   r4   r‰  r�  r$   r#   r7   r  r  Ú
BoolTensorr   r
   r¡   r!  r!   r"   r   rL   rV   rW   s   @r<   rƒ  rƒ  :  sA  ø„ ð	˜}õ 	ò3òAð Øð .2Ø37Ø04Ø59Ø:>Ø8<Ø26Ø6:Ø-1Ø59Ø!%ñ6
à×#Ñ# dÑ*ð6
ð ×)Ñ)¨DÑ0ð6
ð ×&Ñ&¨Ñ-ð	6
ð
 !×+Ñ+¨dÑ2ð6
ð !&× 0Ñ 0°4Ñ 7ð6
ð $×.Ñ.°Ñ5ð6
ð )¨4Ñ/ð6
ð -¨tÑ3ð6
ð —|‘| dÑ*ð6
ð  %Ÿ|™|¨dÑ2ð6
ð ˜$‘;ð6
ð Ð+Ñ,ð6
ð 
ò6
ó ó ô6
r=   rƒ  c                   óÜ   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
ee   defd„«       «       Zˆ xZS )ÚT5GemmaEncoderModelr]   c                 ó¦   •— t         ‰| �  |«       |j                  rt        d«      ‚t	        |j
                  «      | _        | j                  «        y )NzQT5GemmaEncoderModel only supports encoder-only model. Use `T5GemmaModel` instead.)r3   r4   r…  rø   rV  r†  rd  ri   s     €r<   r4   zT5GemmaEncoderModel.__init__Š  s?   ø€ Ü‰Ñ˜Ô à×$Ò$ÜÐpÓqÐqä% f§n¡nÓ5ˆŒØ�‰Õr=   c                 ó6   — | j                   j                  «       S r2   rˆ  rP   s    r<   r‰  z(T5GemmaEncoderModel.get_input_embeddings“  rŠ  r=   c                 ó8   — | j                   j                  |«      S r2   rŒ  rŽ  s     r<   r�  z(T5GemmaEncoderModel.set_input_embeddings–  r�  r=   NrB  r½   r›   rg  rË   r   c                 ó4   —  | j                   d||||dœ|¤Ž}|S )Nr—  r  )r†  )r:   rB  r½   r›   rg  rË   r”  s          r<   rL   zT5GemmaEncoderModel.forward™  s7   € ð '˜$Ÿ,™,ð 
ØØ)Ø%Ø'ñ	
ð
 ñ
ˆð Ðr=   rx  )rR   rS   rT   r+   r4   r‰  r�  r$   r#   r7   r  r  r¡   r!   r"   r   rL   rV   rW   s   @r<   r   r   ˆ  s®   ø„ ð˜}õ ò3òAð Øð .2Ø37Ø04Ø-1ñà×#Ñ# dÑ*ðð ×)Ñ)¨DÑ0ðð ×&Ñ&¨Ñ-ð	ð
 —|‘| dÑ*ðð Ð+Ñ,ðð 
òó ó ôr=   r   c            "       óD  ‡ — e Zd ZddiZddiZddgdgfiZdefˆ fd„Zd	„ Zd
„ Z	e
e	 	 	 	 	 	 	 	 	 	 	 	 	 d#dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dej                  dz  dej                  dz  dej                  dz  dedz  deej(                  z  dee   deej                     ez  fd„«       «       Zdej(                  fd„Zdededed ed!edefˆ fd"„Zˆ xZS )$ÚT5GemmaForConditionalGenerationzlm_head.out_proj.weightz!model.decoder.embed_tokens.weightzlm_head.out_projÚcolwise_gather_outputrk   r.  r]   c                 ó  •— d|_         t        ‰| �	  |«       t        |«      | _        |j
                  j                  | _        t        |j
                  j                  | j                  «      | _	        d| _
        | j                  «        y )NTÚForMaskedLM)r…  r3   r4   rƒ  r1  r=  r+  r*  r^   Úlm_headÚ	loss_typerd  ri   s     €r<   r4   z(T5GemmaForConditionalGeneration.__init__²  sb   ø€ Ø$(ˆÔ!Ü‰Ñ˜Ô ä! &Ó)ˆŒ
Ø Ÿ.™.×3Ñ3ˆŒÜ$ V§^¡^×%?Ñ%?ÀÇÁÓQˆŒØ&ˆŒà�‰Õr=   c                 ó*  — || j                   _        | j                  j                  rl|j                  | j
                  j                  j                  _        |j                  j                  d   | j
                  j                  j                  _	        y y )Nr   )
rª  r'  r]   r:  r9   r1  r=  r]  rO   Únum_embeddingsrŽ  s     r<   Úset_output_embeddingsz5T5GemmaForConditionalGeneration.set_output_embeddings½  sj   € Ø .ˆ�‰Ôð �;‰;×*Ò*Ø5C×5JÑ5JˆD�J‰J×Ñ×+Ñ+Ô2Ø=K×=RÑ=R×=XÑ=XÐYZÑ=[ˆD�J‰J×Ñ×+Ñ+Õ:ð +r=   c                 ó.   — | j                   j                  S r2   )rª  r'  rP   s    r<   Úget_output_embeddingsz5T5GemmaForConditionalGeneration.get_output_embeddingsÆ  s   € Ø�|‰|×$Ñ$Ð$r=   NrB  r½   r›   r‘  r’  r“  r”  rç   rg  r•  Úlabelsr  Úlogits_to_keeprË   r   c                 ó¤  — |�|€|
€| j                  |«      } | j                  d|||||||||	|
|dœ|¤Ž}|j                  }t        |t        «      rt        | d«      n|}| j                  |dd…|dd…f   «      }| j                  «       j                  }|j                  �3||j                  z  }t        j                  |«      }||j                  z  }d}|� | j                  ||| j                  fi |¤Ž}t        |||j                  |j                   |j"                  |j$                  |j&                  |j(                  |j*                  ¬«	      S )aü  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N)rB  r½   r›   r‘  r’  r“  r”  rç   rg  r•  r  )	Úlossr.  rç   r™  rš  r|  r›  rù   rœ  r  )rE  r1  rn  r”   rU   Úslicerª  Úget_decoderr]   Úfinal_logit_softcappingr7   rÆ   Úloss_functionr+  r   rç   r™  rš  r|  r›  rù   rœ  )r:   rB  r½   r›   r‘  r’  r“  r”  rç   rg  r•  r±  r  r²  rË   r�  rk   Úslice_indicesr.  Údecoder_configr´  s                        r<   rL   z'T5GemmaForConditionalGeneration.forwardÉ  ss  € ð: ÐÐ"3Ð";Ð@UÐ@]à $× 1Ñ 1°&Ó 9Ðà.8¨d¯j©jð /
ØØ)Ø%Ø/Ø#9Ø!5Ø+Ø+Ø'Ø"7Øñ/
ð ñ/
ˆð (×9Ñ9ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆØ×)Ñ)Ó+×2Ñ2ˆØ×1Ñ1Ð=Ø˜n×DÑDÑDˆFÜ—Z‘Z Ó'ˆFØ˜n×DÑDÑDˆFàˆØÐà%�4×%Ñ% f¨f°d·o±oÑPÈÑPˆDäØØØ+×;Ñ;Ø"1×"GÑ"GØ.×AÑAØ,×=Ñ=Ø&5×&OÑ&OØ"1×"GÑ"GØ.×AÑAô

ð 
	
r=   c                 ó$   — | j                  |«      S r2   )rE  )r:   r±  s     r<   Ú%prepare_decoder_input_ids_from_labelszET5GemmaForConditionalGeneration.prepare_decoder_input_ids_from_labels  s   € Ø× Ñ  Ó(Ð(r=   Úgeneration_configÚmodel_kwargsÚgeneration_modeÚ
batch_sizeÚmax_cache_lengthc           	      ó€  •— t         ‰| �  |||||«       |j                  du ry|j                  }|€d}nd|j                  v }t	        j
                  | j                  j                  d¬«      «      }d|_        dg|j                  z  |_
        ||dœ}	|j                  d«      }
|
�“t        |
t        «      st        d	«      ‚t        |
j                   «      d
kD  r|
j                   j                  d
«      ryt#        |
j$                  «      }|t&        k(  r|d   d
   j(                  d   |	d<    |di |	¤Ž|
_        n=t        t+        di | j                  j                  d¬«      |dœ¤Žt+        «       «      |d<   t-        | d«      r=| j.                  �0t        | j.                  t        «      st        d«      ‚|d   | _        yyy)a7  Override cache preparation to force full attention on the cross-attention cache.

        The decoder config may declare sliding-window layers, but cross-attention must always use full attention.
        The default `_prepare_cache_for_generation` would otherwise build a sliding cross-attention cache.
        FNÚ	offloadedT)r=  rl  )r]   Ú
offloadingrç   z`The `past_key_values` in `model_kwargs` must be of type `EncoderDecoderCache` for T5Gemma model.r   r”  r*   Úmax_cache_lenÚ_cachezKThe internal cache must be of type `EncoderDecoderCache` for T5Gemma model.r  )r3   Ú_prepare_cache_for_generationr  Úcache_implementationÚcopyÚdeepcopyr]   Úget_text_configrä   rb  rÖ   rü   r”   r
   rø   Úlenrû   r•   rý   r   rO   r	   rØ   rÆ  )r:   r½  r¾  r¿  rÀ  rÁ  rÈ  Úoffload_cacheÚcross_attn_configÚcross_attn_cache_kwargsrç   Úcross_attn_clsr;   s               €r<   rÇ  z=T5GemmaForConditionalGeneration._prepare_cache_for_generation  så  ø€ ô 	‰Ñ-ØØØØØô	
ð ×&Ñ&¨%Ñ/Øà0×EÑEÐØÐ'Ø!‰Mà'Ð+<×+QÑ+QÐQˆMô !ŸM™M¨$¯+©+×*EÑ*EÈdÐ*EÓ*SÓTÐØ+/ÐÔ(Ø)9Ð(:Ð=N×=`Ñ=`Ñ(`ÐÔ%ð (Ø'ñ#
Ðð
 '×*Ñ*Ð+<Ó=ˆØÐ&Ü˜oÔ/BÔCÜ Øvóð ô
 �?×-Ñ-Ó.°Ò2°×7QÑ7Q×7UÑ7UÐVWÔ7XØä! /×"GÑ"GÓHˆNØ¤Ò,Ø;GÐHYÑ;ZÐ[\Ñ;]×;cÑ;cÐdeÑ;fÐ'¨Ñ8á4BÑ4]ÐE\Ñ4]ˆOÕ1ô /BÜñ à"&§+¡+×"=Ñ"=ÀdÐ"=Ó"KØ&3ññô “ó/ˆLÐ*Ñ+ô �4˜Ô" t§{¡{Ð'>Ü˜dŸk™kÔ+>Ô?Ü Ð!nÓoÐoà&Ð'8Ñ9ˆD�Kð	 (?Ð"r=   )NNNNNNNNNNNNr   ) rR   rS   rT   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr+   r4   r®  r°  r$   r#   r7   r  r  rž  r   r
   r!  rU   r¡   r!   r"   rN   r   rL   r¼  r   rp  r   rÇ  rV   rW   s   @r<   r¦  r¦  ­  s  ø„ Ø3Ð5XÐYÐØ"Ð$;Ð<€HØ" oÐ%6¸¸
Ð$CÐD€Hð	˜}õ 	ò\ò%ð Øð .2Ø37Ø04Ø59Ø:>Ø8<Ø26Ø6:Ø26Ø:>Ø*.Ø!%Ø-.ñG
à×#Ñ# dÑ*ðG
ð ×)Ñ)¨DÑ0ðG
ð ×&Ñ&¨Ñ-ð	G
ð
 !×+Ñ+¨dÑ2ðG
ð !&× 0Ñ 0°4Ñ 7ðG
ð $×.Ñ.°Ñ5ðG
ð )¨4Ñ/ðG
ð -¨tÑ3ðG
ð ×(Ñ(¨4Ñ/ðG
ð  %×0Ñ0°4Ñ7ðG
ð × Ñ  4Ñ'ðG
ð ˜$‘;ðG
ð ˜eŸl™lÑ*ðG
ð Ð+Ñ,ðG
ð  
ˆu× Ñ Ñ	! OÑ	3ò!G
ó ó ðG
ðR)¸E¿L¹Ló )ðI:à+ðI:ð ðI:ð (ð	I:ð
 ðI:ð ðI:ð 
÷I:ñ I:r=   r¦  c                   ó”  ‡ — e Zd Zddededz  fˆ fd„Zd„ Zd„ Zee		 	 	 	 	 	 	 	 	 	 dde
j                  dz  de
j                  dz  d	e
j                  dz  d
e
j                  dz  de
j                  dz  de
j                  dz  dedz  de
j                  dz  de
j                  dz  de
j                  dz  dee   defd„«       «       Zˆ xZS )Ú T5GemmaForSequenceClassificationNr]   r…  c                 ó”  •— |�||_         t        ‰| �	  |«       |j                  | _        |j                   rt	        |«      | _        nt        |«      | _        |j                  j                  }|j                   r|j                  j                  }t        |dd«      }t        || j                  |«      | _        | j                  «        y)z¬
        is_encoder_decoder (`Optional`, *optional*):
            Whether use encoder_decoder for sequence classification. When set to False, only encoder is used.
        Nr%  çš™™™™™¹?©r…  r3   r4   r$  rƒ  r1  r   r†  r^   r=  r†   r#  Úscorerd  ©r:   r]   r…  r^   Úclassifier_dropoutr;   s        €r<   r4   z)T5GemmaForSequenceClassification.__init__e  s¤   ø€ ð
 Ð)Ø(:ˆFÔ%Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒà×$Ò$Ü% fÓ-ˆD�Jä,¨VÓ4ˆDŒJà—n‘n×0Ñ0ˆØ×$Ò$Ø Ÿ.™.×4Ñ4ˆKä$ VÐ-FÈÓLÐÜ.¨{¸D¿O¹OÐM_Ó`ˆŒ
Ø�‰Õr=   c                 ó6   — | j                   j                  «       S r2   ©r1  r‰  rP   s    r<   r‰  z5T5GemmaForSequenceClassification.get_input_embeddings|  ó   € Ø�z‰z×.Ñ.Ó0Ð0r=   c                 ó:   — | j                   j                  |«       y r2   ©r1  r�  ©r:   r¼   s     r<   r�  z5T5GemmaForSequenceClassification.set_input_embeddings  ó   € Ø�
‰
×'Ñ'¨Õ.r=   rB  r½   r›   r‘  r’  r“  r”  rg  r•  r±  rË   r   c                 ó°  — | j                   j                  r'|€%|�#t        d| j                  j                  › d�«      ‚| j                   j                  r"|€ |	€|€t        d«      ‚| j                  |«      }| j                   j                  rB | j                  |f||||||||	ddœ	|¤Ž}|j                  }|j                  }|j                  }n; | j                  |f|||dœ|¤Ž}|j                  }|j                  }|j                  }| j                  |«      }|�|j                  d   }n|j                  d   }| j                   j                  €|d	k7  rt        d
«      ‚| j                   j                  €d}�n|�Õ|| j                   j                  k7  j!                  |j"                  t$        j&                  «      }t%        j(                  |j                  d   |j"                  t$        j&                  ¬«      }||z  j+                  d«      }| j                   j                  r[|d	z  }t%        j,                  ||j                  d   d	z
  ¬«      }n.d}t.        j1                  | j                  j                  › d�«       |t%        j(                  ||j"                  ¬«      |f   }d}|
�| j3                  ||
|| j                   ¬«      }t5        ||||¬«      S )áÜ  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nú8Passing input embeddings is currently not supported for ú in encoder-decoder mode.ú°If no `decoder_input_ids` or `decoder_inputs_embeds` are passed, `input_ids` cannot be `None`. Please pass either `input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`.F©	r½   r›   r‘  r’  r“  r”  rg  r•  r  ©r½   r›   rg  r   r*   z=Cannot handle batch sizes > 1 if no padding token is defined.r@   r…   )ÚmaxzŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rj  )r.  r±  Úpooled_logitsr]   ©r´  r.  rk   rW  )r]   r…  ÚNotImplementedErrorr;   rR   rø   rE  r1  rn  r™  rš  rk   rW  rÙ  rO   r?  rŠ   r|   r7   Úint32rˆ   ÚargmaxÚclampÚloggerÚwarning_oncer¸  r   )r:   rB  r½   r›   r‘  r’  r“  r”  rg  r•  r±  rË   Úoutputsrn  rk   rW  r.  rÀ  Úlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesrë  r´  s                          r<   rL   z(T5GemmaForSequenceClassification.forward‚  sð  € ð2 �;‰;×)Ò)¨yÐ/@À]ÐE^Ü%ØJÈ4Ï>É>×KbÑKbÐJcÐc|Ð}óð ð
 �;‰;×)Ò)Ð/@Ð/HÐMbÐMjØÐ Ü ðUóð ð
 !%× 1Ñ 1°)Ó <Ðà�;‰;×)Ò)Ø*4¨$¯*©*Øð+à-Ø)Ø"3Ø'=Ø%9Ø /Ø+Ø&;Øñ+ð ñ+ˆGð !(× 9Ñ 9ÐØ#×9Ñ9ˆMØ ×3Ñ3‰Jà'1 t§z¡zØð(à-Ø)Ø+ñ	(ð
 ñ(ˆGð !(× 9Ñ 9ÐØ#×1Ñ1ˆMØ ×+Ñ+ˆJà—‘Ð-Ó.ˆàÐ Ø"Ÿ™¨Ñ+‰Jà&×,Ñ,¨QÑ/ˆJà�;‰;×#Ñ#Ð+°
¸a²ÜÐ\Ó]Ð]Ø�;‰;×#Ñ#Ð+Ø!#ÒØÐ"à%¨¯©×)AÑ)AÑA×EÑEÀfÇmÁmÔUZ×U`ÑU`ÓaˆLÜ!ŸL™L¨¯©¸Ñ)<ÀVÇ]Á]ÔZ_×ZeÑZeÔfˆMØ"/°,Ñ">×!FÑ!FÀrÓ!JÐà�{‰{×-Ò-Ø" aÑ'Ð"Ü%*§[¡[Ð1CÐIZ×I`ÑI`ÐacÑIdÐghÑIhÔ%iÑ"à!#ÐÜ×ÑØ—>‘>×*Ñ*Ð+ð ,Zð Zôð
 œuŸ|™|¨J¸v¿}¹}ÔMÐOaÐaÑbˆàˆØÐØ×%Ñ%¨V¸FÐR_Ðhl×hsÑhsÐ%ÓtˆDä'ØØ Ø'Ø!ô	
ð 	
r=   r2   ©
NNNNNNNNNN)rR   rS   rT   r+   r!  r4   r‰  r�  r$   r#   r7   r  r¡   r   r  r!   r"   r   rL   rV   rW   s   @r<   rÕ  rÕ  c  sN  ø„ ñ˜}ð À$ÈÁ+õ ò.1ò/ð Øð .2Ø.2Ø04Ø59Ø6:Ø8<Ø26Ø26Ø:>Ø*.ñi
à×#Ñ# dÑ*ði
ð Ÿ™ tÑ+ði
ð ×&Ñ&¨Ñ-ð	i
ð
 !×+Ñ+¨dÑ2ði
ð !&§¡¨tÑ 3ði
ð $×.Ñ.°Ñ5ði
ð )¨4Ñ/ði
ð ×(Ñ(¨4Ñ/ði
ð  %×0Ñ0°4Ñ7ði
ð × Ñ  4Ñ'ði
ð Ð+Ñ,ði
ð 
"òi
ó ó ôi
r=   rÕ  c                   ó”  ‡ — e Zd Zddededz  fˆ fd„Zd„ Zd„ Zee		 	 	 	 	 	 	 	 	 	 dde
j                  dz  de
j                  dz  d	e
j                  dz  d
e
j                  dz  de
j                  dz  de
j                  dz  dedz  de
j                  dz  de
j                  dz  de
j                  dz  dee   defd„«       «       Zˆ xZS )ÚT5GemmaForTokenClassificationNr]   r…  c                 ó”  •— |�||_         t        ‰| �	  |«       |j                  | _        |j                   rt	        |«      | _        nt        |«      | _        |j                  j                  }|j                   r|j                  j                  }t        |dd«      }t        || j                  |«      | _        | j                  «        y)z©
        is_encoder_decoder (`Optional`, *optional*):
            Whether use encoder_decoder for token classification. When set to False, only encoder is used.
        Nr%  r×  rØ  rÚ  s        €r<   r4   z&T5GemmaForTokenClassification.__init__ò  s¤   ø€ ð
 Ð)Ø(:ˆFÔ%Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒà×$Ò$Ü% fÓ-ˆD�Jä,¨VÓ4ˆDŒJà—n‘n×0Ñ0ˆØ×$Ò$Ø Ÿ.™.×4Ñ4ˆKä$ VÐ-FÈÓLÐÜ.¨{¸D¿O¹OÐM_Ó`ˆŒ
à�‰Õr=   c                 ó6   — | j                   j                  «       S r2   rÝ  rP   s    r<   r‰  z2T5GemmaForTokenClassification.get_input_embeddings
  rÞ  r=   c                 ó:   — | j                   j                  |«       y r2   rà  rá  s     r<   r�  z2T5GemmaForTokenClassification.set_input_embeddings  râ  r=   rB  r½   r›   r‘  r’  r“  r”  rg  r•  r±  rË   r   c                 ó”  — | j                   j                  r'|€%|�#t        d| j                  j                  › d�«      ‚| j                   j                  r"|€ |	€|€t        d«      ‚| j                  |«      }| j                   j                  rB | j                  |f||||||||	ddœ	|¤Ž}|j                  }|j                  }|j                  }n; | j                  |f|||dœ|¤Ž}|j                  }|j                  }|j                  }| j                  |«      }d}|
�| j                  ||
| j                   «      }t        ||||¬«      S )	rä  Nrå  ræ  rç  Frè  ré  rì  )r]   r…  rí  r;   rR   rø   rE  r1  rn  r™  rš  rk   rW  rÙ  r¸  r   )r:   rB  r½   r›   r‘  r’  r“  r”  rg  r•  r±  rË   ró  rn  rk   rW  r.  r´  s                     r<   rL   z%T5GemmaForTokenClassification.forward  s�  € ð4 �;‰;×)Ò)¨yÐ/@À]ÐE^Ü%ØJÈ4Ï>É>×KbÑKbÐJcÐc|Ð}óð ð �;‰;×)Ò)Ð/@Ð/HÐMbÐMjØÐ Ü ðUóð ð
 !%× 1Ñ 1°)Ó <Ðà�;‰;×)Ò)Ø*4¨$¯*©*Øð+à-Ø)Ø"3Ø'=Ø%9Ø /Ø+Ø&;Øñ+ð ñ+ˆGð !(× 9Ñ 9ÐØ#×9Ñ9ˆMØ ×3Ñ3‰Jà'1 t§z¡zØð(à-Ø)Ø+ñ	(ð
 ñ(ˆGð !(× 9Ñ 9ÐØ#×1Ñ1ˆMØ ×+Ñ+ˆJà—‘Ð-Ó.ˆàˆØÐØ×%Ñ% f¨f°d·k±kÓBˆDä$ØØØ'Ø!ô	
ð 	
r=   r2   r÷  )rR   rS   rT   r+   r!  r4   r‰  r�  r$   r#   r7   r  r¡   r   r  r!   r"   r   rL   rV   rW   s   @r<   rù  rù  ð  sN  ø„ ñ˜}ð À$ÈÁ+õ ò01ò/ð Øð .2Ø.2Ø04Ø59Ø6:Ø8<Ø26Ø26Ø:>Ø*.ñN
à×#Ñ# dÑ*ðN
ð Ÿ™ tÑ+ðN
ð ×&Ñ&¨Ñ-ð	N
ð
 !×+Ñ+¨dÑ2ðN
ð !&§¡¨tÑ 3ðN
ð $×.Ñ.°Ñ5ðN
ð )¨4Ñ/ðN
ð ×(Ñ(¨4Ñ/ðN
ð  %×0Ñ0°4Ñ7ðN
ð × Ñ  4Ñ'ðN
ð Ð+Ñ,ðN
ð 
òN
ó ó ôN
r=   rù  )r¦  rƒ  r   r0  rÕ  rù  )r*   )ré   NN)_rÉ  Úcollections.abcr   Útypingr   r7   Útorch.nnr5   Ú r   r7  Úactivationsr   Úcache_utilsr   r	   r
   r   Ú
generationr   r   r   Úintegrationsr   r   Úmasking_utilsr   r   r   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r    Úprocessing_utilsr!   Úutilsr"   r#   r$   r%   Úutils.genericr&   r'   Úutils.output_capturingr(   r)   Úconfiguration_t5gemmar+   r,   Ú
get_loggerrR   rñ  ÚModuler.   rY   rm   r¨   r±   r¡   rU   r¸   rI   rN   rÐ   rÒ   rõ   r  r  r#  r*  r0  r  rT  rV  rz  rƒ  r   r¦  rÕ  rù  Ú__all__r  r=   r<   ú<module>r     s-  ðó* Ý $Ý ã Ý å &Ý !ß PÓ Pß KÑ Kß I÷ó õ CÝ 9÷÷ ÷ Lß FÝ &ß RÓ Rß Gß Eß Eð 
ˆ×	Ñ	˜HÓ	%€ô=�R—Y‘Yô =ô(�—‘ô ô&><˜RŸY™Yô ><òB(ñ Ð*Ó+òó ,ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð$ Ø Ø ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �S‰[ð%ð �T‰\ð%ð �T‰\ð%ð ˆ5�<‰<˜Ÿ™Ð%Ñ&ó%ñD Ð)Ó*ôF)˜2Ÿ9™9ó F)ó +ðF)ñR Ð)Ó*ôR)˜BŸI™Ió R)ó +ðR)ôj1Ð4ô 1ôhFÐ4ô FôR §	¡	ô ô	�B—I‘Iô 	ð ô9!˜_ó 9!ó ð9!ðxØ×Ñ $Ñ&ðà—<‘<ðð ˜‘*ðð ‡\�\ó	ô"P
Ð+ô P
ôfk
Ð+ô k
ð\ ôJ
Ð)ó J
ó ðJ
ðZ ô!Ð0ó !ó ð!ôHs:Ð&<¸oô s:ðl ôI
Ð'=ó I
ó ðI
ðX ôo
Ð$:ó o
ó ðo
òd�r=   