Ë
    ÜÓ^jmÊ  ã                   ó"  — d dl Z d dlmZ d dlmZ d dlZd dlmZ d dlm	Z	 ddl
mZ ddlmZmZmZmZ ddlmZ dd	lmZmZmZ dd
lmZmZmZmZ ddlmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z' ddl(m)Z)m*Z* ddl+m,Z, ddl-m.Z.m/Z/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6 ddl7m8Z8 ddl9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?  e1j€                  eA«      ZB e/d¬«      e	 G d„ de8«      «       «       ZC e/d¬«      e	 G d„ de«      «       «       ZD G d„ de=«      ZE G d„ de;«      ZF G d „ d!e>«      ZG G d"„ d#e:«      ZH G d$„ d%e:«      ZI G d&„ d'e «      ZJ G d(„ d)e «      ZK G d*„ d+ej˜                  «      ZM G d,„ d-ej˜                  «      ZNe/ G d.„ d/e<«      «       ZOd0ej                   dz  d1ej¢                  d2eRdz  d3ej¢                  fd4„ZS G d5„ d6eO«      ZT G d7„ d8eO«      ZUe/ G d9„ d:eO«      «       ZVe/ G d;„ d<eO«      «       ZW G d=„ d>eOe«      ZXe/ G d?„ d@eO«      «       ZYe/ G dA„ dBeO«      «       ZZg dC¢Z[y)Dé    N)ÚCallable)ÚAny)Ústricté   )Úinitialization)ÚCacheÚDynamicCacheÚEncoderDecoderCacheÚStaticCache)ÚPreTrainedConfig)ÚGenerationConfigÚGenerationMixinÚGenerationMode)Úcreate_bidirectional_maskÚ(create_bidirectional_sliding_window_maskÚcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚGemma2Config)ÚGemma2AttentionÚ	Gemma2MLPÚGemma2PreTrainedModelÚGemma2RMSNormÚGemma2RotaryEmbeddingÚeager_attention_forwardzgoogle/t5_gemma_module-7b)Ú
checkpointc                   ó.   — e Zd ZU dZdZeed<    e«       Zy)ÚT5GemmaModuleConfigaA  
    query_pre_attn_scalar (`float`, *optional*, defaults to 256):
        scaling factor used on the attention scores
    final_logit_softcapping (`float`, *optional*, defaults to 30.0):
        scaling factor when applying tanh softcapping on the logits.
    attn_logit_softcapping (`float`, *optional*, defaults to 50.0):
        scaling factor when applying tanh softcapping on the attention scores.

    ```python
    >>> from transformers import T5GemmaModuleModel, T5GemmaModuleConfig
    >>> # Initializing a T5GemmaModule t5_gemma_module-7b style configuration
    >>> configuration = T5GemmaModuleConfig()
    >>> # Initializing a model from the t5_gemma_module-7b style configuration
    >>> model = T5GemmaModuleModel(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```FÚ
is_decoderN)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r1   ÚboolÚ__annotations__ÚAttributeErrorÚuse_bidirectional_attention© ó    úv/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/t5gemma/modular_t5gemma.pyr0   r0   C   s   … ñð$ €J�ÓÙ"0Ó"2Ñr;   r0   c                   óæ   ‡ — e Zd ZU dZdZdgZeedœZdZee	e
e
f   z  dz  ed<   dZee	e
e
f   z  dz  ed<   dZeed	<   d
Zeez  ed<   d
Zeez  ed<   d
Zeez  ed<   dZeed<   dZeed<   ˆ fd„Zˆ xZS )ÚT5GemmaConfigaÈ  
    encoder (`Union[T5GemmaModuleConfig, dict]`, optional, *optional*):
        Configuration for the encoder.
    decoder (`Union[T5GemmaModuleConfig, dict]`, optional, *optional*):
        Configuration for the decoder.

    Example:

    ```python
    >>> from transformers import T5GemmaConfig, T5GemmaModel
    >>> t5gemma_config = T5GemmaConfig.from_pretrained("google/t5gemma-2b-2b-prefixlm-it")
    >>> model = T5GemmaModel(t5gemma_config)
    ```Út5gemmaÚpast_key_values)ÚencoderÚdecoderNrA   rB   TÚis_encoder_decoderç        Údropout_rateÚclassifier_dropout_rateÚattention_dropoutÚtie_word_embeddingsi è Ú
vocab_sizec                 óŒ  •— t        | j                  t        «      rt        di | j                  ¤Ž| _        n| j                  €t        «       | _        t        | j                  t        «      rt        di | j                  ¤Ž| _        n| j                  €t        «       | _        d| j                  _        | j                  | j                  _        | j                  | j                  _        d| j                  _        d| j                  _        | j                  | j                  _        | j                  | j                  _        | j                  j                  | j                  _
        |j                  d| j                  j                  «      | _        dD ]   }||vsŒt        | j                  |«      ||<   Œ" t        ‰| �<  di |¤Ž y )NFTÚinitializer_range)Úbos_token_idÚpad_token_idÚeos_token_idr:   )Ú
isinstancerA   Údictr0   rB   r1   rE   rG   Ú	use_cacheÚhidden_sizeÚcross_attention_hidden_sizeÚpoprK   ÚgetattrÚsuperÚ__post_init__)ÚselfÚkwargsÚspecial_token_keyÚ	__class__s      €r<   rW   zT5GemmaConfig.__post_init__z   sP  ø€ Ü�d—l‘l¤DÔ)Ü.Ñ>°·±Ñ>ˆD�LØ�\‰\Ð!Ü.Ó0ˆDŒLä�d—l‘l¤DÔ)Ü.Ñ>°·±Ñ>ˆD�LØ�\‰\Ð!Ü.Ó0ˆDŒLà"'ˆ�‰ÔØ$(×$5Ñ$5ˆ�‰Ô!Ø)-×)?Ñ)?ˆ�‰Ô&à"&ˆ�‰ÔØ!%ˆ�‰ÔØ$(×$5Ñ$5ˆ�‰Ô!Ø)-×)?Ñ)?ˆ�‰Ô&Ø37·<±<×3KÑ3Kˆ�‰Ô0à!'§¡Ð,?ÀÇÁ×A_ÑA_Ó!`ˆÔà!Qò 	UÐØ ¨Ò.Ü,3°D·L±LÐBSÓ,T�Ð(Ò)ð	Uô 	‰ÑÑ' Ó'r;   )r2   r3   r4   r5   Ú
model_typeÚkeys_to_ignore_at_inferencer0   Úsub_configsrA   rP   r   r7   rB   rC   r6   rE   ÚintÚfloatrF   rG   rH   rI   rW   Ú__classcell__©r[   s   @r<   r>   r>   \   s³   ø… ñð €JØ#4Ð"5ÐØ1Ð>QÑR€Kà;?€GÐ  4¨¨S¨¡>Ñ1°DÑ8Ó?Ø;?€GÐ  4¨¨S¨¡>Ñ1°DÑ8Ó?Ø#Ð˜Ó#Ø #€L�#˜‘+Ó#Ø+.Ð˜S 5™[Ó.Ø%(Ð�u˜s‘{Ó(Ø $Ð˜Ó$Ø€J�Ó÷(ð (r;   r>   c                   ó   — e Zd Zy)ÚT5GemmaRMSNormN©r2   r3   r4   r:   r;   r<   rd   rd   ˜   ó   „ Ør;   rd   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú
T5GemmaMLPc                 ól   •— t         ‰| �  |«       t        j                  |j                  «      | _        y ©N)rV   Ú__init__ÚnnÚDropoutrE   Údropout©rX   Úconfigr[   s     €r<   rk   zT5GemmaMLP.__init__�   s&   ø€ Ü‰Ñ˜Ô Ü—z‘z &×"5Ñ"5Ó6ˆ�r;   c                 ó®   — | j                  | j                  |«      «      | j                  |«      z  }| j                  |«      }| j	                  |«      }|S rj   )Úact_fnÚ	gate_projÚup_projrn   Ú	down_proj)rX   ÚxÚhidden_statesru   s       r<   ÚforwardzT5GemmaMLP.forward¡   sH   € ØŸ™ D§N¡N°1Ó$5Ó6¸¿¹Àa»ÑHˆØŸ™ ]Ó3ˆØ—N‘N =Ó1ˆ	ØÐr;   )r2   r3   r4   rk   rx   ra   rb   s   @r<   rh   rh   œ   s   ø„ ô7ör;   rh   c                   ó   — e Zd Zy)ÚT5GemmaRotaryEmbeddingNre   r:   r;   r<   rz   rz   ¨   rf   r;   rz   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚT5GemmaSelfAttentionrp   Ú	layer_idxc                 óH   •— t         ‰| �  ||«       |j                  | _        y rj   )rV   rk   r1   Ú	is_causal©rX   rp   r}   r[   s      €r<   rk   zT5GemmaSelfAttention.__init__­   s   ø€ Ü‰Ñ˜ Ô+à×*Ñ*ˆ�r;   )r2   r3   r4   r0   r_   rk   ra   rb   s   @r<   r|   r|   ¬   s   ø„ ð+Ð2ð +¸s÷ +ñ +r;   r|   c                   óú   ‡ — e Zd Zdedefˆ fd„Z	 ddej                  dej                  dz  dej                  dz  dedz  d	e	e
   d
eej                  ej                  dz  eej                     dz  f   fd„Zˆ xZS )ÚT5GemmaCrossAttentionrp   r}   c                 óŠ  •— t         ‰| �  ||«       | `| `d| _        |j
                  €t        d«      ‚t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        y )NFzBCross-attention needs cross_attention_hidden_size to be specified.©Úbias)rV   rk   Úsliding_windowÚ
layer_typer   rS   Ú
ValueErrorrl   ÚLinearÚnum_key_value_headsÚhead_dimÚattention_biasÚk_projÚv_projr€   s      €r<   rk   zT5GemmaCrossAttention.__init__´   s¤   ø€ Ü‰Ñ˜ Ô+ØÐØˆOØˆŒà×-Ñ-Ð5ÜÐaÓbÐbä—i‘iØ×.Ñ.°×0JÑ0JÈTÏ]É]Ñ0ZÐag×avÑavô
ˆŒô —i‘iØ×.Ñ.°×0JÑ0JÈTÏ]É]Ñ0ZÐag×avÑavô
ˆ�r;   Nrw   Úattention_maskÚencoder_hidden_statesr@   rY   Úreturnc                 ó‚  — |€t        d«      ‚|j                  d d }g |¢d‘| j                  ‘­}| j                  |«      j	                  |«      j                  dd«      }|�1|j                  j                  | j                  «      }	|j                  }
|�	s¾|j                  d d }g |¢d‘| j                  ‘­}| j                  |«      j	                  |«      j                  dd«      }| j                  |«      j	                  |«      j                  dd«      }|�€
j                  ||| j                  «      \  }}d|j                  | j                  <   nF
j                  | j                     j                  }|
j                  | j                     j                  }t!        j"                  | j$                  j&                  t(        «      } || ||||f| j*                  r| j,                  nd| j.                  d | j0                  dœ|¤Ž\  }} |j2                  g |¢d‘­Ž j5                  «       }| j7                  |«      }||fS )Nz5Encoder hidden state is required for cross attention.éÿÿÿÿé   r&   TrD   )rn   Úscalingr†   Úsoftcap)rˆ   Úshaper‹   Úq_projÚviewÚ	transposeÚ
is_updatedÚgetr}   Úcross_attention_cacher�   rŽ   ÚupdateÚlayersÚkeysÚvaluesr   Úget_interfacerp   Ú_attn_implementationr-   ÚtrainingrG   r•   Úattn_logit_softcappingÚreshapeÚ
contiguousÚo_proj)rX   rw   r�   r�   r@   rY   Úinput_shapeÚhidden_shapeÚquery_statesr›   Úcurr_past_key_valuesÚencoder_input_shapeÚencoder_hidden_shapeÚ
key_statesÚvalue_statesÚattention_interfaceÚattn_outputÚattn_weightss                     r<   rx   zT5GemmaCrossAttention.forwardÄ   s?  € ð !Ð(ÜÐTÓUÐUà#×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆØ—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆàÐ&Ø(×3Ñ3×7Ñ7¸¿¹ÓGˆJØ#2×#HÑ#HÐ àÐ"©*Ø"7×"=Ñ"=¸c¸rÐ"BÐØ#LÐ%8Ð#L¸"Ð#L¸d¿m¹mÑ#LÐ ØŸ™Ð%:Ó;×@Ñ@ÐAUÓV×`Ñ`ÐabÐdeÓfˆJØŸ;™;Ð'<Ó=×BÑBÐCWÓX×bÑbÐcdÐfgÓhˆLàÐ*Ø+?×+FÑ+FÀzÐS_Ðae×aoÑaoÓ+pÑ(�
˜LØ=A�×*Ñ*¨4¯>©>Ò:à-×4Ñ4°T·^±^ÑD×IÑIˆJØ/×6Ñ6°t·~±~ÑF×MÑMˆLä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð%
ð /3¯mªm�D×*Ò*ÀØ—L‘LØØ×/Ñ/ñ%
ð ñ%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r;   rj   )r2   r3   r4   r0   r_   rk   ÚtorchÚTensorr   r   r   Útuplerx   ra   rb   s   @r<   r‚   r‚   ³   s¢   ø„ ð
Ð2ð 
¸sõ 
ð* )-ñ3)à—|‘|ð3)ð Ÿ™ tÑ+ð3)ð  %Ÿ|™|¨dÑ2ð	3)ð
  ™ð3)ð Ð-Ñ.ð3)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷3)r;   r‚   c                   óè   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	eej                  f   f
d
„Zˆ xZS )ÚT5GemmaEncoderLayerzEncoder sub-layer.r}   c                 óD  •— t         ‰| �  «        |j                  | _        || _        || _        |j
                  |   | _        t        ||¬«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        |«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t#        j$                  |j&                  «      | _        y ©N)rp   r}   ©Úeps)rV   rk   rR   rp   r}   Úlayer_typesÚattention_typer|   Ú	self_attnrd   Úrms_norm_epsÚpre_self_attn_layernormÚpost_self_attn_layernormrh   ÚmlpÚpre_feedforward_layernormÚpost_feedforward_layernormrl   rm   rE   rn   r€   s      €r<   rk   zT5GemmaEncoderLayer.__init__ý   sá   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØˆŒØ"ˆŒØ$×0Ñ0°Ñ;ˆÔä-ØØô
ˆŒô (6°f×6HÑ6HÈf×NaÑNaÔ'bˆÔ$Ü(6°v×7IÑ7IÈv×ObÑObÔ(cˆÔ%ä˜fÓ%ˆŒÜ)7¸×8JÑ8JÐPV×PcÑPcÔ)dˆÔ&Ü*8¸×9KÑ9KÐQW×QdÑQdÔ*eˆÔ'ä—z‘z &×"5Ñ"5Ó6ˆ�r;   Nrw   Úposition_embeddingsr�   Úposition_idsr‘   c           	      ó>  — |}| j                  |«      } | j                  d||||d dœ|¤Ž\  }}| j                  |«      }|| j                  |«      z   }|}| j	                  |«      }| j                  |«      }| j                  |«      }|| j                  |«      z   }|S )N)rw   rÆ   r�   rÇ   r@   r:   )rÁ   r¿   rÂ   rn   rÄ   rÃ   rÅ   )rX   rw   rÆ   r�   rÇ   rY   ÚresidualÚ_s           r<   rx   zT5GemmaEncoderLayer.forward  s½   € ð !ˆØ×4Ñ4°]ÓCˆØ)˜4Ÿ>™>ð 
Ø'Ø 3Ø)Ø%Ø ñ
ð ñ
Ñˆ�qð ×5Ñ5°mÓDˆØ  4§<¡<°Ó#>Ñ>ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ  4§<¡<°Ó#>Ñ>ˆØÐr;   )NNN)r2   r3   r4   r5   r_   rk   r´   rµ   r¶   Ú
LongTensorÚFloatTensorrx   ra   rb   s   @r<   r¸   r¸   ú   s‘   ø„ Ùð7¨#õ 7ð. IMØ.2Ø04ñà—|‘|ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ÿ™ tÑ+ð	ð
 ×&Ñ&¨Ñ-ðð 
ˆu× Ñ Ð!Ñ	"÷r;   r¸   c                   ó8  ‡ — e Zd ZdZdefˆ fd„Z	 	 	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	e
dz  d
edz  dej                  dz  dej                  dz  dej                  fd„Zˆ xZS )ÚT5GemmaDecoderLayerz2Decoder sub-layer: an extra cross-attention layer.r}   c                 ó   •— t         ‰| �  «        |j                  | _        || _        || _        |j
                  |   | _        t        ||¬«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        |«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t#        j$                  |j&                  «      | _        t+        ||¬«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        y rº   )rV   rk   rR   rp   r}   r½   r¾   r|   r¿   rd   rÀ   rÁ   rÂ   rh   rÃ   rÄ   rÅ   rl   rm   rE   rn   r‚   Ú
cross_attnÚpre_cross_attn_layernormÚpost_cross_attn_layernormr€   s      €r<   rk   zT5GemmaDecoderLayer.__init__1  s&  ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØˆŒØ"ˆŒØ$×0Ñ0°Ñ;ˆÔä-ØØô
ˆŒô (6°f×6HÑ6HÈf×NaÑNaÔ'bˆÔ$Ü(6°v×7IÑ7IÈv×ObÑObÔ(cˆÔ%ä˜fÓ%ˆŒÜ)7¸×8JÑ8JÐPV×PcÑPcÔ)dˆÔ&Ü*8¸×9KÑ9KÐQW×QdÑQdÔ*eˆÔ'ä—z‘z &×"5Ñ"5Ó6ˆŒÜ/°vÈÔSˆŒÜ(6°v×7IÑ7IÈv×ObÑObÔ(cˆÔ%Ü)7¸×8JÑ8JÐPV×PcÑPcÔ)dˆÕ&r;   Nrw   rÆ   r�   rÇ   r@   rQ   r�   Úencoder_attention_maskr‘   c	           
      ó  — |}
| j                  |«      } | j                  d|||||�|j                  nd |dœ|	¤Ž\  }}| j                  |«      }|
| j	                  |«      z   }|}
| j                  |«      } | j                  d|||||dœ|	¤Ž\  }}| j                  |«      }|
| j	                  |«      z   }|}
| j                  |«      }| j                  |«      }| j                  |«      }|
| j	                  |«      z   }|S )N)rw   rÆ   r�   rÇ   r@   rQ   )rw   r�   r�   r@   rQ   r:   )rÁ   r¿   Úself_attention_cacherÂ   rn   rÑ   rÐ   rÒ   rÄ   rÃ   rÅ   )rX   rw   rÆ   r�   rÇ   r@   rQ   r�   rÓ   rY   rÉ   rÊ   s               r<   rx   zT5GemmaDecoderLayer.forwardH  s;  € ð !ˆØ×4Ñ4°]ÓCˆØ)˜4Ÿ>™>ð 
Ø'Ø 3Ø)Ø%ØDSÐD_˜O×@Ò@ÐeiØñ
ð ñ
Ñˆ�qð ×5Ñ5°mÓDˆØ  4§<¡<°Ó#>Ñ>ˆà ˆØ×5Ñ5°mÓDˆØ*˜4Ÿ?™?ð 
Ø'Ø"7Ø1Ø+Øñ
ð ñ
Ñˆ�qð ×6Ñ6°}ÓEˆØ  4§<¡<°Ó#>Ñ>ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ  4§<¡<°Ó#>Ñ>ˆØÐr;   )NNNNFNN)r2   r3   r4   r5   r_   rk   r´   rµ   r¶   rË   r
   r6   rÌ   rx   ra   rb   s   @r<   rÎ   rÎ   .  sÚ   ø„ Ù<ðe¨#õ eð4 IMØ.2Ø04Ø6:Ø!&Ø59Ø6:ñ,à—|‘|ð,ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð,ð Ÿ™ tÑ+ð	,ð
 ×&Ñ&¨Ñ-ð,ð -¨tÑ3ð,ð ˜$‘;ð,ð  %Ÿ|™|¨dÑ2ð,ð !&§¡¨tÑ 3ð,ð 
×	Ñ	÷,r;   rÎ   c                   ój   ‡ — e Zd ZdZd	dededefˆ fd„Zdej                  dej                  fd„Z	ˆ xZ
S )
ÚT5GemmaClassificationHeadz-Head for sentence-level classification tasks.rR   Ú
num_labelsrF   c                 óŽ   •— t         ‰| �  «        t        j                  |¬«      | _        t        j
                  ||«      | _        y )N)Úp)rV   rk   rl   rm   rn   r‰   Úout_proj)rX   rR   rØ   rF   r[   s       €r<   rk   z"T5GemmaClassificationHead.__init__z  s1   ø€ Ü‰ÑÔÜ—z‘zÐ$;Ô<ˆŒÜŸ	™	 +¨zÓ:ˆ�r;   rw   r‘   c                 óJ   — | j                  |«      }| j                  |«      }|S rj   )rn   rÛ   )rX   rw   s     r<   rx   z!T5GemmaClassificationHead.forward  s$   € ØŸ™ ]Ó3ˆØŸ™ mÓ4ˆØÐr;   )rD   )r2   r3   r4   r5   r_   r`   rk   r´   rµ   rx   ra   rb   s   @r<   r×   r×   w  s<   ø„ Ù7ñ; Cð ;°Sð ;ÐSXõ ;ð
 U§\¡\ð °e·l±l÷ r;   r×   c                   ój   ‡ — e Zd ZdZd	dededefˆ fd„Zdej                  dej                  fd„Z	ˆ xZ
S )
ÚT5GemmaLMHeadz.Head for language modeling (generation) tasks.rR   rI   r…   c                 ó\   •— t         ‰| �  «        t        j                  |||¬«      | _        y )Nr„   )rV   rk   rl   r‰   rÛ   )rX   rR   rI   r…   r[   s       €r<   rk   zT5GemmaLMHead.__init__ˆ  s"   ø€ Ü‰ÑÔÜŸ	™	 +¨zÀÔEˆ�r;   rw   r‘   c                 ó(   — | j                  |«      }|S rj   )rÛ   )rX   rw   Úlogitss      r<   rx   zT5GemmaLMHead.forwardŒ  s   € Ø—‘˜}Ó-ˆØˆr;   )F)r2   r3   r4   r5   r_   r6   rk   r´   rµ   rx   ra   rb   s   @r<   rÞ   rÞ   …  s?   ø„ Ù8ñF Cð F°Sð FÀõ Fð U§\¡\ð °e·l±l÷ r;   rÞ   c                   ó`   — e Zd ZU eed<   dZdZddgZdZ e	j                  «       d„ «       Zd„ Zy)	ÚT5GemmaPreTrainedModelrp   ÚmodelTr¸   rÎ   Nc                 ó@  — t        j                  | |«       | j                  j                  }t	        |t
        «      r­|j                  j                  j                  d   dz  }t        j                  |j                  j                  d||z  ¬«       t        |j                  d«      rA|j                  j                  �*t        j                  |j                  j                  «       y y y t	        |t        «      rm| j                  j                  sV|j                  j                  j                  d   dz  }t        j                  |j                  j                  d||z  ¬«       y y d|j                   j"                  v r t        j                  |j                  «       y y )Nr   g      à¿rD   )ÚmeanÚstdr…   ÚRMSNorm)r   Ú_init_weightsrp   rK   rO   r×   rÛ   Úweightr—   ÚinitÚnormal_Úhasattrr…   Úzeros_rÞ   rH   r[   r2   )rX   Úmodulerç   Úscales       r<   ré   z$T5GemmaPreTrainedModel._init_weightsš  s+  € ô 	×%Ñ% d¨FÔ3Ø�k‰k×+Ñ+ˆÜ�fÔ7Ô8Ø—O‘O×*Ñ*×0Ñ0°Ñ3°tÑ;ˆEÜ�L‰L˜Ÿ™×/Ñ/°c¸sÀU¹{ÕKÜ�v—‘¨Ô/°F·O±O×4HÑ4HÐ4TÜ—‘˜FŸO™O×0Ñ0Õ1ð 5UÐ/ä˜¤Ô.Ø—;‘;×2Ò2ØŸ™×.Ñ.×4Ñ4°QÑ7¸4Ñ?�Ü—‘˜VŸ_™_×3Ñ3¸#À3ÈÁ;ÖOð 3ð ˜&×*Ñ*×3Ñ3Ñ3Ü�K‰K˜Ÿ™Õ&ð 4r;   c                 ó`  — | j                   j                  j                  }| j                   j                  j                  }|€t	        d«      ‚|j                  |j                  «      }|ddd…f   j                  «       |ddd…f<   ||d<   |€t	        d«      ‚|j                  |dk(  |«       |S )	zú
        Shifts input_ids to the right, prepends the decoder_start_token_id, and handles
        pad_token_id replacement for labels that were -100.
        This is a common preparation step for decoder inputs in sequence-to-sequence models.
        Nz:self.model.config.decoder.bos_token_id has to be defined. .r“   r”   ).r   z9self.model.config.decoder.pad_token_id has to be defined.iœÿÿÿ)	rp   rB   rL   rM   rˆ   Ú	new_zerosr—   ÚcloneÚmasked_fill_)rX   Ú	input_idsÚdecoder_start_token_idrM   Úshifted_input_idss        r<   Ú_shift_rightz#T5GemmaPreTrainedModel._shift_right¬  sº   € ð "&§¡×!4Ñ!4×!AÑ!AÐØ—{‘{×*Ñ*×7Ñ7ˆà!Ð)ÜÐYÓZÐZð &×/Ñ/°	·±Ó@ÐØ%.¨s°C°R°C¨xÑ%8×%>Ñ%>Ó%@Ð˜#˜q™r˜'Ñ"Ø$:Ð˜&Ñ!àÐÜÐXÓYÐYð 	×&Ñ&Ð'8¸DÑ'@À,ÔOà Ð r;   )r2   r3   r4   r>   r7   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_can_record_outputsr´   Úno_gradré   rø   r:   r;   r<   rã   rã   ‘  sE   … àÓØÐØ&*Ð#Ø.Ð0EÐFÐàÐà€U‡]�]ƒ_ñ'ó ð'ó"!r;   rã   Ú	token_idsrw   rM   r‘   c                 ó  — | �<|€t        d«      ‚| |k7  j                  |j                  t        j                  «      }|S t        j
                  |j                  d   |j                  d   f|j                  t        j                  ¬«      }|S )z%Construct the default attention mask.z3`pad_token_id` is required for padding information.r   r”   ©ÚdeviceÚdtype)rˆ   Útor  r´   ÚlongÚonesr—   )rþ   rw   rM   r�   s       r<   Úmake_default_2d_attention_maskr  Ç  sŒ   € ð ÐØÐÜÐRÓSÐSØ# |Ñ3×7Ñ7¸×8LÑ8LÌeÏjÉjÓYˆð
 Ðô Ÿ™Ø× Ñ  Ñ# ]×%8Ñ%8¸Ñ%;Ð<À]×EYÑEYÔaf×akÑakô
ˆð Ðr;   c                   óÚ   ‡ — e Zd ZeedœZˆ fd„Zee	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  dee   d	eez  fd
„«       «       Zˆ xZS )ÚT5GemmaEncoder)Ú
attentionsrw   c           	      óT  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        |j                  |j                  ¬«      | _        d| _        t        j                  t        |j                  «      D �cg c]  }t!        ||«      ‘Œ c}«      | _        t        j$                  |j&                  «      | _        t+        |¬«      | _        | j/                  «        y c c}w ©Nr»   F©rp   )rV   rk   rM   Úpadding_idxrI   rl   Ú	EmbeddingrR   Úembed_tokensrd   rÀ   ÚnormÚgradient_checkpointingÚ
ModuleListÚrangeÚnum_hidden_layersr¸   rŸ   rm   rE   rn   rz   Ú
rotary_embÚ	post_initr€   s      €r<   rk   zT5GemmaEncoder.__init__Þ  óÜ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ" 6×#5Ñ#5¸6×;NÑ;NÔOˆŒ	Ø&+ˆÔ#ä—m‘mÜEJÈ6×KcÑKcÓEdÖe¸	Ô  ¨Õ3Òeó
ˆŒô —z‘z &×"5Ñ"5Ó6ˆŒÜ0¸Ô?ˆŒð 	�‰Õùò fó   Ã D%Nrõ   r�   rÇ   Úinputs_embedsrY   r‘   c                 ó‚  — |d u |d uz  rt        d«      ‚|j                  dd «       |€| j                  |«      }|€?t        j                  |j
                  d   |j                  ¬«      }|j                  d«      }|€!t        ||| j                  j                  «      }t        |x}t        «      s'| j                  ||dœ}t        di |¤Žt        di |¤Ždœ}|}t        j                  | j                  j                   dz  |j"                  ¬	«      }	||	z  }| j%                  |«      }| j'                  ||«      }
t)        | j*                  d | j                  j,                   «      D ]+  \  }} |||
|| j                  j.                  |      |fi |¤Ž}Œ- | j1                  |«      }| j%                  |«      }t3        |¬
«      S )Nú:You must specify exactly one of input_ids or inputs_embedsr@   r”   ©r  r   )rp   r  r�   ©Úfull_attentionÚsliding_attentionç      à?©r  )Úlast_hidden_stater:   )rˆ   rT   r  r´   Úaranger—   r  Ú	unsqueezer  rp   rM   rO   rP   r   r   ÚtensorrR   r  rn   r  Ú	enumeraterŸ   r  r½   r  r   )rX   rõ   r�   rÇ   r  rY   Úself_attn_mask_mappingÚmask_kwargsrw   Ú
normalizerrÆ   ÚiÚlayer_modules                r<   rx   zT5GemmaEncoder.forwardð  sË  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZð 	�
‰
Ð$ dÔ+àÐ Ø ×-Ñ-¨iÓ8ˆMàÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\ˆLØ'×1Ñ1°!Ó4ˆLàÐ!Ü;¸IÀ}ÐVZ×VaÑVa×VnÑVnÓoˆNä°NÐBÐ0ÄDÔIàŸ+™+Ø!.Ø"0ñˆKô #<Ñ"J¸kÑ"JÜ%MÑ%\ÐP[Ñ%\ñ&Ð"ð
 &ˆÜ—\‘\ $§+¡+×"9Ñ"9¸3Ñ">Àm×FYÑFYÔZˆ
Ø%¨
Ñ2ˆØŸ™ ]Ó3ˆà"Ÿo™o¨m¸\ÓJÐä(¨¯©Ð5T°t·{±{×7TÑ7TÐ)UÓVò 	‰OˆAˆ|Ù(ØØ#Ø& t§{¡{×'>Ñ'>¸qÑ'AÑBØñ	ð
 ñ‰Mð	ð Ÿ	™	 -Ó0ˆØŸ™ ]Ó3ˆÜØ+ô
ð 	
r;   ©NNNN)r2   r3   r4   r|   r¸   rü   rk   r#   r%   r´   rË   rµ   rÌ   r   r   r¶   r   rx   ra   rb   s   @r<   r  r  Ø  s¯   ø„ à*Ø,ñÐô
ð$  Øð .2Ø.2Ø04Ø26ñ6
à×#Ñ# dÑ*ð6
ð Ÿ™ tÑ+ð6
ð ×&Ñ&¨Ñ-ð	6
ð
 ×(Ñ(¨4Ñ/ð6
ð Ð+Ñ,ð6
ð 
�Ñ	 ò6
ó ó  ô6
r;   r  c                   óT  ‡ — e Zd Z eed¬«       eed¬«      edœZˆ fd„Ze	e
	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  d	edz  d
ej                  dz  dedz  dej                  dz  dej                  dz  dee   deez  fd„«       «       Zˆ xZS )ÚT5GemmaDecoderr”   )Úindex)r	  Úcross_attentionsrw   c           	      óT  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        |j                  |j                  ¬«      | _        d| _        t        j                  t        |j                  «      D �cg c]  }t!        ||«      ‘Œ c}«      | _        t        j$                  |j&                  «      | _        t+        |¬«      | _        | j/                  «        y c c}w r  )rV   rk   rM   r  rI   rl   r  rR   r  rd   rÀ   r  r  r  r  r  rÎ   rŸ   rm   rE   rn   rz   r  r  r€   s      €r<   rk   zT5GemmaDecoder.__init__2  r  r  Nrõ   r�   rÇ   r@   r  rQ   r�   rÓ   rY   r‘   c	                 ó¤  — |d u |d uz  rt        d«      ‚|€t        d«      ‚|€| j                  |«      }| j                  s,|r*|€(t        t	        | j
                  ¬«      t	        «       «      }|€V|�|j                  «       nd}
t        j                  |j                  d   |j                  ¬«      |
z   }|j                  d«      }|€#|€!t        ||| j
                  j                  «      }t        |x}t        «      s7| j
                  |||�|j                   nd |dœ}t#        di |¤Žt%        di |¤Ždœ}t        |x}t        «      sd	t'        | j
                  |||¬
«      i}|}t        j(                  | j
                  j*                  dz  |j,                  ¬«      }||z  }| j/                  |«      }| j1                  ||«      }t3        | j4                  d | j
                  j6                   «      D ]2  \  }} ||||| j
                  j8                  |      |||||d	   fi |	¤Ž}Œ4 | j;                  |«      }| j/                  |«      }t=        ||¬«      S )Nr  z0`encoder_hidden_states` must be given in decoderr  r   r”   r  )rp   r  r�   r@   rÇ   r  r  )rp   r  r�   r�   r   r!  )r"  r@   r:   )rˆ   r  r¤   r
   r	   rp   Úget_seq_lengthr´   r#  r—   r  r$  r  rM   rO   rP   rÕ   r   r   r   r%  rR   r  rn   r  r&  rŸ   r  r½   r  r   )rX   rõ   r�   rÇ   r@   r  rQ   r�   rÓ   rY   Úpast_seen_tokensr'  r(  Úcross_attn_mask_mappingrw   r)  rÆ   r*  r+  s                      r<   rx   zT5GemmaDecoder.forwardD  su  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZØ Ð(ÜÐOÓPÐPàÐ Ø ×-Ñ-¨iÓ8ˆMà�}Š}¡¨Ð/Fô 2´,ÀdÇkÁkÔ2RÔT`ÓTbÓcˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLàÐ! oÐ&=Ü;¸IÀ}ÐVZ×VaÑVa×VnÑVnÓoˆNä°NÐBÐ0ÄDÔIàŸ+™+Ø!.Ø"0ØKZÐKf ?×#GÒ#GÐlpØ ,ñˆKô #5Ñ"C°{Ñ"CÜ%FÑ%UÈÑ%Uñ&Ð"ô
 Ð5KÐKÐ1ÌTÔRà Ô";ØŸ;™;Ø"/Ø#9Ø*?ô	#ð'Ð#ð &ˆÜ—\‘\ $§+¡+×"9Ñ"9¸3Ñ">Àm×FYÑFYÔZˆ
Ø%¨
Ñ2ˆØŸ™ ]Ó3ˆà"Ÿo™o¨m¸\ÓJÐä(¨¯©Ð5T°t·{±{×7TÑ7TÐ)UÓVò 	‰OˆAˆ|Ù(ØØ#Ø& t§{¡{×'>Ñ'>¸qÑ'AÑBØØØØ%Ø'Ð(8Ñ9ñ
ð ñ
‰Mð	ð Ÿ	™	 -Ó0ˆØŸ™ ]Ó3ˆÜ8Ø+Ø+ô
ð 	
r;   )NNNNNNNN)r2   r3   r4   r$   r|   r‚   rÎ   rü   rk   r#   r%   r´   rË   rµ   r
   rÌ   r6   r   r   r¶   r   rx   ra   rb   s   @r<   r.  r.  +  s  ø„ á$Ð%9ÀÔCÙ*Ð+@ÈÔJØ,ñÐôð$  Øð .2Ø.2Ø04Ø6:Ø26Ø!%Ø59Ø6:ñP
à×#Ñ# dÑ*ðP
ð Ÿ™ tÑ+ðP
ð ×&Ñ&¨Ñ-ð	P
ð
 -¨tÑ3ðP
ð ×(Ñ(¨4Ñ/ðP
ð ˜$‘;ðP
ð  %Ÿ|™|¨dÑ2ðP
ð !&§¡¨tÑ 3ðP
ð Ð+Ñ,ðP
ð 
Ð:Ñ	:òP
ó ó  ôP
r;   r.  c                   ó€  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  dedz  dedz  de	j                  dz  de	j                  dz  dedz  dee   defd„«       «       Zˆ xZS )ÚT5GemmaModelrp   c                 óÚ   •— t         ‰| �  |«       |j                  st        d«      ‚t	        |j
                  «      | _        t        |j                  «      | _        | j                  «        y )NzVT5GemmaModel only support encoder-decoder modeling. Use `T5GemmaEncoderModel` instead.)	rV   rk   rC   rˆ   r  rA   r.  rB   r  ro   s     €r<   rk   zT5GemmaModel.__init__›  sO   ø€ Ü‰Ñ˜Ô à×(Ò(ÜÐuÓvÐvä% f§n¡nÓ5ˆŒÜ% f§n¡nÓ5ˆŒà�‰Õr;   c                 ó6   — | j                   j                  «       S rj   ©rA   Úget_input_embeddings©rX   s    r<   r;  z!T5GemmaModel.get_input_embeddings¦  ó   € Ø�|‰|×0Ñ0Ó2Ð2r;   c                 ó8   — | j                   j                  |«      S rj   ©rA   Úset_input_embeddings©rX   Únew_embeddingss     r<   r@  z!T5GemmaModel.set_input_embeddings©  ó   € Ø�|‰|×0Ñ0°Ó@Ð@r;   Nrõ   r�   rÇ   Údecoder_input_idsÚdecoder_attention_maskÚdecoder_position_idsÚencoder_outputsr@   r  Údecoder_inputs_embedsrQ   rY   r‘   c                 ó†  — |€ | j                   d||||	dœ|¤Ž}|j                  } | j                  d||||
||||dœ|¤Ž}t        |j                  |j                  |j                  dd«      r|j                  n|j                  f|j                  |j                  |j                  |j                  |j                  ¬«      S )aX  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        ©rõ   r�   rÇ   r  )rõ   r�   rÇ   r  r@   r�   rÓ   rQ   Úoutput_hidden_statesF)r"  r@   Údecoder_hidden_statesÚdecoder_attentionsr0  Úencoder_last_hidden_stater�   Úencoder_attentionsr:   )	rA   r"  rB   r   r@   rœ   rw   r	  r0  )rX   rõ   r�   rÇ   rD  rE  rF  rG  r@   r  rH  rQ   rY   r�   Údecoder_outputss                  r<   rx   zT5GemmaModel.forward¬  sî   € ð, Ð"Ø*˜dŸl™lð Ø#Ø-Ø)Ø+ñ	ð
 ñˆOð !0× AÑ AÐà&˜$Ÿ,™,ð 

Ø'Ø1Ø-Ø/Ø+Ø"7Ø#1Øñ

ð ñ

ˆô "Ø-×?Ñ?Ø+×;Ñ;à�z‰zÐ0°%Ô8ð #2×"?Ò"?à!×3Ñ3Ð5Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô
ð 	
r;   )NNNNNNNNNNN)r2   r3   r4   r>   rk   r;  r@  r!   r    r´   rË   rÌ   Ú
BoolTensorr   r
   rµ   r6   r   r   r   rx   ra   rb   s   @r<   r7  r7  ™  sA  ø„ ð	˜}õ 	ò3òAð Øð .2Ø37Ø04Ø59Ø:>Ø8<Ø26Ø6:Ø-1Ø59Ø!%ñ6
à×#Ñ# dÑ*ð6
ð ×)Ñ)¨DÑ0ð6
ð ×&Ñ&¨Ñ-ð	6
ð
 !×+Ñ+¨dÑ2ð6
ð !&× 0Ñ 0°4Ñ 7ð6
ð $×.Ñ.°Ñ5ð6
ð )¨4Ñ/ð6
ð -¨tÑ3ð6
ð —|‘| dÑ*ð6
ð  %Ÿ|™|¨dÑ2ð6
ð ˜$‘;ð6
ð Ð+Ñ,ð6
ð 
ò6
ó ó ô6
r;   r7  c                   óÜ   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
ee   defd„«       «       Zˆ xZS )ÚT5GemmaEncoderModelrp   c                 ó¦   •— t         ‰| �  |«       |j                  rt        d«      ‚t	        |j
                  «      | _        | j                  «        y )NzQT5GemmaEncoderModel only supports encoder-only model. Use `T5GemmaModel` instead.)rV   rk   rC   rˆ   r  rA   r  ro   s     €r<   rk   zT5GemmaEncoderModel.__init__é  s?   ø€ Ü‰Ñ˜Ô à×$Ò$ÜÐpÓqÐqä% f§n¡nÓ5ˆŒØ�‰Õr;   c                 ó6   — | j                   j                  «       S rj   r:  r<  s    r<   r;  z(T5GemmaEncoderModel.get_input_embeddingsò  r=  r;   c                 ó8   — | j                   j                  |«      S rj   r?  rA  s     r<   r@  z(T5GemmaEncoderModel.set_input_embeddingsõ  rC  r;   Nrõ   r�   rÇ   r  rY   r‘   c                 ó4   —  | j                   d||||dœ|¤Ž}|S )NrJ  r:   )rA   )rX   rõ   r�   rÇ   r  rY   rG  s          r<   rx   zT5GemmaEncoderModel.forwardø  s7   € ð '˜$Ÿ,™,ð 
ØØ)Ø%Ø'ñ	
ð
 ñ
ˆð Ðr;   r,  )r2   r3   r4   r>   rk   r;  r@  r!   r    r´   rË   rÌ   rµ   r   r   r   rx   ra   rb   s   @r<   rS  rS  ç  s®   ø„ ð˜}õ ò3òAð Øð .2Ø37Ø04Ø-1ñà×#Ñ# dÑ*ðð ×)Ñ)¨DÑ0ðð ×&Ñ&¨Ñ-ð	ð
 —|‘| dÑ*ðð Ð+Ñ,ðð 
òó ó ôr;   rS  c            "       óD  ‡ — e Zd ZddiZddiZddgdgfiZdefˆ fd„Zd	„ Zd
„ Z	e
e	 	 	 	 	 	 	 	 	 	 	 	 	 d#dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dej                  dz  dej                  dz  dej                  dz  dedz  deej(                  z  dee   deej                     ez  fd„«       «       Zdej(                  fd„Zdededed ed!edefˆ fd"„Zˆ xZS )$ÚT5GemmaForConditionalGenerationzlm_head.out_proj.weightz!model.decoder.embed_tokens.weightzlm_head.out_projÚcolwise_gather_outputrw   rá   rp   c                 ó  •— d|_         t        ‰| �	  |«       t        |«      | _        |j
                  j                  | _        t        |j
                  j                  | j                  «      | _	        d| _
        | j                  «        y )NTÚForMaskedLM)rC   rV   rk   r7  rä   rB   rI   rÞ   rR   Úlm_headÚ	loss_typer  ro   s     €r<   rk   z(T5GemmaForConditionalGeneration.__init__  sb   ø€ Ø$(ˆÔ!Ü‰Ñ˜Ô ä! &Ó)ˆŒ
Ø Ÿ.™.×3Ñ3ˆŒÜ$ V§^¡^×%?Ñ%?ÀÇÁÓQˆŒØ&ˆŒà�‰Õr;   c                 ó*  — || j                   _        | j                  j                  rl|j                  | j
                  j                  j                  _        |j                  j                  d   | j
                  j                  j                  _	        y y )Nr   )
r]  rÛ   rp   rH   rê   rä   rB   r  r—   Únum_embeddingsrA  s     r<   Úset_output_embeddingsz5T5GemmaForConditionalGeneration.set_output_embeddings  sj   € Ø .ˆ�‰Ôð �;‰;×*Ò*Ø5C×5JÑ5JˆD�J‰J×Ñ×+Ñ+Ô2Ø=K×=RÑ=R×=XÑ=XÐYZÑ=[ˆD�J‰J×Ñ×+Ñ+Õ:ð +r;   c                 ó.   — | j                   j                  S rj   )r]  rÛ   r<  s    r<   Úget_output_embeddingsz5T5GemmaForConditionalGeneration.get_output_embeddings%  s   € Ø�|‰|×$Ñ$Ð$r;   Nrõ   r�   rÇ   rD  rE  rF  rG  r@   r  rH  ÚlabelsrQ   Úlogits_to_keeprY   r‘   c                 ó¤  — |�|€|
€| j                  |«      } | j                  d|||||||||	|
|dœ|¤Ž}|j                  }t        |t        «      rt        | d«      n|}| j                  |dd…|dd…f   «      }| j                  «       j                  }|j                  �3||j                  z  }t        j                  |«      }||j                  z  }d}|� | j                  ||| j                  fi |¤Ž}t        |||j                  |j                   |j"                  |j$                  |j&                  |j(                  |j*                  ¬«	      S )aü  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N)rõ   r�   rÇ   rD  rE  rF  rG  r@   r  rH  rQ   )	Úlossrá   r@   rL  rM  r0  rN  r�   rO  r:   )rø   rä   r"  rO   r_   Úslicer]  Úget_decoderrp   Úfinal_logit_softcappingr´   ÚtanhÚloss_functionrI   r   r@   rL  rM  r0  rN  r�   rO  )rX   rõ   r�   rÇ   rD  rE  rF  rG  r@   r  rH  rd  rQ   re  rY   rP  rw   Úslice_indicesrá   Údecoder_configrg  s                        r<   rx   z'T5GemmaForConditionalGeneration.forward(  ss  € ð: ÐÐ"3Ð";Ð@UÐ@]à $× 1Ñ 1°&Ó 9Ðà.8¨d¯j©jð /
ØØ)Ø%Ø/Ø#9Ø!5Ø+Ø+Ø'Ø"7Øñ/
ð ñ/
ˆð (×9Ñ9ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆØ×)Ñ)Ó+×2Ñ2ˆØ×1Ñ1Ð=Ø˜n×DÑDÑDˆFÜ—Z‘Z Ó'ˆFØ˜n×DÑDÑDˆFàˆØÐà%�4×%Ñ% f¨f°d·o±oÑPÈÑPˆDäØØØ+×;Ñ;Ø"1×"GÑ"GØ.×AÑAØ,×=Ñ=Ø&5×&OÑ&OØ"1×"GÑ"GØ.×AÑAô

ð 
	
r;   c                 ó$   — | j                  |«      S rj   )rø   )rX   rd  s     r<   Ú%prepare_decoder_input_ids_from_labelszET5GemmaForConditionalGeneration.prepare_decoder_input_ids_from_labelss  s   € Ø× Ñ  Ó(Ð(r;   Úgeneration_configÚmodel_kwargsÚgeneration_modeÚ
batch_sizeÚmax_cache_lengthc           	      ó€  •— t         ‰| �  |||||«       |j                  du ry|j                  }|€d}nd|j                  v }t	        j
                  | j                  j                  d¬«      «      }d|_        dg|j                  z  |_
        ||dœ}	|j                  d«      }
|
�“t        |
t        «      st        d	«      ‚t        |
j                   «      d
kD  r|
j                   j                  d
«      ryt#        |
j$                  «      }|t&        k(  r|d   d
   j(                  d   |	d<    |di |	¤Ž|
_        n=t        t+        di | j                  j                  d¬«      |dœ¤Žt+        «       «      |d<   t-        | d«      r=| j.                  �0t        | j.                  t        «      st        d«      ‚|d   | _        yyy)a7  Override cache preparation to force full attention on the cross-attention cache.

        The decoder config may declare sliding-window layers, but cross-attention must always use full attention.
        The default `_prepare_cache_for_generation` would otherwise build a sliding cross-attention cache.
        FNÚ	offloadedT)rB   r  )rp   Ú
offloadingr@   z`The `past_key_values` in `model_kwargs` must be of type `EncoderDecoderCache` for T5Gemma model.r   rG  r”   Úmax_cache_lenÚ_cachezKThe internal cache must be of type `EncoderDecoderCache` for T5Gemma model.r:   )rV   Ú_prepare_cache_for_generationrQ   Úcache_implementationÚcopyÚdeepcopyrp   Úget_text_configr†   r  r½   rœ   rO   r
   rˆ   Úlenr›   Útyper�   r   r—   r	   rí   rz  )rX   rq  rr  rs  rt  ru  r|  Úoffload_cacheÚcross_attn_configÚcross_attn_cache_kwargsr@   Úcross_attn_clsr[   s               €r<   r{  z=T5GemmaForConditionalGeneration._prepare_cache_for_generationv  så  ø€ ô 	‰Ñ-ØØØØØô	
ð ×&Ñ&¨%Ñ/Øà0×EÑEÐØÐ'Ø!‰Mà'Ð+<×+QÑ+QÐQˆMô !ŸM™M¨$¯+©+×*EÑ*EÈdÐ*EÓ*SÓTÐØ+/ÐÔ(Ø)9Ð(:Ð=N×=`Ñ=`Ñ(`ÐÔ%ð (Ø'ñ#
Ðð
 '×*Ñ*Ð+<Ó=ˆØÐ&Ü˜oÔ/BÔCÜ Øvóð ô
 �?×-Ñ-Ó.°Ò2°×7QÑ7Q×7UÑ7UÐVWÔ7XØä! /×"GÑ"GÓHˆNØ¤Ò,Ø;GÐHYÑ;ZÐ[\Ñ;]×;cÑ;cÐdeÑ;fÐ'¨Ñ8á4BÑ4]ÐE\Ñ4]ˆOÕ1ô /BÜñ à"&§+¡+×"=Ñ"=ÀdÐ"=Ó"KØ&3ññô “ó/ˆLÐ*Ñ+ô �4˜Ô" t§{¡{Ð'>Ü˜dŸk™kÔ+>Ô?Ü Ð!nÓoÐoà&Ð'8Ñ9ˆD�Kð	 (?Ð"r;   )NNNNNNNNNNNNr   ) r2   r3   r4   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr>   rk   ra  rc  r!   r    r´   rË   rÌ   rQ  r   r
   r6   r_   rµ   r   r   r¶   r   rx   rp  r   rP   r   r{  ra   rb   s   @r<   rY  rY    s  ø„ Ø3Ð5XÐYÐØ"Ð$;Ð<€HØ" oÐ%6¸¸
Ð$CÐD€Hð	˜}õ 	ò\ò%ð Øð .2Ø37Ø04Ø59Ø:>Ø8<Ø26Ø6:Ø26Ø:>Ø*.Ø!%Ø-.ñG
à×#Ñ# dÑ*ðG
ð ×)Ñ)¨DÑ0ðG
ð ×&Ñ&¨Ñ-ð	G
ð
 !×+Ñ+¨dÑ2ðG
ð !&× 0Ñ 0°4Ñ 7ðG
ð $×.Ñ.°Ñ5ðG
ð )¨4Ñ/ðG
ð -¨tÑ3ðG
ð ×(Ñ(¨4Ñ/ðG
ð  %×0Ñ0°4Ñ7ðG
ð × Ñ  4Ñ'ðG
ð ˜$‘;ðG
ð ˜eŸl™lÑ*ðG
ð Ð+Ñ,ðG
ð  
ˆu× Ñ Ñ	! OÑ	3ò!G
ó ó ðG
ðR)¸E¿L¹Ló )ðI:à+ðI:ð ðI:ð (ð	I:ð
 ðI:ð ðI:ð 
÷I:ñ I:r;   rY  c                   ó”  ‡ — e Zd Zddededz  fˆ fd„Zd„ Zd„ Zee		 	 	 	 	 	 	 	 	 	 dde
j                  dz  de
j                  dz  d	e
j                  dz  d
e
j                  dz  de
j                  dz  de
j                  dz  dedz  de
j                  dz  de
j                  dz  de
j                  dz  dee   defd„«       «       Zˆ xZS )Ú T5GemmaForSequenceClassificationNrp   rC   c                 ó”  •— |�||_         t        ‰| �	  |«       |j                  | _        |j                   rt	        |«      | _        nt        |«      | _        |j                  j                  }|j                   r|j                  j                  }t        |dd«      }t        || j                  |«      | _        | j                  «        y)z¬
        is_encoder_decoder (`Optional`, *optional*):
            Whether use encoder_decoder for sequence classification. When set to False, only encoder is used.
        NrF   çš™™™™™¹?©rC   rV   rk   rØ   r7  rä   rS  rA   rR   rB   rU   r×   Úscorer  ©rX   rp   rC   rR   Úclassifier_dropoutr[   s        €r<   rk   z)T5GemmaForSequenceClassification.__init__Ä  s¤   ø€ ð
 Ð)Ø(:ˆFÔ%Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒà×$Ò$Ü% fÓ-ˆD�Jä,¨VÓ4ˆDŒJà—n‘n×0Ñ0ˆØ×$Ò$Ø Ÿ.™.×4Ñ4ˆKä$ VÐ-FÈÓLÐÜ.¨{¸D¿O¹OÐM_Ó`ˆŒ
Ø�‰Õr;   c                 ó6   — | j                   j                  «       S rj   ©rä   r;  r<  s    r<   r;  z5T5GemmaForSequenceClassification.get_input_embeddingsÛ  ó   € Ø�z‰z×.Ñ.Ó0Ð0r;   c                 ó:   — | j                   j                  |«       y rj   ©rä   r@  ©rX   Úvalues     r<   r@  z5T5GemmaForSequenceClassification.set_input_embeddingsÞ  ó   € Ø�
‰
×'Ñ'¨Õ.r;   rõ   r�   rÇ   rD  rE  rF  rG  r  rH  rd  rY   r‘   c                 ó°  — | j                   j                  r'|€%|�#t        d| j                  j                  › d�«      ‚| j                   j                  r"|€ |	€|€t        d«      ‚| j                  |«      }| j                   j                  rB | j                  |f||||||||	ddœ	|¤Ž}|j                  }|j                  }|j                  }n; | j                  |f|||dœ|¤Ž}|j                  }|j                  }|j                  }| j                  |«      }|�|j                  d   }n|j                  d   }| j                   j                  €|d	k7  rt        d
«      ‚| j                   j                  €d}�n|�Õ|| j                   j                  k7  j!                  |j"                  t$        j&                  «      }t%        j(                  |j                  d   |j"                  t$        j&                  ¬«      }||z  j+                  d«      }| j                   j                  r[|d	z  }t%        j,                  ||j                  d   d	z
  ¬«      }n.d}t.        j1                  | j                  j                  › d�«       |t%        j(                  ||j"                  ¬«      |f   }d}|
�| j3                  ||
|| j                   ¬«      }t5        ||||¬«      S )áÜ  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nú8Passing input embeddings is currently not supported for ú in encoder-decoder mode.ú°If no `decoder_input_ids` or `decoder_inputs_embeds` are passed, `input_ids` cannot be `None`. Please pass either `input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`.F©	r�   rÇ   rD  rE  rF  rG  r  rH  rQ   ©r�   rÇ   r  r   r”   z=Cannot handle batch sizes > 1 if no padding token is defined.r“   r   )ÚmaxzŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r  )rá   rd  Úpooled_logitsrp   ©rg  rá   rw   r	  )rp   rC   ÚNotImplementedErrorr[   r2   rˆ   rø   rä   r"  rL  rM  rw   r	  rŽ  r—   rM   r  r  r´   Úint32r#  ÚargmaxÚclampÚloggerÚwarning_oncerl  r   )rX   rõ   r�   rÇ   rD  rE  rF  rG  r  rH  rd  rY   Úoutputsr"  rw   r	  rá   rt  Úlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesr¡  rg  s                          r<   rx   z(T5GemmaForSequenceClassification.forwardá  sð  € ð2 �;‰;×)Ò)¨yÐ/@À]ÐE^Ü%ØJÈ4Ï>É>×KbÑKbÐJcÐc|Ð}óð ð
 �;‰;×)Ò)Ð/@Ð/HÐMbÐMjØÐ Ü ðUóð ð
 !%× 1Ñ 1°)Ó <Ðà�;‰;×)Ò)Ø*4¨$¯*©*Øð+à-Ø)Ø"3Ø'=Ø%9Ø /Ø+Ø&;Øñ+ð ñ+ˆGð !(× 9Ñ 9ÐØ#×9Ñ9ˆMØ ×3Ñ3‰Jà'1 t§z¡zØð(à-Ø)Ø+ñ	(ð
 ñ(ˆGð !(× 9Ñ 9ÐØ#×1Ñ1ˆMØ ×+Ñ+ˆJà—‘Ð-Ó.ˆàÐ Ø"Ÿ™¨Ñ+‰Jà&×,Ñ,¨QÑ/ˆJà�;‰;×#Ñ#Ð+°
¸a²ÜÐ\Ó]Ð]Ø�;‰;×#Ñ#Ð+Ø!#ÒØÐ"à%¨¯©×)AÑ)AÑA×EÑEÀfÇmÁmÔUZ×U`ÑU`ÓaˆLÜ!ŸL™L¨¯©¸Ñ)<ÀVÇ]Á]ÔZ_×ZeÑZeÔfˆMØ"/°,Ñ">×!FÑ!FÀrÓ!JÐà�{‰{×-Ò-Ø" aÑ'Ð"Ü%*§[¡[Ð1CÐIZ×I`ÑI`ÐacÑIdÐghÑIhÔ%iÑ"à!#ÐÜ×ÑØ—>‘>×*Ñ*Ð+ð ,Zð Zôð
 œuŸ|™|¨J¸v¿}¹}ÔMÐOaÐaÑbˆàˆØÐØ×%Ñ%¨V¸FÐR_Ðhl×hsÑhsÐ%ÓtˆDä'ØØ Ø'Ø!ô	
ð 	
r;   rj   ©
NNNNNNNNNN)r2   r3   r4   r>   r6   rk   r;  r@  r!   r    r´   rË   rµ   r   rÌ   r   r   r   rx   ra   rb   s   @r<   rŠ  rŠ  Â  sN  ø„ ñ˜}ð À$ÈÁ+õ ò.1ò/ð Øð .2Ø.2Ø04Ø59Ø6:Ø8<Ø26Ø26Ø:>Ø*.ñi
à×#Ñ# dÑ*ði
ð Ÿ™ tÑ+ði
ð ×&Ñ&¨Ñ-ð	i
ð
 !×+Ñ+¨dÑ2ði
ð !&§¡¨tÑ 3ði
ð $×.Ñ.°Ñ5ði
ð )¨4Ñ/ði
ð ×(Ñ(¨4Ñ/ði
ð  %×0Ñ0°4Ñ7ði
ð × Ñ  4Ñ'ði
ð Ð+Ñ,ði
ð 
"òi
ó ó ôi
r;   rŠ  c                   ó”  ‡ — e Zd Zddededz  fˆ fd„Zd„ Zd„ Zee		 	 	 	 	 	 	 	 	 	 dde
j                  dz  de
j                  dz  d	e
j                  dz  d
e
j                  dz  de
j                  dz  de
j                  dz  dedz  de
j                  dz  de
j                  dz  de
j                  dz  dee   defd„«       «       Zˆ xZS )ÚT5GemmaForTokenClassificationNrp   rC   c                 ó”  •— |�||_         t        ‰| �	  |«       |j                  | _        |j                   rt	        |«      | _        nt        |«      | _        |j                  j                  }|j                   r|j                  j                  }t        |dd«      }t        || j                  |«      | _        | j                  «        y)z©
        is_encoder_decoder (`Optional`, *optional*):
            Whether use encoder_decoder for token classification. When set to False, only encoder is used.
        NrF   rŒ  r�  r�  s        €r<   rk   z&T5GemmaForTokenClassification.__init__Q  s¤   ø€ ð
 Ð)Ø(:ˆFÔ%Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒà×$Ò$Ü% fÓ-ˆD�Jä,¨VÓ4ˆDŒJà—n‘n×0Ñ0ˆØ×$Ò$Ø Ÿ.™.×4Ñ4ˆKä$ VÐ-FÈÓLÐÜ.¨{¸D¿O¹OÐM_Ó`ˆŒ
à�‰Õr;   c                 ó6   — | j                   j                  «       S rj   r’  r<  s    r<   r;  z2T5GemmaForTokenClassification.get_input_embeddingsi  r“  r;   c                 ó:   — | j                   j                  |«       y rj   r•  r–  s     r<   r@  z2T5GemmaForTokenClassification.set_input_embeddingsl  r˜  r;   rõ   r�   rÇ   rD  rE  rF  rG  r  rH  rd  rY   r‘   c                 ó”  — | j                   j                  r'|€%|�#t        d| j                  j                  › d�«      ‚| j                   j                  r"|€ |	€|€t        d«      ‚| j                  |«      }| j                   j                  rB | j                  |f||||||||	ddœ	|¤Ž}|j                  }|j                  }|j                  }n; | j                  |f|||dœ|¤Ž}|j                  }|j                  }|j                  }| j                  |«      }d}|
�| j                  ||
| j                   «      }t        ||||¬«      S )	rš  Nr›  rœ  r�  Frž  rŸ  r¢  )rp   rC   r£  r[   r2   rˆ   rø   rä   r"  rL  rM  rw   r	  rŽ  rl  r   )rX   rõ   r�   rÇ   rD  rE  rF  rG  r  rH  rd  rY   r©  r"  rw   r	  rá   rg  s                     r<   rx   z%T5GemmaForTokenClassification.forwardo  s�  € ð4 �;‰;×)Ò)¨yÐ/@À]ÐE^Ü%ØJÈ4Ï>É>×KbÑKbÐJcÐc|Ð}óð ð �;‰;×)Ò)Ð/@Ð/HÐMbÐMjØÐ Ü ðUóð ð
 !%× 1Ñ 1°)Ó <Ðà�;‰;×)Ò)Ø*4¨$¯*©*Øð+à-Ø)Ø"3Ø'=Ø%9Ø /Ø+Ø&;Øñ+ð ñ+ˆGð !(× 9Ñ 9ÐØ#×9Ñ9ˆMØ ×3Ñ3‰Jà'1 t§z¡zØð(à-Ø)Ø+ñ	(ð
 ñ(ˆGð !(× 9Ñ 9ÐØ#×1Ñ1ˆMØ ×+Ñ+ˆJà—‘Ð-Ó.ˆàˆØÐØ×%Ñ% f¨f°d·k±kÓBˆDä$ØØØ'Ø!ô	
ð 	
r;   rj   r­  )r2   r3   r4   r>   r6   rk   r;  r@  r!   r    r´   rË   rµ   r   rÌ   r   r   r   rx   ra   rb   s   @r<   r¯  r¯  O  sN  ø„ ñ˜}ð À$ÈÁ+õ ò01ò/ð Øð .2Ø.2Ø04Ø59Ø6:Ø8<Ø26Ø26Ø:>Ø*.ñN
à×#Ñ# dÑ*ðN
ð Ÿ™ tÑ+ðN
ð ×&Ñ&¨Ñ-ð	N
ð
 !×+Ñ+¨dÑ2ðN
ð !&§¡¨tÑ 3ðN
ð $×.Ñ.°Ñ5ðN
ð )¨4Ñ/ðN
ð ×(Ñ(¨4Ñ/ðN
ð  %×0Ñ0°4Ñ7ðN
ð × Ñ  4Ñ'ðN
ð Ð+Ñ,ðN
ð 
òN
ó ó ôN
r;   r¯  )r>   r0   rY  r7  rS  rã   rŠ  r¯  )\r}  Úcollections.abcr   Útypingr   r´   Útorch.nnrl   Úhuggingface_hub.dataclassesr   Ú r   rë   Úcache_utilsr   r	   r
   r   Úconfiguration_utilsr   Ú
generationr   r   r   Úmasking_utilsr   r   r   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r    r!   r"   Úutils.genericr#   Úutils.output_capturingr$   r%   Úgemma2.configuration_gemma2r'   Úgemma2.modeling_gemma2r(   r)   r*   r+   r,   r-   Ú
get_loggerr2   r§  r0   r>   rd   rh   rz   r|   r‚   r¸   rÎ   ÚModuler×   rÞ   rã   rË   rµ   r_   r  r  r.  r7  rS  rY  rŠ  r¯  Ú__all__r:   r;   r<   ú<module>rÊ     si  ðó Ý $Ý ã Ý Ý .å &ß PÓ PÝ 3ß KÑ K÷ó õ CÝ 9÷÷ ÷ GÝ &÷ó õ 8ß EÝ 6÷÷ ð 
ˆ×	Ñ	˜HÓ	%€ñ Ð6Ô7Øô3˜,ó 3ó ó 8ð3ñ. Ð6Ô7Øô7(Ð$ó 7(ó ó 8ð7(ôt	�]ô 	ô	�ô 	ô	Ð2ô 	ô+˜?ô +ôD)˜Oô D)ôN1Ð4ô 1ôhFÐ4ô FôR §	¡	ô ô	�B—I‘Iô 	ð ô2!Ð2ó 2!ó ð2!ðjØ×Ñ $Ñ&ðà—<‘<ðð ˜‘*ðð ‡\�\ó	ô"P
Ð+ô P
ôfk
Ð+ô k
ð\ ôJ
Ð)ó J
ó ðJ
ðZ ô!Ð0ó !ó ð!ôHs:Ð&<¸oô s:ðl ôI
Ð'=ó I
ó ðI
ðX ôo
Ð$:ó o
ó ðo
òd	�r;   