Ë
    ÜÓ^j'
 ã                   ól  — d dl Z d dlmZ d dlmZ d dlZd dlmZ ddlm	Z
 ddlmZ ddlmZmZmZmZ ddlmZmZmZ dd	lmZmZ dd
lmZmZmZ ddlmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z-m.Z. ddl/m0Z0 ddl1m2Z2m3Z3m4Z4m5Z5 ddl6m7Z7m8Z8 ddl9m:Z:m;Z; ddl<m=Z= ddl>m?Z?m@Z@mAZAmBZB  G d„ dej†                  «      ZD G d„ dej†                  «      ZE G d„ dej†                  «      ZFd„ ZG ed«      dQd „«       ZHd!ej’                  d"eJd#ej’                  fd$„ZK	 	 	 dRd%ej†                  d&ej’                  d'ej’                  d(ej’                  d)ej’                  dz  d*eLeJz  d+eLdz  d,eLdz  d#eMej’                  ej’                  f   fd-„ZN eeH«       G d.„ d/ej†                  «      «       ZO eeH«       G d0„ d1ej†                  «      «       ZP G d2„ d3e «      ZQ G d4„ d5e «      ZR G d6„ d7ej†                  «      ZS G d8„ d9ej†                  «      ZT G d:„ d;ej†                  «      ZU G d<„ d=ej¬                  «      ZWe3 G d>„ d?e.«      «       ZXdSd@eJd#efdA„ZY G dB„ dCeX«      ZZ G dD„ dEeX«      Z[ G dF„ dGeX«      Z\e3 G dH„ dIeX«      «       Z] G dJ„ dKeXe«      Z^e3 G dL„ dMeX«      «       Z_e3 G dN„ dOeX«      «       Z`g dP¢Zay)Té    N)ÚCallable)ÚOptionalé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCacheÚStaticCache)ÚGenerationConfigÚGenerationMixinÚGenerationMode)Úuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚBaseModelOutputWithPoolingÚSeq2SeqLMOutputÚSeq2SeqModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚtorch_compilable_check)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú	AutoModelé   )ÚT5Gemma2ConfigÚT5Gemma2DecoderConfigÚT5Gemma2EncoderConfigÚT5Gemma2TextConfigc                   ó<   ‡ — e Zd Zddedefˆ fd„Zd„ Zd„ Zd„ Zˆ xZ	S )ÚT5Gemma2RMSNormÚdimÚepsc                 óŠ   •— t         ‰| �  «        || _        t        j                  t        j                  |«      «      | _        y ©N)ÚsuperÚ__init__r4   ÚnnÚ	ParameterÚtorchÚzerosÚweight)Úselfr3   r4   Ú	__class__s      €úy/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/t5gemma2/modeling_t5gemma2.pyr8   zT5Gemma2RMSNorm.__init__8   s.   ø€ Ü‰ÑÔØˆŒÜ—l‘l¤5§;¡;¨sÓ#3Ó4ˆ�ó    c                 óŒ   — |t        j                  |j                  d«      j                  dd¬«      | j                  z   «      z  S )Nr*   éÿÿÿÿT)Úkeepdim)r;   ÚrsqrtÚpowÚmeanr4   )r>   Úxs     r@   Ú_normzT5Gemma2RMSNorm._norm=   s4   € Ø”5—;‘;˜qŸu™u Q›xŸ}™}¨R¸˜}Ó>ÀÇÁÑIÓJÑJÐJrA   c                 ó¢   — | j                  |j                  «       «      }|d| j                  j                  «       z   z  }|j                  |«      S )Nç      ð?)rI   Úfloatr=   Útype_as)r>   rH   Úoutputs      r@   ÚforwardzT5Gemma2RMSNorm.forward@   sC   € Ø—‘˜AŸG™G›IÓ&ˆð ˜3 §¡×!2Ñ!2Ó!4Ñ4Ñ5ˆØ�~‰~˜aÓ Ð rA   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)Útupler=   Úshaper4   ©r>   s    r@   Ú
extra_reprzT5Gemma2RMSNorm.extra_reprG   s'   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$·(±(°Ð<Ð<rA   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__ÚintrL   r8   rI   rO   rT   Ú__classcell__©r?   s   @r@   r2   r2   7   s&   ø„ ñ5˜Cð 5 eõ 5ò
Kò!ö=rA   r2   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚT5Gemma2MLPÚconfigc                 ó  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _	        t        |j                     | _        t        j                  |j                  «      | _        y )NF©Úbias)r7   r8   r]   Úhidden_sizeÚintermediate_sizer9   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Úhidden_activationÚact_fnÚDropoutÚdropout_rateÚdropout©r>   r]   r?   s     €r@   r8   zT5Gemma2MLP.__init__L   s¼   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×5Ñ5Ñ6ˆŒÜ—z‘z &×"5Ñ"5Ó6ˆ�rA   c                 ó®   — | j                  | j                  |«      «      | j                  |«      z  }| j                  |«      }| j	                  |«      }|S r6   )rh   rd   re   rk   rf   )r>   rH   Úhidden_statesrf   s       r@   rO   zT5Gemma2MLP.forwardW   sH   € ØŸ™ D§N¡N°1Ó$5Ó6¸¿¹Àa»ÑHˆØŸ™ ]Ó3ˆØ—N‘N =Ó1ˆ	ØÐrA   )rU   rV   rW   r0   r8   rO   rY   rZ   s   @r@   r\   r\   K   s   ø„ ð	7Ð1õ 	7örA   r\   c                   óÒ   ‡ — e Zd ZU ej                  ed<   ddefˆ fd„Ze	 	 	 	 ddedz  de	d   de
dz  dedz  d	ed
ef   f
d„«       Z ej                  «       edd„«       «       Zˆ xZS )ÚT5Gemma2RotaryEmbeddingÚinv_freqNr]   c                 ót  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        t        t        |j                  «      «      | _        i | _	        | j                  D ]Ç  }| j
                  j                  |   }|€Œ|d   | j                  |<   | j                  }| j                  |   dk7  rt        | j                  |      } || j
                  |¬«      \  }}| j                  |› d�|d¬«       | j                  |› d�|j                  «       d¬«       t        | |› d�|«       ŒÉ y )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqF©Ú
persistentÚ_original_inv_freqÚ_attention_scaling)r7   r8   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr]   ÚlistÚsetÚlayer_typesrs   Úrope_parametersÚcompute_default_rope_parametersr   Úregister_bufferÚcloneÚsetattr)	r>   r]   Údevicerv   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingr?   s	           €r@   r8   z T5Gemma2RotaryEmbedding.__init__a   s6  ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØˆŒÜ¤ F×$6Ñ$6Ó 7Ó8ˆÔØˆŒØ×*Ñ*ò 	UˆJØŸ+™+×5Ñ5°jÑAˆKØÐ"Øà)4°[Ñ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÑ)¨YÒ6Ü2°4·>±>À*Ñ3MÑN�Ù4@ÀÇÁÐYcÔ4dÑ1ˆMÐ1Ø× Ñ  J <¨yÐ!9¸=ÐUZÐ Ô[Ø× Ñ  J <Ð/AÐ!BÀM×DWÑDWÓDYÐfkÐ ÔlÜ�D˜Z˜LÐ(:Ð;Ð=SÕTñ	UrA   r‡   ztorch.deviceÚseq_lenrv   Úreturnztorch.Tensorc                 ó   — | j                   |   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a|  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNrK   r   r*   ©Údtype©r‡   r’   )	r‚   Úgetattrra   Únum_attention_headsr;   ÚarangeÚint64ÚtorL   )r]   r‡   rŒ   rv   Úbaser3   Úattention_factorrq   s           r@   rƒ   z7T5Gemma2RotaryEmbedding.compute_default_rope_parametersv   s•   € ð2 ×%Ñ% jÑ1°,Ñ?ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)rA   c                 óN  — t        | |› d�«      }t        | |› d�«      }|d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬	«      5  |j                  «       |j                  «       z  j                  dd
«      }	t        j                  |	|	fd¬«      }
|
j                  «       |z  }|
j                  «       |z  }d d d «       j	                  |j                  ¬«      j	                  |j                  ¬«      fS # 1 sw Y   ŒAxY w)Nrw   r{   r   rC   r,   ÚmpsÚcpuF)Údevice_typeÚenabledr*   ©r3   r‘   )r”   rL   ÚexpandrR   r˜   r‡   Ú
isinstanceÚtypeÚstrr&   Ú	transposer;   ÚcatÚcosÚsinr’   )r>   rH   Úposition_idsrv   rq   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedrž   ÚfreqsÚembr§   r¨   s                r@   rO   zT5Gemma2RotaryEmbedding.forwardš   sl  € ô ˜4 J <¨yÐ!9Ó:ˆÜ# D¨Z¨LÐ8JÐ*KÓLÐà$ Tª1¨d ]Ñ3×9Ñ9Ó;×BÑBÀ<×CUÑCUÐVWÑCXÐZ\Ð^_Ó`×cÑcÐde×dlÑdlÓmÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	0Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)Ð/Ñ/ˆCØ—'‘'“)Ð/Ñ/ˆC÷		0ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	0ð 	0ús   Ã*A1FÆF$r6   )NNNN)rU   rV   rW   r;   ÚTensorÚ__annotations__r0   r8   Ústaticmethodr   rX   r¤   rQ   rL   rƒ   Úno_gradr   rO   rY   rZ   s   @r@   rp   rp   ^   s«   ø… Ø�l‰lÓñUÐ1õ Uð* à,0Ø+/Ø"Ø!%ñ	!*Ø" TÑ)ð!*à˜Ñ(ð!*ð �t‘ð!*ð ˜$‘Jð	!*ð
 
ˆ~˜uÐ$Ñ	%ò!*ó ð!*ðF €U‡]�]ƒ_Øò<ó ó ô<rA   rp   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..NrC   r*   r    )rR   r;   r¦   )rH   Úx1Úx2s      r@   Úrotate_halfr¶   ­   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'rA   Úrotary_pos_embc                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer¶   )ÚqÚkr§   r¨   Úunsqueeze_dimÚq_embedÚk_embeds          r@   Úapply_rotary_pos_embr¿   ´   sY   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐrA   rn   Ún_repr�   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r,   N)rR   r¡   Úreshape)rn   rÀ   ÚbatchÚnum_key_value_headsÚslenr�   s         r@   Ú	repeat_kvrÆ   Î   so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTrA   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskrk   ÚscalingÚsoftcapc                 ó|  — |€| j                   dz  }t        || j                  «      }	t        || j                  «      }
t        j                  ||	j                  dd«      «      |z  }|�||z  }t        j                  |«      }||z  }|�||z   }t        j                  j                  |dt        j                  ¬«      j                  |j                  «      }t        j                  j                  ||| j                  ¬«      }t        j                  ||
«      }|j                  dd«      j                  «       }||fS )Nç      à¿r*   r   rC   )r3   r’   )ÚpÚtrainingr,   )r�   rÆ   Únum_key_value_groupsr;   Úmatmulr¥   Útanhr9   Ú
functionalÚsoftmaxÚfloat32r˜   r’   rk   rÑ   Ú
contiguous)rÇ   rÈ   rÉ   rÊ   rË   rk   rÌ   rÍ   ÚkwargsÚ
key_statesÚvalue_statesÚattn_weightsÚattn_outputs                r@   Úeager_attention_forwardrÞ   Ú   s  € ð €Ø—/‘/ 4Ñ'ˆä˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LàÐØ# gÑ-ˆÜ—z‘z ,Ó/ˆØ# gÑ-ˆØÐ!Ø# nÑ4ˆô —=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØ˜Ð$Ð$rA   c                   óü   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej                  dej                  dej                  dz  d	e	dz  d
e
e   deej                  ej                  dz  eej                     dz  f   fd„Zˆ xZS )ÚT5Gemma2SelfAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr]   Ú	layer_idxc                 óö  •— t         ‰| �  «        t        |d«      r|j                  |   nd | _        || _        || _        t        |d|j                  |j                  z  «      | _
        |j                  |j                  z  | _        |j                  dz  | _        | j
                  j                  | _        d| _        t#        j$                  |j                  |j                  | j                  z  |j&                  ¬«      | _        t#        j$                  |j                  |j                  | j                  z  |j&                  ¬«      | _        t#        j$                  |j                  |j                  | j                  z  |j&                  ¬«      | _        t#        j$                  |j                  | j                  z  |j                  |j&                  ¬«      | _        | j
                  j0                  | _        | j                  dk(  r|j2                  nd | _        | j                  dk(  | _        t7        |j                  |j8                  ¬«      | _        t7        |j                  |j8                  ¬«      | _        y ©Nr�   r�   rÏ   Fr_   Úsliding_attention)r3   r4   ©r7   r8   Úhasattrr�   rv   r]   rá   r”   ra   r•   r�   rÄ   rÒ   Úquery_pre_attn_scalarrÌ   Úattention_dropoutÚ	is_causalr9   rc   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚattn_logit_softcappingÚsliding_windowÚ
is_slidingr2   Úrms_norm_epsÚq_normÚk_norm©r>   r]   rá   r?   s      €r@   r8   zT5Gemma2SelfAttention.__init__   óã  ø€ Ü‰ÑÔÜ;BÀ6È=Ô;Y˜&×,Ñ,¨YÒ7Ð_cˆŒØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø×3Ñ3°TÑ9ˆŒØ!%§¡×!>Ñ!>ˆÔØˆŒä—i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒð '+§k¡k×&HÑ&HˆÔ#Ø7;·±ÐJ]Ò7]˜f×3Ò3ÐcgˆÔØŸ/™/Ð-@Ñ@ˆŒä%¨&¯/©/¸v×?RÑ?RÔSˆŒÜ%¨&¯/©/¸v×?RÑ?RÔSˆ�rA   Nrn   Úposition_embeddingsrË   Úpast_key_valuesrÙ   r�   c                 ód  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
| j                  |«      }| j                  |	«      }	|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t         «      } || ||	|
|f| j"                  r| j$                  nd| j&                  | j(                  dœ|¤Ž\  }} |j*                  g |¢d‘­Ž j-                  «       }| j/                  |«      }||fS )NrC   r,   r*   ç        )rk   rÌ   rð   )rR   r�   rë   Úviewr¥   rì   rí   ró   rô   r¿   Úupdaterá   r   Úget_interfacer]   Ú_attn_implementationrÞ   rÑ   rè   rÌ   rð   rÂ   rØ   rî   )r>   rn   r÷   rË   rø   rÙ   Úinput_shapeÚhidden_shapeÚquery_statesrÚ   rÛ   r§   r¨   Úattention_interfacerÝ   rÜ   s                   r@   rO   zT5Gemma2SelfAttention.forward  s¶  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà—{‘{ <Ó0ˆØ—[‘[ Ó,ˆ
à&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð /3¯mªm�D×*Ò*ÀØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(rA   ©NNN)rU   rV   rW   Ú__doc__r0   rX   r8   r;   r¯   r   r!   r"   rQ   rO   rY   rZ   s   @r@   rà   rà   ü   sª   ø„ áGðTÐ1ð T¸cõ TðB -1Ø.2Ø(,ñ*)à—|‘|ð*)ð #Ÿ\™\ð*)ð Ÿ™ tÑ+ð	*)ð
  ™ð*)ð Ð+Ñ,ð*)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷*)rA   rà   c                   ó.  ‡ — e Zd ZdZdedefˆ fd„Z	 ddej                  de	ej                  ej                  f   dej                  dz  d	ej                  d
e
dz  dee   de	ej                  ej                  dz  e	ej                     dz  f   fd„Zˆ xZS )ÚT5Gemma2MergedAttentionz6Merged self-attention and cross-attention for decoder.r]   rá   c                 óö  •— t         ‰| �  «        t        |d«      r|j                  |   nd | _        || _        || _        t        |d|j                  |j                  z  «      | _
        |j                  |j                  z  | _        |j                  dz  | _        | j
                  j                  | _        d| _        t#        j$                  |j                  |j                  | j                  z  |j&                  ¬«      | _        t#        j$                  |j                  |j                  | j                  z  |j&                  ¬«      | _        t#        j$                  |j                  |j                  | j                  z  |j&                  ¬«      | _        t#        j$                  |j                  | j                  z  |j                  |j&                  ¬«      | _        | j
                  j0                  | _        | j                  dk(  r|j2                  nd | _        | j                  dk(  | _        t7        |j                  |j8                  ¬«      | _        t7        |j                  |j8                  ¬«      | _        y rã   rå   rõ   s      €r@   r8   z T5Gemma2MergedAttention.__init__O  rö   rA   Nrn   r÷   Úmerged_attention_maskÚencoder_hidden_statesrø   rÙ   r�   c                 ó¦  — |j                   d d }g |¢d‘| j                  ‘­}|j                   d d }	g |	¢d‘| j                  ‘­}
| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }| j                  |«      }| j                  |«      }|\  }}t        ||||«      \  }}|�]|j                  }|j                  ||| j                  «      \  }}|j                  j                  | j                  «      }|j                  }|�s­| j                  |«      j                  |
«      j	                  dd«      }| j                  |«      j                  |
«      j	                  dd«      }| j                  |«      }|�€j                  ||| j                  «      \  }}d|j                  | j                  <   nFj                   | j                     j"                  }|j                   | j                     j$                  }|}|	d   }t'        j(                  ||gd¬«      }t'        j(                  ||gd¬«      }t+        j,                  | j.                  j0                  t2        «      } || ||||f| j4                  r| j6                  nd| j8                  dœ|¤Ž\  }} |j:                  g |¢d‘­Ž j=                  «       }| j?                  |«      }|�|dd | …f   }|d| d …f   }nd	\  }}|||fS )
NrC   r,   r*   Tr    rú   )rk   rÌ   .)NN) rR   r�   rë   rû   r¥   rì   rí   ró   rô   r¿   Úself_attention_cacherü   rá   Ú
is_updatedÚgetÚcross_attention_cacheÚlayersÚkeysÚvaluesr;   r¦   r   rý   r]   rþ   rÞ   rÑ   rè   rÌ   rÂ   rØ   rî   )r>   rn   r÷   r  r	  rø   rÙ   rÿ   r   Úcross_input_shapeÚcross_hidden_shaper  rÚ   rÛ   r§   r¨   r  r  r  Úcross_key_statesÚcross_value_statesÚcross_key_sizer  rÝ   rÜ   Úself_attn_weightsÚcross_attn_weightss                              r@   rO   zT5Gemma2MergedAttention.forwardm  sv  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆØ1×7Ñ7¸¸Ð<ÐØDÐ0ÐD°"ÐD°d·m±mÑDÐð —{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà—{‘{ <Ó0ˆØ—[‘[ Ó,ˆ
à&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&à#2×#GÑ#GÐ Ø';×'BÑ'BÀ:È|Ð]a×]kÑ]kÓ'lÑ$ˆJ˜ð )×3Ñ3×7Ñ7¸¿¹ÓGˆJØ$3×$IÑ$IÐ!àÐ"©*Ø#Ÿ{™{Ð+@ÓA×FÑFÐGYÓZ×dÑdÐefÐhiÓjÐØ!%§¡Ð-BÓ!C×!HÑ!HÐI[Ó!\×!fÑ!fÐghÐjkÓ!lÐà#Ÿ{™{Ð+;Ó<ÐàÐ*Ø7L×7SÑ7SØ$Ð&8¸$¿.¹.ó8Ñ4Ð Ð"4ð >B�×*Ñ*¨4¯>©>Ò:à4×;Ñ;¸D¿N¹NÑK×PÑPÐØ!6×!=Ñ!=¸d¿n¹nÑ!M×!TÑ!TÐð $ˆØ*¨1Ñ-ˆÜ—Y‘Y 
Ð,<Ð=À1ÔEˆ
Ü—y‘y ,Ð0BÐ!CÈÔKˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØ!ð	%
ð /3¯mªm�D×*Ò*ÀØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆð Ð#Ø ,¨SÐ2B°N°?Ð2BÐ-BÑ CÐØ!-¨c°N°?Ñ3CÐ.CÑ!DÑà4>Ñ1ÐÐ1ØÐ-Ð/AÐAÐArA   r6   )rU   rV   rW   r  r0   rX   r8   r;   r¯   rQ   r
   r!   r   rO   rY   rZ   s   @r@   r  r  K  sÓ   ø„ á@ðTÐ1ð T¸cõ TðN 7;ñTBð —|‘|ðTBð # 5§<¡<°·±Ð#=Ñ>ð	TBð
  %Ÿ|™|¨dÑ2ðTBð  %Ÿ|™|ðTBð -¨tÑ3ðTBð Ð-Ñ.ðTBð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷TBrA   r  c                   óè   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	eej                  f   f
d
„Zˆ xZS )ÚT5Gemma2EncoderLayerzEncoder sub-layer.rá   c                 óD  •— t         ‰| �  «        |j                  | _        || _        || _        |j
                  |   | _        t        ||¬«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        |«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t#        j$                  |j&                  «      | _        y ©N)r]   rá   ©r4   )r7   r8   ra   r]   rá   r�   Úattention_typerà   Ú	self_attnr2   rò   Úpre_self_attn_layernormÚpost_self_attn_layernormr\   ÚmlpÚpre_feedforward_layernormÚpost_feedforward_layernormr9   ri   rj   rk   rõ   s      €r@   r8   zT5Gemma2EncoderLayer.__init__Ç  sâ   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØˆŒØ"ˆŒØ$×0Ñ0°Ñ;ˆÔä.ØØô
ˆŒô (7°v×7IÑ7IÈv×ObÑObÔ'cˆÔ$Ü(7¸×8JÑ8JÐPV×PcÑPcÔ(dˆÔ%ä˜vÓ&ˆŒÜ)8¸×9KÑ9KÐQW×QdÑQdÔ)eˆÔ&Ü*9¸&×:LÑ:LÐRX×ReÑReÔ*fˆÔ'ä—z‘z &×"5Ñ"5Ó6ˆ�rA   Nrn   r÷   rË   r©   r�   c           	      ó>  — |}| j                  |«      } | j                  d||||d dœ|¤Ž\  }}| j                  |«      }|| j                  |«      z   }|}| j	                  |«      }| j                  |«      }| j                  |«      }|| j                  |«      z   }|S )N)rn   r÷   rË   r©   rø   © ©r   r  r!  rk   r#  r"  r$  )r>   rn   r÷   rË   r©   rÙ   ÚresidualÚ_s           r@   rO   zT5Gemma2EncoderLayer.forwardÛ  s½   € ð !ˆØ×4Ñ4°]ÓCˆØ)˜4Ÿ>™>ð 
Ø'Ø 3Ø)Ø%Ø ñ
ð ñ
Ñˆ�qð ×5Ñ5°mÓDˆØ  4§<¡<°Ó#>Ñ>ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ  4§<¡<°Ó#>Ñ>ˆØÐrA   r  )rU   rV   rW   r  rX   r8   r;   r¯   rQ   Ú
LongTensorÚFloatTensorrO   rY   rZ   s   @r@   r  r  Ä  s‘   ø„ Ùð7¨#õ 7ð. IMØ.2Ø04ñà—|‘|ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ÿ™ tÑ+ð	ð
 ×&Ñ&¨Ñ-ðð 
ˆu× Ñ Ð!Ñ	"÷rA   r  c                   ó  ‡ — e Zd ZdZdefˆ fd„Z	 	 	 	 	 ddej                  deej                  ej                  f   dej                  dz  dej                  dz  d	e
dz  d
edz  dej                  dz  dej                  fd„Zˆ xZS )ÚT5Gemma2DecoderLayerzFDecoder sub-layer: merged attention instead of vanilla self-attention.rá   c                 óD  •— t         ‰| �  «        |j                  | _        || _        || _        |j
                  |   | _        t        ||¬«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        |«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t#        j$                  |j&                  «      | _        y r  )r7   r8   ra   r]   rá   r�   r  r  r  r2   rò   r   r!  r\   r"  r#  r$  r9   ri   rj   rk   rõ   s      €r@   r8   zT5Gemma2DecoderLayer.__init__û  sä   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØˆŒØ"ˆŒØ$×0Ñ0°Ñ;ˆÔô 1ØØô
ˆŒô (7°v×7IÑ7IÈv×ObÑObÔ'cˆÔ$Ü(7¸×8JÑ8JÐPV×PcÑPcÔ(dˆÔ%ä˜vÓ&ˆŒÜ)8¸×9KÑ9KÐQW×QdÑQdÔ)eˆÔ&Ü*9¸&×:LÑ:LÐRX×ReÑReÔ*fˆÔ'ä—z‘z &×"5Ñ"5Ó6ˆ�rA   Nrn   r÷   r  r©   rø   Ú	use_cacher	  r�   c                 óD  — |}	| j                  |«      } | j                  d|||||||dœ|¤Ž\  }}
}
| j                  |«      }|	| j                  |«      z   }|}	| j	                  |«      }| j                  |«      }| j                  |«      }|	| j                  |«      z   }|S )N)rn   r÷   r  r©   rø   r/  r	  r&  r'  )r>   rn   r÷   r  r©   rø   r/  r	  rÙ   r(  r)  s              r@   rO   zT5Gemma2DecoderLayer.forward  sÅ   € ð !ˆØ×4Ñ4°]ÓCˆà,˜dŸn™nð 	
Ø'Ø 3Ø"7Ø%Ø+ØØ"7ñ	
ð ñ	
Ñˆ�q˜!ð ×5Ñ5°mÓDˆØ  4§<¡<°Ó#>Ñ>ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ  4§<¡<°Ó#>Ñ>ˆØÐrA   )NNNFN)rU   rV   rW   r  rX   r8   r;   r¯   rQ   r*  r
   Úboolr+  rO   rY   rZ   s   @r@   r-  r-  ø  s¸   ø„ ÙPð7¨#õ 7ð2 6:Ø04Ø6:Ø!&Ø59ñ à—|‘|ð ð # 5§<¡<°·±Ð#=Ñ>ð ð  %Ÿ|™|¨dÑ2ð	 ð
 ×&Ñ&¨Ñ-ð ð -¨tÑ3ð ð ˜$‘;ð ð  %Ÿ|™|¨dÑ2ð ð 
×	Ñ	÷ rA   r-  c                   ój   ‡ — e Zd ZdZd	dededefˆ fd„Zdej                  dej                  fd„Z	ˆ xZ
S )
ÚT5Gemma2LMHeadz.Head for language modeling (generation) tasks.ra   Ú
vocab_sizer`   c                 ó\   •— t         ‰| �  «        t        j                  |||¬«      | _        y )Nr_   )r7   r8   r9   rc   Úout_proj)r>   ra   r4  r`   r?   s       €r@   r8   zT5Gemma2LMHead.__init__6  s"   ø€ Ü‰ÑÔÜŸ	™	 +¨zÀÔEˆ�rA   rn   r�   c                 ó(   — | j                  |«      }|S r6   )r6  )r>   rn   Úlogitss      r@   rO   zT5Gemma2LMHead.forward:  s   € Ø—‘˜}Ó-ˆØˆrA   )F)rU   rV   rW   r  rX   r1  r8   r;   r¯   rO   rY   rZ   s   @r@   r3  r3  3  s?   ø„ Ù8ñF Cð F°Sð FÀõ Fð U§\¡\ð °e·l±l÷ rA   r3  c                   ój   ‡ — e Zd ZdZd	dededefˆ fd„Zdej                  dej                  fd„Z	ˆ xZ
S )
ÚT5Gemma2ClassificationHeadz-Head for sentence-level classification tasks.ra   Ú
num_labelsÚclassifier_dropout_ratec                 óŽ   •— t         ‰| �  «        t        j                  |¬«      | _        t        j
                  ||«      | _        y )N)rÐ   )r7   r8   r9   ri   rk   rc   r6  )r>   ra   r;  r<  r?   s       €r@   r8   z#T5Gemma2ClassificationHead.__init__B  s1   ø€ Ü‰ÑÔÜ—z‘zÐ$;Ô<ˆŒÜŸ	™	 +¨zÓ:ˆ�rA   rn   r�   c                 óJ   — | j                  |«      }| j                  |«      }|S r6   )rk   r6  )r>   rn   s     r@   rO   z"T5Gemma2ClassificationHead.forwardG  s$   € ØŸ™ ]Ó3ˆØŸ™ mÓ4ˆØÐrA   )rú   ©rU   rV   rW   r  rX   rL   r8   r;   r¯   rO   rY   rZ   s   @r@   r:  r:  ?  s<   ø„ Ù7ñ; Cð ;°Sð ;ÐSXõ ;ð
 U§\¡\ð °e·l±l÷ rA   r:  c                   óD   ‡ — e Zd Zdefˆ fd„Zdej                  fd„Zˆ xZS )ÚT5Gemma2MultiModalProjectorr]   c                 óŽ  •— t         ‰| �  «        t        j                  t	        j
                  |j                  j                  |j                  j                  «      «      | _	        t        |j                  j                  |j                  j                  ¬«      | _        t        |j                  j                  |j                  j                  z  «      | _        t        |j"                  dz  «      | _        | j                   | j$                  z  | _        t        j(                  | j&                  | j&                  ¬«      | _        y )Nr  ç      à?)Úkernel_sizeÚstride)r7   r8   r9   r:   r;   r<   Úvision_configra   Útext_configÚmm_input_projection_weightr2   Úlayer_norm_epsÚmm_soft_emb_normrX   Ú
image_sizeÚ
patch_sizeÚpatches_per_imageÚmm_tokens_per_imageÚtokens_per_siderD  Ú	AvgPool2dÚavg_poolrl   s     €r@   r8   z$T5Gemma2MultiModalProjector.__init__N  sì   ø€ Ü‰ÑÔä*,¯,©,Ü�K‰K˜×,Ñ,×8Ñ8¸&×:LÑ:L×:XÑ:XÓYó+
ˆÔ'ô !0Ø× Ñ ×,Ñ,°&×2FÑ2F×2UÑ2Uô!
ˆÔô "% V×%9Ñ%9×%DÑ%DÈ×H\ÑH\×HgÑHgÑ%gÓ!hˆÔÜ" 6×#=Ñ#=¸sÑ#BÓCˆÔØ×1Ñ1°T×5IÑ5IÑIˆÔÜŸ™°×1AÑ1AÈ$×JZÑJZÔ[ˆ�rA   Úvision_outputsc                 ó¢  — |j                   \  }}}|j                  dd«      }|j                  ||| j                  | j                  «      }|j	                  «       }| j                  |«      }|j                  d«      }|j                  dd«      }| j                  |«      }t        j                  || j                  «      }|j                  |«      S )Nr,   r*   )rR   r¥   rÂ   rM  rØ   rQ  ÚflattenrJ  r;   rÓ   rH  rM   )	r>   rR  Ú
batch_sizer)  ra   Úreshaped_vision_outputsÚpooled_vision_outputsÚnormed_vision_outputsÚprojected_vision_outputss	            r@   rO   z#T5Gemma2MultiModalProjector.forward^  sÍ   € Ø%3×%9Ñ%9Ñ"ˆ
�A�{à"0×":Ñ":¸1¸aÓ"@ÐØ"9×"AÑ"AØ˜ T×%;Ñ%;¸T×=SÑ=Só#
Ðð #:×"DÑ"DÓ"FÐà $§¡Ð.EÓ FÐØ 5× =Ñ =¸aÓ @ÐØ 5× ?Ñ ?ÀÀ1Ó EÐà $× 5Ñ 5Ð6KÓ LÐä#(§<¡<Ð0EÀt×GfÑGfÓ#gÐ Ø'×/Ñ/°Ó?Ð?rA   )	rU   rV   rW   r/   r8   r;   r¯   rO   rY   rZ   s   @r@   rA  rA  M  s$   ø„ ð\Ð4õ \ð @ e§l¡l÷ @rA   rA  c                   ób   ‡ — e Zd ZdZ	 	 d
dededededef
ˆ fd„Zdej                  fˆ fd	„Z	ˆ xZ
S )ÚT5Gemma2TextScaledWordEmbeddingzCT5Gemma2 Embedding: override to add eoi token embedding separately.Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scaleÚeoi_token_indexc                 ó   •— t         ‰| �  |||«       || _        | j                  dt	        j
                  |«      d¬«       || _        t        j                  t	        j                  | j                  «      «      | _        y )Nr_  Frx   )r7   r8   Úscalar_embed_scaler„   r;   Útensorr`  r9   r:   r<   r]  Úeoi_embedding)r>   r\  r]  r^  r_  r`  r?   s         €r@   r8   z(T5Gemma2TextScaledWordEmbedding.__init__t  se   ø€ ô 	‰Ñ˜¨¸ÔDØ"-ˆÔØ×Ñ˜]¬E¯L©L¸Ó,EÐRWÐÔXØ.ˆÔÜŸ\™\¬%¯+©+°d×6HÑ6HÓ*IÓJˆÕrA   Ú	input_idsc                 óð   •— t         ‰| �  |«      | j                  j                  | j                  j
                  «      z  }| j                  j                  |j
                  «      ||| j                  k(  <   |S r6   )r7   rO   r_  r˜   r=   r’   rd  r`  )r>   re  Úinput_embeddingsr?   s      €r@   rO   z'T5Gemma2TextScaledWordEmbedding.forward‚  sf   ø€ Ü ™7™?¨9Ó5¸×8HÑ8H×8KÑ8KÈDÏKÉK×L]ÑL]Ó8^Ñ^ÐØ>B×>PÑ>P×>SÑ>SÐTd×TjÑTjÓ>kÐ˜ d×&:Ñ&:Ñ:Ñ;ØÐrA   )rK   é è r?  rZ   s   @r@   r[  r[  q  s^   ø„ ÙMð !Ø&ñKàðKð ðKð ð	Kð
 ðKð õKð  §¡÷  ñ  rA   r[  c                   ó¤   ‡ — e Zd ZU eed<   dZdZg d¢ZdgZdZ	dZ
dZdZdZdZdZ ej"                  «       ˆ fd	„«       Zd
ej&                  fd„Zˆ xZS )ÚT5Gemma2PreTrainedModelr]   ÚmodelT)r  r-  ÚSiglipVisionEmbeddingsÚSiglipEncoderLayerÚ#SiglipMultiheadAttentionPoolingHeadrø   FN)ÚimageÚtextc                 ó°  •— t         ‰| �  |«       t        |t        «      r t	        j
                  |j                  «       y t        |t        «      rJt	        j
                  |j                  «       t	        j                  |j                  |j                  «       y t        |t        «      rÁ|j                  j                  j                  d   dz  }t	        j                   |j                  j                  d| j"                  j$                  |z  ¬«       t'        |j                  d«      rA|j                  j(                  �*t	        j
                  |j                  j(                  «       y y y d|j*                  j,                  v r t	        j
                  |j                  «       y t        |t.        «      r£|j0                  D ]“  }|j2                  }|j4                  |   dk7  rt6        |j4                  |      } ||j"                  |¬«      \  }}t	        j8                  t;        ||› d	�«      |«       t	        j8                  t;        ||› d
�«      |«       Œ• y y )Nr   rÏ   rú   )rG   Ústdr`   ÚRMSNormrt   ru   rw   rz   )r7   Ú_init_weightsr¢   rA  ÚinitÚzeros_rH  r[  rd  Ú	constant_r_  rb  r:  r6  r=   rR   Únormal_r]   Úinitializer_rangeræ   r`   r?   rU   rp   r�   rƒ   rs   r   Úcopy_r”   )r>   rÇ   Úscalerv   r‰   rŠ   r)  r?   s          €r@   rt  z%T5Gemma2PreTrainedModel._init_weights¤  sÂ  ø€ ä‰Ñ˜fÔ%Ü�fÔ9Ô:Ü�K‰K˜×9Ñ9Õ:Ü˜Ô ?Ô@Ü�K‰K˜×,Ñ,Ô-Ü�N‰N˜6×-Ñ-¨v×/HÑ/HÕIÜ˜Ô :Ô;Ø—O‘O×*Ñ*×0Ñ0°Ñ3°tÑ;ˆEÜ�L‰L˜Ÿ™×/Ñ/°c¸t¿{¹{×?\Ñ?\Ð_dÑ?dÕeÜ�v—‘¨Ô/°F·O±O×4HÑ4HÐ4TÜ—‘˜FŸO™O×0Ñ0Õ1ð 5UÐ/ð ˜&×*Ñ*×3Ñ3Ñ3Ü�K‰K˜Ÿ™Õ&Ü˜Ô 7Ô8Ø$×0Ñ0ò ^�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ò<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ô#UÑ �˜qÜ—
‘
œ7 6¨j¨\¸Ð+CÓDÀmÔTÜ—
‘
œ7 6¨j¨\Ð9KÐ+LÓMÈ}Õ]ñ^ð 9rA   Úlabelsc                 ó<  — | j                   j                  }|j                  }|j                  }|€t	        d«      ‚|j                  |j                  «      }|ddd…f   j                  «       |ddd…f<   ||d<   |€t	        d«      ‚|j                  |dk(  |«       |S )	zú
        Shifts input_ids to the right, prepends the decoder_start_token_id, and handles
        pad_token_id replacement for labels that were -100.
        This is a common preparation step for decoder inputs in sequence-to-sequence models.
        Nz:self.model.config.decoder.bos_token_id has to be defined. .rC   r,   ).r   z9self.model.config.decoder.pad_token_id has to be defined.iœÿÿÿ)	r]   ÚdecoderÚbos_token_idÚpad_token_idÚ
ValueErrorÚ	new_zerosrR   r…   Úmasked_fill_)r>   r|  Údecoder_configÚdecoder_start_token_idr€  Úshifted_input_idss         r@   Ú%prepare_decoder_input_ids_from_labelsz=T5Gemma2PreTrainedModel.prepare_decoder_input_ids_from_labels½  sµ   € ð Ÿ™×,Ñ,ˆØ!/×!<Ñ!<ÐØ%×2Ñ2ˆà!Ð)ÜÐYÓZÐZð #×,Ñ,¨V¯\©\Ó:ÐØ%+¨C°°"°¨HÑ%5×%;Ñ%;Ó%=Ð˜#˜q™r˜'Ñ"Ø$:Ð˜&Ñ!àÐÜÐXÓYÐYð 	×&Ñ&Ð'8¸DÑ'@À,ÔOà Ð rA   )rU   rV   rW   r-   r°   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendÚ_can_record_outputsÚinput_modalitiesr;   r²   rt  r¯   r‡  rY   rZ   s   @r@   rj  rj  ˆ  s   ø… àÓØÐØ&*Ð#òÐð $5Ð"5Ðð !ÐØ€NàÐà!ÐØ"&ÐàÐØ(Ðà€U‡]�]ƒ_ó^ó ð^ð0!¸E¿L¹L÷ !rA   rj  rð   c           
      óT   ‡ ‡— dt         dt         dt         dt         dt        f
ˆˆ fd„}|S )zL
    This creates uni/bidirectional attention mask with sliding window.
    Ú	batch_idxÚhead_idxÚq_idxÚkv_idxr�   c                 ót   •— ‰	r‰
d}}n‰
dz   dz  ‰
dz  dz   }}||z
  }|dk\  ||k  z  }|dk  | |k  z  }||z  S )Nr   r,   r*   r&  )r”  r•  r–  r—  Úleft_window_sizeÚright_window_sizeÚdistÚ	left_maskÚ
right_maskré   rð   s            €€r@   Ú
inner_maskz0sliding_window_mask_function.<locals>.inner_maskÞ  sp   ø€ ÙØ2@À!Ð/Ñà4BÀQÑ4FÈ1Ñ3LÈ~ÐbcÑNcÐfgÑNgÐ/Ðà�v‰~ˆØ˜Q‘Y 4Ð*:Ñ#:Ñ;ˆ	Ø˜Q‘h D 5Ð+<Ñ#<Ñ=ˆ
Ø˜:Ñ%Ð%rA   )rX   r1  )rð   ré   rž  s   `` r@   Úsliding_window_mask_functionrŸ  Ù  s3   ù€ ð
	&œcð 	&¬Sð 	&¼ð 	&Äcð 	&Ìdö 	&ð ÐrA   c                   ó  ‡ — e Zd ZU eed<   eedœZ	 ddedefˆ fd„Z	e
ee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  d	ej                   dz  d
ej                  dz  dee   defd„«       «       «       Zˆ xZS )ÚT5Gemma2TextEncoderr]   )Ú
attentionsrn   r`  c           	      ó^  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        |j                  |j                  | j                  |j                  dz  |¬«      | _        t        |j                  |j                  ¬«      | _
        d| _        t        j                  t        |j                  «      D �cg c]  }t!        ||«      ‘Œ c}«      | _        t        j$                  |j&                  «      | _        t+        |«      | _        | j/                  «        y c c}w ©NrC  )r_  r`  r  F)r7   r8   r€  r^  r4  r[  ra   Úembed_tokensr2   rò   ÚnormÚgradient_checkpointingr9   Ú
ModuleListÚrangeÚnum_hidden_layersr  r  ri   rj   rk   rp   Ú
rotary_embÚ	post_init©r>   r]   r`  rá   r?   s       €r@   r8   zT5Gemma2TextEncoder.__init__ó  sñ   ø€ ô
 	‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒä;Ø×ÑØ×ÑØ×ÑØ×*Ñ*¨CÑ/Ø+ô
ˆÔô $ F×$6Ñ$6¸F×<OÑ<OÔPˆŒ	Ø&+ˆÔ#ä—m‘mÜFKÈF×LdÑLdÓFeÖf¸Ô! &¨)Õ4Òfó
ˆŒô —z‘z &×"5Ñ"5Ó6ˆŒÜ1°&Ó9ˆŒð 	�‰Õùò gó   ÃD*Nre  rË   r©   Úinputs_embedsÚtoken_type_idsrÙ   r�   c           
      óŽ  — |d u |d uz  rt        d«      ‚|j                  dd «       |€| j                  |«      }|€>t        j                  d|j
                  d   |j                  ¬«      j                  d«      }t        |x}t        «      sJ| j                  ||dœ}t        di |¤Žt        di |¤dt        | j                  j                  d¬	«      i¤Žd
œ}|}	i }
t        | j                  j                  «      D ]  }| j!                  |	||«      |
|<   Œ | j#                  |	«      }	t%        | j&                  d | j                  j(                   «      D ]E  \  }} ||	|
| j                  j                  |      || j                  j                  |      |fi |¤Ž}	ŒG | j+                  |	«      }	| j#                  |	«      }	t-        |	¬«      S )Nú:You must specify exactly one of input_ids or inputs_embedsrø   r   r,   ©r‡   )r]   r¯  rË   Úand_mask_functionF)ré   ©Úfull_attentionrä   )Úlast_hidden_stater&  )r�  Úpopr¥  r;   r–   rR   r‡   r¹   r¢   Údictr]   r   rŸ  rð   r€   r�   r«  rk   Ú	enumerater  rª  r¦  r   )r>   re  rË   r©   r¯  r°  rÙ   Úself_attn_mask_mappingÚmask_kwargsrn   r÷   rv   ÚiÚlayer_modules                 r@   rO   zT5Gemma2TextEncoder.forward  sÝ  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZð 	�
‰
Ð$ dÔ+àÐ Ø ×-Ñ-¨iÓ8ˆMàÐÜ Ÿ<™<¨¨=×+>Ñ+>¸qÑ+AÈ-×J^ÑJ^Ô_×iÑiÐjkÓlˆLä°NÐBÐ0ÄDÔIàŸ+™+Ø!.Ø"0ñˆKô #<Ñ"J¸kÑ"JÜ%>ñ &Ø!ñ&ä&BÀ4Ç;Á;×C]ÑC]ÐinÔ&oò&ñ&Ð"ð &ˆð !ÐÜ˜dŸk™k×5Ñ5Ó6ò 	gˆJØ.2¯o©o¸mÈ\Ð[eÓ.fÐ 
Ò+ð	gð Ÿ™ ]Ó3ˆä(¨¯©Ð5T°t·{±{×7TÑ7TÐ)UÓVò 	‰OˆAˆ|Ù(ØØ# D§K¡K×$;Ñ$;¸AÑ$>Ñ?Ø& t§{¡{×'>Ñ'>¸qÑ'AÑBØñ	ð
 ñ‰Mð	ð Ÿ	™	 -Ó0ˆØŸ™ ]Ó3ˆÜØ+ô
ð 	
rA   ©rh  )NNNNN)rU   rV   rW   r0   r°   rà   r  r‘  rX   r8   r'   r)   r#   r;   r*  r¯   r+  r!   r"   r   rO   rY   rZ   s   @r@   r¡  r¡  ì  så   ø… ØÓà+Ø-ñÐð  'ñà"ðð õð8  ØØð .2Ø.2Ø04Ø26à.2ñ<
à×#Ñ# dÑ*ð<
ð Ÿ™ tÑ+ð<
ð ×&Ñ&¨Ñ-ð	<
ð
 ×(Ñ(¨4Ñ/ð<
ð Ÿ™ tÑ+ð<
ð Ð+Ñ,ð<
ð 
ò<
ó ó ó  ô<
rA   r¡  c                   óÊ  ‡ — e Zd ZU eed<   	 ddedefˆ fd„Zd„ Zd„ Ze	e
dej                  dee   deez  fd	„«       «       Zd
ej$                  dz  dej&                  dz  dej&                  fd„Ze
	 	 	 	 	 	 dd
ej$                  dz  dej                  dz  dej$                  dz  dej&                  dz  dej&                  dz  dej                  dz  dee   defd„«       Zˆ xZS )ÚT5Gemma2Encoderr]   r`  c                 óú   •— t         ‰| �  |«       t        j                  |j                  |¬«      | _        t        j                  |j                  ¬«      | _	        t        |«      | _        | j                  «        y )N)r`  ©r]   )r7   r8   r¡  Ú_from_configrG  Ú
text_modelr+   Úfrom_configrF  Úvision_towerrA  Úmulti_modal_projectorr¬  )r>   r]   r`  r?   s      €r@   r8   zT5Gemma2Encoder.__init__T  sb   ø€ ô
 	‰Ñ˜Ô ä-×:Ñ:¸6×;MÑ;MÐ_nÐ:ÓoˆŒÜ%×1Ñ1¸×9MÑ9MÔNˆÔÜ%@ÀÓ%HˆÔ"ð 	�‰ÕrA   c                 ó6   — | j                   j                  «       S r6   )rÅ  Úget_input_embeddingsrS   s    r@   rÊ  z$T5Gemma2Encoder.get_input_embeddingsb  s   € Ø�‰×3Ñ3Ó5Ð5rA   c                 ó8   — | j                   j                  |«      S r6   )rÅ  Úset_input_embeddings©r>   Únew_embeddingss     r@   rÌ  z$T5Gemma2Encoder.set_input_embeddingse  s   € Ø�‰×3Ñ3°NÓCÐCrA   Úpixel_valuesrÙ   r�   c                 óx   —  | j                   d|ddœ|¤Ž}|j                  }| j                  |«      }||_        |S )NT)rÏ  Úreturn_dictr&  )rÇ  r·  rÈ  Úpooler_output)r>   rÏ  rÙ   rR  r·  Úimage_featuress         r@   Úget_image_featuresz"T5Gemma2Encoder.get_image_featuresh  sM   € ð +˜×*Ñ*Ða¸ÐRVÑaÐZ`ÑaˆØ*×<Ñ<ÐØ×3Ñ3Ð4EÓFˆØ'5ˆÔ$àÐrA   re  Nr¯  rÓ  c                 ó$  — | j                   j                  }|€f|€t        d«      ‚| | j                  «       t	        j
                  |t        j                  |j                  ¬«      «      k(  }|j                  d«      }n||k(  }|j                  «       }|j                  d«      j                  |j                  «      }|j                  d   |j                  d   z  }t        ||j                  d   z  |j                  «       k(  d|› d|› �«       |S )zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        z9Either `input_ids` or `inputs_embeds` has to be provided.)r’   r‡   rC   r   r,   z6Image features and image tokens do not match: tokens: z, features )r]   Úimage_token_idr�  rÊ  r;   rc  Úlongr‡   ÚallÚsumr¹   r˜   rR   r%   Únumel)r>   re  r¯  rÓ  rÖ  Úspecial_image_maskÚn_image_tokensÚn_image_featuress           r@   Úget_image_placeholder_maskz*T5Gemma2Encoder.get_image_placeholder_maskv  s  € ð Ÿ™×3Ñ3ˆØÐØÐ$Ü Ð!\Ó]Ð]Ø!.Ð2M°$×2KÑ2KÓ2MÜ—‘˜^´5·:±:Àm×FZÑFZÔ[ó3ñ "Ðð "4×!7Ñ!7¸Ó!;Ñà!*¨nÑ!<Ðà+×/Ñ/Ó1ˆØ/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐØ)×/Ñ/°Ñ2°^×5IÑ5IÈ!Ñ5LÑLÐÜØ˜]×0Ñ0°Ñ4Ñ4¸×8LÑ8LÓ8NÑNØDÀ^ÐDTÐT_Ð`pÐ_qÐrô	
ð "Ð!rA   rË   r©   r°  c                 ój  — |d u |d uz  rt        d«      ‚|€| j                  j                  |«      }|�i| j                  |d¬«      j                  }|j                  |j                  |j                  «      }| j                  |||¬«      }	|j                  |	|«      } | j                  d|||dœ|¤Ž}
|
S )Nr²  T)rÑ  )r¯  rÓ  )r¯  rË   r©   r&  )
r�  rÅ  r¥  rÔ  rÒ  r˜   r‡   r’   rÞ  Úmasked_scatter)r>   re  rË   r©   r¯  rÏ  r°  rÙ   rÓ  Ú
image_maskÚoutputss              r@   rO   zT5Gemma2Encoder.forward”  sÚ   € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ŸO™O×8Ñ8¸ÓCˆMàÐ#Ø!×4Ñ4°\ÈtÐ4ÓT×bÑbˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNà×8Ñ8Ø¨À~ð 9ó ˆJð *×8Ñ8¸À^ÓTˆMà!�$—/‘/ð 
Ø'Ø)Ø%ñ
ð ñ	
ˆð ˆrA   r¿  )NNNNNN)rU   rV   rW   r/   r°   rX   r8   rÊ  rÌ  r$   r#   r;   r¯   r!   r"   rQ   r   rÔ  r*  r+  rÞ  r   rO   rY   rZ   s   @r@   rÁ  rÁ  Q  sk  ø… Ø!Ó!ð
  'ñà%ðð õò6òDð Øð
Ø!ŸL™Lð
Ø4:Ð;MÑ4Nð
à	Ð+Ñ	+ò
ó ó ð
ð"à×#Ñ# dÑ*ð"ð ×(Ñ(¨4Ñ/ð"ð ×)Ñ)ó	"ð< ð .2Ø.2Ø04Ø26Ø15à.2ñ!à×#Ñ# dÑ*ð!ð Ÿ™ tÑ+ð!ð ×&Ñ&¨Ñ-ð	!ð
 ×(Ñ(¨4Ñ/ð!ð ×'Ñ'¨$Ñ.ð!ð Ÿ™ tÑ+ð!ð Ð+Ñ,ð!ð 
ò!ó ô!rA   rÁ  c                   óp  ‡ — e Zd ZU eed<    eed¬«       eed¬«      edœZddede	fˆ fd„Z
eee	 	 	 	 	 	 	 	 dd	ej                  dz  d
ej                   dz  dej                  dz  dedz  dej$                  dz  dedz  dej                   dz  dej                   dz  dee   defd„«       «       «       Zˆ xZS )ÚT5Gemma2Decoderr]   r,   )Úindexr*   )r¢  Úcross_attentionsrn   r`  c           	      ó^  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        |j                  |j                  |j                  |j                  dz  |¬«      | _        t        |j                  |j                  ¬«      | _
        d| _        t        j                  t        |j                  «      D �cg c]  }t!        ||«      ‘Œ c}«      | _        t        j$                  |j&                  «      | _        t+        |«      | _        | j/                  «        y c c}w r¤  )r7   r8   r€  r^  r4  r[  ra   r¥  r2   rò   r¦  r§  r9   r¨  r©  rª  r-  r  ri   rj   rk   rp   r«  r¬  r­  s       €r@   r8   zT5Gemma2Decoder.__init__Á  sí   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒä;Ø×ÑØ×ÑØ×ÑØ×*Ñ*¨CÑ/Ø+ô
ˆÔô $ F×$6Ñ$6¸F×<OÑ<OÔPˆŒ	Ø&+ˆÔ#ä—m‘mÜFKÈF×LdÑLdÓFeÖf¸Ô! &¨)Õ4Òfó
ˆŒô —z‘z &×"5Ñ"5Ó6ˆŒÜ1°&Ó9ˆŒØ�‰Õùò	 gr®  Nre  rË   r©   rø   r¯  r/  r	  Úencoder_attention_maskrÙ   r�   c	           
      óè  — |d u |d uz  rt        d«      ‚|€t        d«      ‚|€| j                  |«      }| j                  s,|r*|€(t        t	        | j
                  ¬«      t	        «       «      }|€V|�|j                  «       nd}
t        j                  |j                  d   |j                  ¬«      |
z   }|j                  d«      }t        |x}t        «      s;d„ }| j
                  |||�|j                  nd ||dœ}t        di |¤Žt!        di |¤Žd	œ}t        |x}t        «      sd
t#        | j
                  |||¬«      i}t        j$                  |d
   |d
   gd¬«      t        j$                  |d   |d
   gd¬«      d	œ}|}i }t'        | j
                  j(                  «      D ]  }| j+                  |||«      ||<   Œ | j-                  |«      }t/        | j0                  d | j
                  j2                   «      D ]H  \  }} |||| j
                  j(                  |      || j
                  j(                  |      ||||fi |	¤Ž}ŒJ | j5                  |«      }| j-                  |«      }t7        ||¬«      S )Nr²  z0`encoder_hidden_states` must be given in decoderrÃ  r   r,   r³  c                  óL   — t        j                  dt         j                  ¬«      S )NTr‘   )r;   rc  r1  )Úargss    r@   ú<lambda>z)T5Gemma2Decoder.forward.<locals>.<lambda>ù  s   € ´E·L±LÀÌUÏZÉZÔ4X€ rA   )r]   r¯  rË   rø   r©   r´  rµ  r¶  )r]   r¯  rË   r	  r´  rC   r    rä   )r·  rø   r&  )r�  r¥  rÑ   r
   r	   r]   Úget_seq_lengthr;   r–   rR   r‡   r¹   r¢   r¹  r  r   r   r   r¦   r€   r�   r«  rk   rº  r  rª  r¦  r   )r>   re  rË   r©   rø   r¯  r/  r	  rè  rÙ   Úpast_seen_tokensr»  Údummy_and_mask_functionr¼  Úcross_attn_mask_mappingÚmerged_attn_mask_mappingrn   r÷   rv   r½  r¾  s                        r@   rO   zT5Gemma2Decoder.forward×  s¾  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZØ Ð(ÜÐOÓPÐPàÐ Ø ×-Ñ-¨iÓ8ˆMà�}Š}¡¨Ð/FÜ1´,ÀdÇkÁkÔ2RÔT`ÓTbÓcˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä°NÐBÐ0ÄDÔIñ 'YÐ#àŸ+™+Ø!.Ø"0ØKZÐKf ?×#GÒ#GÐlpØ ,Ø%<ñˆKô #5Ñ"C°{Ñ"CÜ%FÑ%UÈÑ%Uñ&Ð"ô
 Ð5KÐKÐ1ÌTÔRà Ô";ØŸ;™;Ø"/Ø#9Ø*?Ø&=ô#ð'Ð#ô $Ÿi™iØ'Ð(8Ñ9Ð;RÐScÑ;dÐeÐkmôô "'§¡Ø'Ð(;Ñ<Ð>UÐVfÑ>gÐhÐnpô"ñ	$
Ð ð &ˆð !ÐÜ˜dŸk™k×5Ñ5Ó6ò 	gˆJØ.2¯o©o¸mÈ\Ð[eÓ.fÐ 
Ò+ð	gð Ÿ™ ]Ó3ˆä(¨¯©Ð5T°t·{±{×7TÑ7TÐ)UÓVò 
	‰OˆAˆ|Ù(ØØ# D§K¡K×$;Ñ$;¸AÑ$>Ñ?Ø(¨¯©×)@Ñ)@ÀÑ)CÑDØØØØ%ñ	ð ñ	‰Mð
	ð Ÿ	™	 -Ó0ˆØŸ™ ]Ó3ˆÜ8Ø+Ø+ô
ð 	
rA   r¿  )NNNNNNNN)rU   rV   rW   r.   r°   r(   r  r-  r‘  rX   r8   r'   r)   r#   r;   r*  r¯   r
   r+  r1  r!   r"   r   rO   rY   rZ   s   @r@   rä  rä  ¹  s5  ø… Ø!Ó!á$Ð%<ÀAÔFÙ*Ð+BÈ!ÔLØ-ñÐñÐ4ð Àsõ ð,  ØØð .2Ø.2Ø04Ø6:Ø26Ø!%Ø59Ø6:ñ]
à×#Ñ# dÑ*ð]
ð Ÿ™ tÑ+ð]
ð ×&Ñ&¨Ñ-ð	]
ð
 -¨tÑ3ð]
ð ×(Ñ(¨4Ñ/ð]
ð ˜$‘;ð]
ð  %Ÿ|™|¨dÑ2ð]
ð !&§¡¨tÑ 3ð]
ð Ð+Ñ,ð]
ð 
3ò]
ó ó ó  ô]
rA   rä  c                   óÄ  ‡ — e Zd ZdddœZdefˆ fd„Zddedz  fd„Zd	„ Zd
„ Z	d„ Z
ee	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                   dz  dej                  dz  dedz  dedz  dej&                  dz  dej&                  dz  dedz  dee   defd„«       «       Zˆ xZS )ÚT5Gemma2Modelz&encoder.text_model.embed_tokens.weightz-encoder.text_model.embed_tokens.eoi_embedding)zdecoder.embed_tokens.weightz"decoder.embed_tokens.eoi_embeddingr]   c                 óØ   •— t         ‰| �  |«       t        |j                  |j                  «      | _        t        |j                  |j                  «      | _        | j                  «        y r6   )r7   r8   rÁ  Úencoderr`  rä  r~  r¬  rl   s     €r@   r8   zT5Gemma2Model.__init__A  sL   ø€ Ü‰Ñ˜Ô ô ' v§~¡~°v×7MÑ7MÓNˆŒÜ& v§~¡~°v×7MÑ7MÓNˆŒà�‰ÕrA   NÚmodalityc                 ó   — | j                   S r6   )rõ  ©r>   rö  s     r@   Úget_encoderzT5Gemma2Model.get_encoderJ  ó   € Ø�|‰|ÐrA   c                 ó   — | j                   S r6   ©r~  rS   s    r@   Úget_decoderzT5Gemma2Model.get_decoderM  rú  rA   c                 ó6   — | j                   j                  «       S r6   )rõ  rÊ  rS   s    r@   rÊ  z"T5Gemma2Model.get_input_embeddingsP  s   € Ø�|‰|×0Ñ0Ó2Ð2rA   c                 ó8   — | j                   j                  |«      S r6   )rõ  rÌ  rÍ  s     r@   rÌ  z"T5Gemma2Model.set_input_embeddingsS  s   € Ø�|‰|×0Ñ0°Ó@Ð@rA   re  rÏ  rË   r©   Údecoder_input_idsÚdecoder_attention_maskÚdecoder_position_idsÚencoder_outputsrø   r¯  Údecoder_inputs_embedsr/  rÙ   r�   c                 óN  — |€ | j                   d||||
|ddœ|¤Ž}|j                  } | j                  d|||||	|||ddœ	|¤Ž}t        |j                  |j                  |j
                  |j                  |j                  |j                  |j
                  |j                  ¬«      S )aX  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        T)re  rË   r©   r¯  rÏ  rÑ  )	re  rË   r©   r¯  rø   r	  rè  r/  rÑ  )r·  rø   Údecoder_hidden_statesÚdecoder_attentionsræ  Úencoder_last_hidden_stater	  Úencoder_attentionsr&  )rõ  r·  r~  r   rø   rn   r¢  ræ  )r>   re  rÏ  rË   r©   r   r  r  r  rø   r¯  r  r/  rÙ   r	  Údecoder_outputss                   r@   rO   zT5Gemma2Model.forwardV  sÜ   € ð6 Ð"Ø*˜dŸl™lð Ø#Ø-Ø)Ø+Ø)Ø ñð ñˆOð !0× AÑ AÐð '˜$Ÿ,™,ð 
Ø'Ø1Ø-Ø/Ø+Ø"7Ø#1ØØñ
ð ñ
ˆô "Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
rA   r6   )NNNNNNNNNNNN)rU   rV   rW   Ú_tied_weights_keysr-   r8   r¤   rù  rý  rÊ  rÌ  r$   r#   r;   r*  r+  Ú
BoolTensorr   r
   r¯   r1  r!   r"   r   rO   rY   rZ   s   @r@   ró  ró  :  s�  ø„ ð (PØ.]ñÐð
˜~õ ñ C¨$¡Jó òò3òAð Øð .2Ø15Ø37Ø04à59Ø:>Ø8<à26Ø6:Ø-1Ø59Ø!%ñ!=
ð ×#Ñ# dÑ*ð=
ð ×'Ñ'¨$Ñ.ð	=
ð
 ×)Ñ)¨DÑ0ð=
ð ×&Ñ&¨Ñ-ð=
ð !×+Ñ+¨dÑ2ð=
ð !&× 0Ñ 0°4Ñ 7ð=
ð $×.Ñ.°Ñ5ð=
ð )¨4Ñ/ð=
ð -¨tÑ3ð=
ð —|‘| dÑ*ð=
ð  %Ÿ|™|¨dÑ2ð=
ð  ˜$‘;ð!=
ð" Ð+Ñ,ð#=
ð$ 
ò%=
ó ó ô=
rA   ró  c            $       óÂ  ‡ — e Zd ZddiZddiZddgdgfiZdefˆ fd„Zd	„ Zd
„ Z	d„ Z
d„ Zd*dedz  fd„Zd„ Zeedej$                  dee   deez  fd„«       «       Zed„ «       Zee	 	 	 	 	 	 	 	 	 	 	 	 	 	 d+dej4                  dz  dej6                  dz  dej6                  dz  dej4                  dz  dej4                  dz  dej8                  dz  dej4                  dz  dedz  dedz  dej6                  dz  dej6                  dz  d ej4                  dz  d!edz  d"e ej$                  z  dee   deej6                     e!z  f d#„«       «       Z"d$e#d%e$d&e%d'e d(e defˆ fd)„Z&ˆ xZ'S ),Ú T5Gemma2ForConditionalGenerationzlm_head.out_proj.weightz,model.encoder.text_model.embed_tokens.weightzlm_head.out_projÚcolwise_gather_outputrn   r8  r]   c                 ó  •— t         ‰| �  |«       t        |«      | _        |j                  j
                  | _        t        |j                  j                  | j
                  «      | _        d| _	        | j                  «        y )NÚForMaskedLM)r7   r8   ró  rk  r~  r4  r3  ra   Úlm_headÚ	loss_typer¬  rl   s     €r@   r8   z)T5Gemma2ForConditionalGeneration.__init__Ÿ  sZ   ø€ Ü‰Ñ˜Ô ä" 6Ó*ˆŒ
Ø Ÿ.™.×3Ñ3ˆŒÜ% f§n¡n×&@Ñ&@À$Ç/Á/ÓRˆŒØ&ˆŒà�‰ÕrA   c                 ó&   — || j                   _        y r6   ©r  r6  rÍ  s     r@   Úset_output_embeddingsz6T5Gemma2ForConditionalGeneration.set_output_embeddings©  s   € Ø .ˆ�‰ÕrA   c                 ó.   — | j                   j                  S r6   r  rS   s    r@   Úget_output_embeddingsz6T5Gemma2ForConditionalGeneration.get_output_embeddings¬  s   € Ø�|‰|×$Ñ$Ð$rA   c                 ó6   — | j                   j                  «       S r6   ©rk  rÊ  rS   s    r@   rÊ  z5T5Gemma2ForConditionalGeneration.get_input_embeddings¯  ó   € Ø�z‰z×.Ñ.Ó0Ð0rA   c                 ó:   — | j                   j                  |«       y r6   ©rk  rÌ  ©r>   rÊ   s     r@   rÌ  z5T5Gemma2ForConditionalGeneration.set_input_embeddings²  ó   € Ø�
‰
×'Ñ'¨Õ.rA   Nrö  c                 ó:   — | j                   j                  |¬«      S )N)rö  )rk  rù  rø  s     r@   rù  z,T5Gemma2ForConditionalGeneration.get_encoderµ  s   € Ø�z‰z×%Ñ%¨xÐ%Ó8Ð8rA   c                 ó6   — | j                   j                  «       S r6   )rk  rý  rS   s    r@   rý  z,T5Gemma2ForConditionalGeneration.get_decoder¸  s   € Ø�z‰z×%Ñ%Ó'Ð'rA   rÏ  rÙ   r�   c                 óD   —  | j                  «       j                  |fi |¤ŽS r6   )rù  rÔ  )r>   rÏ  rÙ   s      r@   rÔ  z3T5Gemma2ForConditionalGeneration.get_image_features»  s%   € ð
 5ˆt×ÑÓ!×4Ñ4°\ÑLÀVÑLÐLrA   c                 ó6   — | j                  «       j                  S r6   )rù  rÇ  rS   s    r@   rÇ  z-T5Gemma2ForConditionalGeneration.vision_towerÂ  s   € à×ÑÓ!×.Ñ.Ð.rA   re  rË   r©   r   r  r  r  rø   r¯  r  r|  r/  Úlogits_to_keepc                 óž  — |�|€|€| j                  |«      } | j                  d|||||||||	|
||dœ|¤Ž}|j                  }t        |t        «      rt        | d«      n|}| j                  |dd…|dd…f   «      }| j                  j                  }|j                  �3||j                  z  }t        j                  |«      }||j                  z  }d}|� | j                  ||| j                  fi |¤Ž}t        |||j                  |j                   |j"                  |j$                  |j&                  |j(                  |j*                  ¬«	      S )aü  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N)re  rÏ  rË   r©   r   r  r  r  rø   r¯  r  r/  )	Úlossr8  rø   r  r  ræ  r  r	  r	  r&  )r‡  rk  r·  r¢   rX   Úslicer  r]   r~  Úfinal_logit_softcappingr;   rÔ   Úloss_functionr4  r   rø   r  r  ræ  r  r	  r	  )r>   re  rÏ  rË   r©   r   r  r  r  rø   r¯  r  r|  r/  r$  rÙ   r
  rn   Úslice_indicesr8  r„  r&  s                         r@   rO   z(T5Gemma2ForConditionalGeneration.forwardÆ  sr  € ðB ÐÐ"3Ð";Ð@UÐ@]à $× JÑ JÈ6Ó RÐà.8¨d¯j©jð /
ØØ%Ø)Ø%Ø/Ø#9Ø!5Ø+Ø+Ø'Ø"7Øñ/
ð ñ/
ˆð  (×9Ñ9ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàŸ™×,Ñ,ˆØ×1Ñ1Ð=Ø˜n×DÑDÑDˆFÜ—Z‘Z Ó'ˆFØ˜n×DÑDÑDˆFàˆØÐà%�4×%Ñ% f¨f°d·o±oÑPÈÑPˆDäØØØ+×;Ñ;Ø"1×"GÑ"GØ.×AÑAØ,×=Ñ=Ø&5×&OÑ&OØ"1×"GÑ"GØ.×AÑAô

ð 
	
rA   Úgeneration_configÚmodel_kwargsÚgeneration_moderU  Úmax_cache_lengthc           	      ó€  •— t         ‰| �  |||||«       |j                  du ry|j                  }|€d}nd|j                  v }t	        j
                  | j                  j                  d¬«      «      }d|_        dg|j                  z  |_
        ||dœ}	|j                  d«      }
|
�“t        |
t        «      st        d	«      ‚t        |
j                   «      d
kD  r|
j                   j                  d
«      ryt#        |
j$                  «      }|t&        k(  r|d   d
   j(                  d   |	d<    |di |	¤Ž|
_        n=t        t+        di | j                  j                  d¬«      |dœ¤Žt+        «       «      |d<   t-        | d«      r=| j.                  �0t        | j.                  t        «      st        d«      ‚|d   | _        yyy)zMOverride cache preparation to support T5Gemma2-specific EncoderDecoder Cache.FNÚ	offloadedTrü  r¶  )r]   Ú
offloadingrø   zaThe `past_key_values` in `model_kwargs` must be of type `EncoderDecoderCache` for T5Gemma2 model.r   r  r,   Úmax_cache_lenÚ_cachezLThe internal cache must be of type `EncoderDecoderCache` for T5Gemma2 model.r&  )r7   Ú_prepare_cache_for_generationr/  Úcache_implementationÚcopyÚdeepcopyr]   Úget_text_configrð   rª  r�   r  r¢   r
   r�  Úlenr  r£   r  r   rR   r	   ræ   r3  )r>   r+  r,  r-  rU  r.  r5  Úoffload_cacheÚcross_attn_configÚcross_attn_cache_kwargsrø   Úcross_attn_clsr?   s               €r@   r4  z>T5Gemma2ForConditionalGeneration._prepare_cache_for_generation  sç  ø€ ô 	‰Ñ-ØØØØØô	
ð ×&Ñ&¨%Ñ/Øà0×EÑEÐØÐ'Ø!‰Mà'Ð+<×+QÑ+QÐQˆMô !ŸM™M¨$¯+©+×*EÑ*EÈdÐ*EÓ*SÓTÐð ,0ÐÔ(Ø)9Ð(:Ð=N×=`Ñ=`Ñ(`ÐÔ%ð (Ø'ñ#
Ðð
 '×*Ñ*Ð+<Ó=ˆØÐ&Ü˜oÔ/BÔCÜ Øwóð ô
 �?×-Ñ-Ó.°Ò2°×7QÑ7Q×7UÑ7UÐVWÔ7XØä! /×"GÑ"GÓHˆNØ¤Ò,Ø;GÐHYÑ;ZÐ[\Ñ;]×;cÑ;cÐdeÑ;fÐ'¨Ñ8á4BÑ4]ÐE\Ñ4]ˆOÕ1ô /BÜñ à"&§+¡+×"=Ñ"=ÀdÐ"=Ó"KØ&3ññô “ó/ˆLÐ*Ñ+ô �4˜Ô" t§{¡{Ð'>Ü˜dŸk™kÔ+>Ô?Ü Ð!oÓpÐpà&Ð'8Ñ9ˆD�Kð	 (?Ð"rA   r6   )NNNNNNNNNNNNNr   )(rU   rV   rW   r  Ú_tp_planÚ_pp_planr-   r8   r  r  rÊ  rÌ  r¤   rù  rý  r$   r#   r;   r¯   r!   r"   rQ   r   rÔ  ÚpropertyrÇ  r*  r+  r  r   r
   r1  rX   r   rO   r   r¹  r   r4  rY   rZ   s   @r@   r  r  ˜  s¦  ø„ à!Ð#QðÐð #Ð$;Ð<€HØ" oÐ%6¸¸
Ð$CÐD€Hð˜~õ ò/ò%ò1ò/ñ9 C¨$¡Jó 9ò(ð ØðMØ!ŸL™LðMØ4:Ð;MÑ4NðMà	Ð+Ñ	+òMó ó ðMð
 ñ/ó ð/ð Øð .2Ø15Ø37Ø04à59Ø:>Ø8<à26Ø6:Ø26Ø:>Ø*.Ø!%Ø-.ñ%M
ð ×#Ñ# dÑ*ðM
ð ×'Ñ'¨$Ñ.ð	M
ð
 ×)Ñ)¨DÑ0ðM
ð ×&Ñ&¨Ñ-ðM
ð !×+Ñ+¨dÑ2ðM
ð !&× 0Ñ 0°4Ñ 7ðM
ð $×.Ñ.°Ñ5ðM
ð )¨4Ñ/ðM
ð -¨tÑ3ðM
ð ×(Ñ(¨4Ñ/ðM
ð  %×0Ñ0°4Ñ7ðM
ð  × Ñ  4Ñ'ð!M
ð" ˜$‘;ð#M
ð$ ˜eŸl™lÑ*ð%M
ð& Ð+Ñ,ð'M
ð( 
ˆu× Ñ Ñ	! OÑ	3ò)M
ó ó ðM
ð^I:à+ðI:ð ðI:ð (ð	I:ð
 ðI:ð ðI:ð 
÷I:ñ I:rA   r  c                   ó¨  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  dedz  de	j                  dz  de	j                  dz  de	j                  dz  dee   defd„«       «       Zˆ xZS )Ú!T5Gemma2ForSequenceClassificationr]   c                 ó"  •— t         ‰| �  |«       |j                  | _        |j                  j                  | _        t        |«      | _        t        |dd«      }t        | j                  | j                  |«      | _	        | j                  «        y ©Nr<  gš™™™™™¹?©r7   r8   r;  r~  ra   ró  rk  r”   r:  Úscorer¬  ©r>   r]   Úclassifier_dropoutr?   s      €r@   r8   z*T5Gemma2ForSequenceClassification.__init__e  sp   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØ!Ÿ>™>×5Ñ5ˆÔä" 6Ó*ˆŒ
ä$ VÐ-FÈÓLÐÜ/°×0@Ñ0@À$Ç/Á/ÐSeÓfˆŒ
Ø�‰ÕrA   c                 ó6   — | j                   j                  «       S r6   r  rS   s    r@   rÊ  z6T5Gemma2ForSequenceClassification.get_input_embeddingsp  r  rA   c                 ó:   — | j                   j                  |«       y r6   r  r  s     r@   rÌ  z6T5Gemma2ForSequenceClassification.set_input_embeddingss  r  rA   Nre  rÏ  rË   r©   r   r  r  r  r¯  r  r|  rÙ   r�   c                 óv  — |	€|
�#t        d| j                  j                  › d�«      ‚|€t        d«      ‚|€| j	                  |«      } | j
                  |f||||||||	|
ddœ
|¤Ž}|j                  }|j                  }|j                  }| j                  |«      }|j                  d   }|| j                  j                  k7  j                  |j                  t        j                   «      }t        j"                  |j                  d   |j                  t        j                   ¬	«      }||z  j%                  d«      }t        j&                  ||j                  d   d
z
  ¬«      }|t        j"                  ||j                  ¬«      |f   }d}|�| j)                  |||| j                  ¬«      }t+        ||||¬«      S )áÜ  
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, decoder_sequence_length)`, *optional*):
            Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0,
            config.decoder.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nú8Passing input embeddings is currently not supported for ú.úYou have to specify input_idsF©
rÏ  rË   r©   r   r  r  r  r¯  r  r/  r   rC   r“   r,   )Úmaxr³  )r8  r|  Úpooled_logitsr]   ©r&  r8  rn   r¢  )ÚNotImplementedErrorr?   rU   r�  r‡  rk  r·  r  r  rF  rR   r]   r€  r˜   r‡   r;   Úint32r–   ÚargmaxÚclampr)  r   )r>   re  rÏ  rË   r©   r   r  r  r  r¯  r  r|  rÙ   râ  r·  rn   r¢  r8  rU  Únon_pad_maskÚtoken_indicesÚlast_non_pad_tokenrR  r&  s                           r@   rO   z)T5Gemma2ForSequenceClassification.forwardv  sÕ  € ð4 Ð$Ð(=Ð(IÜ%ØJÈ4Ï>É>×KbÑKbÐJcÐcdÐeóð ð ÐÜÐ<Ó=Ð=àÐ$Ø $× JÑ JÈ9Ó UÐà&0 d§j¡jØð'
à%Ø)Ø%Ø/Ø#9Ø!5Ø+Ø'Ø"7Øñ'
ð ñ'
ˆð $×5Ñ5ÐØ×5Ñ5ˆØ×/Ñ/ˆ
à—‘Ð-Ó.ˆà—_‘_ QÑ'ˆ
à)¨T¯[©[×-EÑ-EÑE×IÑIÈ&Ï-É-ÔY^×YdÑYdÓeˆÜŸ™Ð%6×%<Ñ%<¸RÑ%@ÈÏÉÔ^c×^iÑ^iÔjˆØ+¨lÑ:×BÑBÀ2ÓFÐÜ"Ÿ[™[Ð);ÐAR×AXÑAXÐY[ÑA\Ð_`ÑA`ÔaÐàœuŸ|™|¨J¸v¿}¹}ÔMÐOaÐaÑbˆàˆØÐØ×%Ñ%¨V¸FÐR_Ðhl×hsÑhsÐ%ÓtˆDä'ØØ Ø'Ø!ô	
ð 	
rA   ©NNNNNNNNNNN)rU   rV   rW   r-   r8   rÊ  rÌ  r$   r#   r;   r*  r+  r¯   r   r!   r"   r   rO   rY   rZ   s   @r@   rB  rB  c  s\  ø„ ð	˜~õ 	ò1ò/ð Øð .2Ø15Ø.2Ø04Ø59Ø6:Ø8<Ø26Ø26Ø:>Ø*.ñJ
à×#Ñ# dÑ*ðJ
ð ×'Ñ'¨$Ñ.ðJ
ð Ÿ™ tÑ+ð	J
ð
 ×&Ñ&¨Ñ-ðJ
ð !×+Ñ+¨dÑ2ðJ
ð !&§¡¨tÑ 3ðJ
ð $×.Ñ.°Ñ5ðJ
ð )¨4Ñ/ðJ
ð ×(Ñ(¨4Ñ/ðJ
ð  %×0Ñ0°4Ñ7ðJ
ð × Ñ  4Ñ'ðJ
ð Ð+Ñ,ðJ
ð 
"òJ
ó ó ôJ
rA   rB  c                   ó¨  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  dedz  de	j                  dz  de	j                  dz  de	j                  dz  dee   defd„«       «       Zˆ xZS )ÚT5Gemma2ForTokenClassificationr]   c                 ó"  •— t         ‰| �  |«       |j                  | _        |j                  j                  | _        t        |«      | _        t        |dd«      }t        | j                  | j                  |«      | _	        | j                  «        y rD  rE  rG  s      €r@   r8   z'T5Gemma2ForTokenClassification.__init__Ç  sp   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØ!Ÿ>™>×5Ñ5ˆÔä" 6Ó*ˆŒ
ä$ VÐ-FÈÓLÐÜ/°×0@Ñ0@À$Ç/Á/ÐSeÓfˆŒ
à�‰ÕrA   c                 ó6   — | j                   j                  «       S r6   r  rS   s    r@   rÊ  z3T5Gemma2ForTokenClassification.get_input_embeddingsÓ  r  rA   c                 ó:   — | j                   j                  |«       y r6   r  r  s     r@   rÌ  z3T5Gemma2ForTokenClassification.set_input_embeddingsÖ  r  rA   Nre  rÏ  rË   r©   r   r  r  r  r¯  r  r|  rÙ   r�   c                 ó–  — |	€|
�#t        d| j                  j                  › d�«      ‚|€t        d«      ‚|€| j	                  |«      } | j
                  |f||||||||	|
ddœ
|¤Ž}|j                  }|j                  }|j                  }| j                  |«      }d}|�| j                  ||| j                  «      }t        ||||¬«      S )rL  NrM  rN  rO  FrP  rS  )rT  r?   rU   r�  r‡  rk  r·  r  r  rF  r)  r]   r   )r>   re  rÏ  rË   r©   r   r  r  r  r¯  r  r|  rÙ   râ  r·  rn   r¢  r8  r&  s                      r@   rO   z&T5Gemma2ForTokenClassification.forwardÙ  s  € ð4 Ð$Ð(=Ð(IÜ%ØJÈ4Ï>É>×KbÑKbÐJcÐcdÐeóð ð ÐÜÐ<Ó=Ð=àÐ$Ø $× JÑ JÈ9Ó UÐà&0 d§j¡jØð'
à%Ø)Ø%Ø/Ø#9Ø!5Ø+Ø'Ø"7Øñ'
ð ñ'
ˆð $×5Ñ5ÐØ×5Ñ5ˆØ×/Ñ/ˆ
à—‘Ð-Ó.ˆàˆØÐØ×%Ñ% f¨f°d·k±kÓBˆDä$ØØØ'Ø!ô	
ð 	
rA   r[  )rU   rV   rW   r-   r8   rÊ  rÌ  r$   r#   r;   r*  r+  r¯   r   r!   r"   r   rO   rY   rZ   s   @r@   r]  r]  Å  s\  ø„ ð
˜~õ 
ò1ò/ð Øð .2Ø15Ø.2Ø04Ø59Ø6:Ø8<Ø26Ø26Ø:>Ø*.ñ@
à×#Ñ# dÑ*ð@
ð ×'Ñ'¨$Ñ.ð@
ð Ÿ™ tÑ+ð	@
ð
 ×&Ñ&¨Ñ-ð@
ð !×+Ñ+¨dÑ2ð@
ð !&§¡¨tÑ 3ð@
ð $×.Ñ.°Ñ5ð@
ð )¨4Ñ/ð@
ð ×(Ñ(¨4Ñ/ð@
ð  %×0Ñ0°4Ñ7ð@
ð × Ñ  4Ñ'ð@
ð Ð+Ñ,ð@
ð 
ò@
ó ó ô@
rA   r]  )r  ró  rÁ  rj  rB  r]  )r,   )rú   NN)T)br6  Úcollections.abcr   Útypingr   r;   Útorch.nnr9   Ú r   ru  Úactivationsr   Úcache_utilsr   r	   r
   r   Ú
generationr   r   r   Úintegrationsr   r   Úmasking_utilsr   r   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r    Úprocessing_utilsr!   Úutilsr"   r#   r$   r%   Úutils.genericr&   r'   Úutils.output_capturingr(   r)   Úautor+   Úconfiguration_t5gemma2r-   r.   r/   r0   ÚModuler2   r\   rp   r¶   r¿   r¯   rX   rÆ   rL   rQ   rÞ   rà   r  r  r-  r3  r:  rA  Ú	Embeddingr[  rj  rŸ  r¡  rÁ  rä  ró  r  rB  r]  Ú__all__r&  rA   r@   ú<module>ry     s  ðó* Ý $Ý ã Ý å &Ý !ß PÓ Pß KÑ Kß Iß mÑ mÝ BÝ 9÷÷ ñ ÷ Lß FÝ &ß aÓ aß Gß EÝ ß tÓ tô=�b—i‘iô =ô(�"—)‘)ô ô&L<˜bŸi™iô L<ò^(ñ Ð*Ó+òó ,ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð$ Ø Ø ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �S‰[ð%ð �T‰\ð%ð �T‰\ð%ð ˆ5�<‰<˜Ÿ™Ð%Ñ&ó%ñD Ð)Ó*ôK)˜BŸI™Ió K)ó +ðK)ñ\ Ð)Ó*ôuB˜bŸi™ió uBó +ðuBôp1Ð5ô 1ôh8Ð5ô 8ôv	�R—Y‘Yô 	ô §¡ô ô!@ "§)¡)ô !@ôH  b§l¡lô  ð. ôM!˜oó M!ó ðM!ñ`°ð Èó ô&b
Ð1ô b
ôJeÐ-ô eôP~
Ð-ô ~
ðB ôZ
Ð+ó Z
ó ðZ
ôzH:Ð'>Àô H:ðV ô^
Ð(?ó ^
ó ð^
ðB ôU
Ð%<ó U
ó ðU
òp�rA   