Ë
    ÛÓ^j¾£  ã                   ó¸  — d dl mZ d dlmZmZ d dlZd dlmZ d dlm	Z	 ddl
mZ ddlmZmZ ddlmZ dd	lmZmZmZmZmZmZmZ dd
lmZmZ ddlmZmZm Z  ddl!m"Z"m#Z# ddl$m%Z%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1 ddl2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z; ddl<m=Z=m>Z>m?Z?m@Z@ ddlAmBZB  e-j†                  eD«      ZE e+d¬«      e	 G d„ de1e«      «       «       ZF e+d¬«      e	 G d„ de«      «       «       ZG G d„ de@«      ZH G d„ de=«      ZI G d „ d!ej”                  «      ZK G d"„ d#e5«      ZL G d$„ d%e8«      ZM G d&„ d'e9ejœ                  «      ZO G d(„ d)e3«      ZP G d*„ d+e«      ZQdZR G d,„ d-e7«      ZSd.eTd/eeTeTeTeTgeUf   fd0„ZV G d1„ d2e6«      ZW G d3„ d4e4«      ZX G d5„ d6ejœ                  «      ZYdLd7ej´                  d8ej¶                  dz  d/ej´                  fd9„Z\d:ed;ej´                  d<ej´                  dz  d=edz  d>ej´                  dz  d?ej´                  d/e]fd@„Z^ G dA„ dBe?«      Z_ G dC„ dDe>«      Z` e+dE¬F«       G dG„ dHeeS«      «       Za G dI„ dJeeS«      Zbg dK¢Zcy)Mé    )ÚCallable)ÚAnyÚOptionalN)Ústricté   )Úinitialization)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Ú_preprocess_mask_argumentsÚblockwise_overlayÚcreate_causal_maskÚcreate_masks_for_generateÚ!create_sliding_window_causal_maskÚmaybe_pad_block_sequence_idsÚsliding_window_overlay)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚ SequenceClassifierOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmaybe_autocasté   )ÚGemma2Config)	ÚGemma2AttentionÚGemma2ForCausalLMÚ	Gemma2MLPÚGemma2ModelÚGemma2PreTrainedModelÚGemma2RMSNormÚGemma2RotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)ÚPaliGemmaCausalLMOutputWithPastÚ!PaliGemmaForConditionalGenerationÚPaliGemmaModelÚPaligemmaModelOutputWithPast)ÚSiglipVisionConfigzgoogle/gemma-3-4b-it)Ú
checkpointc            
       óÊ   — e Zd ZU dZdZddddddddddœ	Zddd	œZd
Zee	d<   dZ
ee	d<   dZee   dz  e	d<   dZedz  e	d<   dZedz  e	d<   dZedz  e	d<   dZedz  e	d<   d„ Zd„ Zy)ÚGemma3TextConfigaé  
    query_pre_attn_scalar (`float`, *optional*, defaults to 256):
        scaling factor used on the attention scores
    final_logit_softcapping (`float`, *optional*):
        Scaling factor when applying tanh softcapping on the logits.
    attn_logit_softcapping (`float`, *optional*):
        Scaling factor when applying tanh softcapping on the attention scores.
    use_bidirectional_attention (`bool`, *optional*, defaults to `False`):
        If True, the model will attend to all text tokens instead of using a causal mask. This does not change
        behavior for vision tokens.

    ```python
    >>> from transformers import Gemma3TextModel, Gemma3TextConfig
    >>> # Initializing a Gemma3Text gemma3_text-7b style configuration
    >>> configuration = Gemma3TextConfig()
    >>> # Initializing a model from the gemma3_text-7b style configuration
    >>> model = Gemma3TextModel(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    Úgemma3_textÚcolwiseÚreplicated_with_grad_allreduceÚrowwise)	zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.q_normzlayers.*.self_attn.k_normzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projg    €„.Ag     ˆÃ@)ÚglobalÚlocali@  Ú
vocab_sizei   Úmax_position_embeddingsNÚlayer_typesÚfinal_logit_softcappingÚattn_logit_softcappingÚrope_parametersFÚuse_bidirectional_attentionc                 óN  — | j                   r| j                  dz  dz   | _        |j                  dd«      | _        | j                  €Et        | j                  «      D �cg c]!  }t        |dz   | j                  z  «      rdnd‘Œ# c}| _        t        j                  di |¤Ž y c c}w )Nr"   é   Úsliding_window_patterné   Úsliding_attentionÚfull_attention© )
rA   Úsliding_windowÚgetÚ_sliding_window_patternr=   ÚrangeÚnum_hidden_layersÚboolr   Ú__post_init__)ÚselfÚkwargsÚis      út/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/gemma3/modular_gemma3.pyrO   zGemma3TextConfig.__post_init__s   s¢   € Ø×+Ò+Ø#'×#6Ñ#6¸!Ñ#;¸qÑ"@ˆDÔð (.§z¡zÐ2JÈAÓ'NˆÔ$à×ÑÐ#ô ˜t×5Ñ5Ó6ö àô (,¨Q°©U°d×6RÑ6RÑ,RÔ'SÑ#ÐYiÑiò ˆDÔô
 	×&Ñ&Ñ0¨Ó0ùò s   Á&B"c                 óz  — |j                  dd «      }ddiddidœ}| j                  �| j                  n|| _        |�| j                  d   j                  |«       | j                  j                  d«      €ddi| j                  d<   | j                  d   j	                  d|j                  d| j
                  d   «      «       | j                  j                  d«      €ddi| j                  d<   | j                  d   j	                  d|j                  d	| j
                  d
   «      «       | j                  «        |S )NÚrope_scalingÚ	rope_typeÚdefault)rF   rG   rG   Ú
rope_thetar9   rF   Úrope_local_base_freqr:   )Úpopr@   ÚupdaterJ   Ú
setdefaultÚdefault_thetaÚstandardize_rope_params)rP   rQ   rU   Údefault_rope_paramss       rS   Úconvert_rope_params_to_dictz,Gemma3TextConfig.convert_rope_params_to_dict‚   sI  € Ø—z‘z .°$Ó7ˆð
 #.¨yÐ!9Ø*¨IÐ6ñ
Ðð 8<×7KÑ7KÐ7W˜t×3Ò3Ð]pˆÔØÐ#Ø× Ñ Ð!1Ñ2×9Ñ9¸,ÔGð ×Ñ×#Ñ#Ð$4Ó5Ð=Ø6AÀ9Ð5MˆD× Ñ Ð!1Ñ2Ø×ÑÐ-Ñ.×9Ñ9Ø˜&Ÿ*™* \°4×3EÑ3EÀhÑ3OÓPô	
ð ×Ñ×#Ñ#Ð$7Ó8Ð@Ø9DÀiÐ8PˆD× Ñ Ð!4Ñ5Ø×ÑÐ0Ñ1×<Ñ<Ø˜&Ÿ*™*Ð%;¸T×=OÑ=OÐPWÑ=XÓYô	
ð
 	×$Ñ$Ô&Øˆó    )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚbase_model_tp_planr]   r;   ÚintÚ__annotations__r<   r=   ÚlistÚstrr>   Úfloatr?   r@   ÚdictrA   rN   rO   r`   rH   ra   rS   r4   r4   D   sª   … ñð, €Jà%.Ø%.Ø%.Ø%EØ%EØ%.Ø"+Ø )Ø"+ñ
Ðð  +°XÑ>€Mà€J�ÓØ#*Ð˜SÓ*Ø$(€K��c‘˜TÑ!Ó(Ø,0Ð˜U T™\Ó0Ø+/Ð˜E D™LÓ/Ø#'€O�T˜D‘[Ó'Ø/4Ð ¨¡Ó4ò1óra   r4   c                   óþ   ‡ — e Zd ZU dZdZddddœZeedœZdZ	ee
eef   z  dz  ed	<   dZee
eef   z  dz  ed
<   dZedz  ed<   dZedz  ed<   dZedz  ed<   dZedz  ed<   dZedz  ed<   dZedz  ed<   ˆ fd„Zˆ xZS )ÚGemma3Configa  
    mm_tokens_per_image (`int`, *optional*, defaults to 256):
        The number of tokens per image embedding.
    boi_token_index (`int`, *optional*, defaults to 255999):
        The begin-of-image token index to wrap the image prompt.
    eoi_token_index (`int`, *optional*, defaults to 256000):
        The end-of-image token index to wrap the image prompt.

    Example:

    ```python
    >>> from transformers import Gemma3ForConditionalGeneration, Gemma3Config, SiglipVisionConfig, Gemma3TextConfig

    >>> # Initializing a Siglip-like vision config
    >>> vision_config = SiglipVisionConfig()

    >>> # Initializing a Gemma3 Text config
    >>> text_config = Gemma3TextConfig()

    >>> # Initializing a Gemma3 gemma-3-4b style configuration
    >>> configuration = Gemma3Config(vision_config, text_config)

    >>> # Initializing a model from the gemma-3-4b style configuration
    >>> model = Gemma3TextConfig(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Úgemma3Úimage_token_indexÚboi_token_indexÚeoi_token_index)Úimage_token_idÚboi_token_idÚeoi_token_id)Útext_configÚvision_configNrw   rx   é   Úmm_tokens_per_imageiÿç i è i   g{®Gáz”?Úinitializer_rangeTÚtie_word_embeddingsc                 ó¸  •— | j                   €%t        «       | _         t        j                  d«       n4t	        | j                   t
        «      rt        di | j                   ¤Ž| _         t	        | j                  t
        «      rt        di | j                  ¤Ž| _        n0| j                  €$t        «       | _        t        j                  d«       t        ‰| �$  di |¤Ž y )Nz@text_config is None, using default Gemma3TextConfig text config.zFvision_config is None, using default SiglipVisionConfig vision config.rH   )
rw   r4   ÚloggerÚinfoÚ
isinstancerm   rx   r1   ÚsuperrO   )rP   rQ   Ú	__class__s     €rS   rO   zGemma3Config.__post_init__Ô   s§   ø€ Ø×ÑÐ#Ü/Ó1ˆDÔÜ�K‰KÐZÕ[Ü˜×(Ñ(¬$Ô/Ü/ÑC°$×2BÑ2BÑCˆDÔä�d×(Ñ(¬$Ô/Ü!3Ñ!I°d×6HÑ6HÑ!IˆDÕØ×ÑÐ'Ü!3Ó!5ˆDÔÜ�K‰KÐ`Ôaä‰ÑÑ' Ó'ra   )rb   rc   rd   re   rf   Úattribute_mapr4   r1   Úsub_configsrw   rm   rk   r   ri   rx   rz   rh   rr   rs   rq   r{   rl   r|   rN   rO   Ú__classcell__©r‚   s   @rS   ro   ro       sÍ   ø… ñð: €Jà-Ø)Ø)ñ€Mð (Ø+ñ€Kð
 =A€KÐ! D¨¨c¨¡NÑ2°TÑ9Ó@Ø@D€MÐ%¨¨S°#¨X©Ñ6¸Ñ=ÓDØ&)Ð˜˜t™Ó)Ø")€O�S˜4‘ZÓ)Ø")€O�S˜4‘ZÓ)Ø$+Ð�s˜T‘zÓ+Ø&*Ð�u˜t‘|Ó*Ø'+Ð˜ ™Ó+÷(ð (ra   ro   c                   ó   — e Zd Zy)ÚGemma3ModelOutputWithPastN©rb   rc   rd   rH   ra   rS   rˆ   rˆ   ä   ó   „ Øra   rˆ   c                   ó   — e Zd Zy)ÚGemma3CausalLMOutputWithPastNr‰   rH   ra   rS   rŒ   rŒ   è   rŠ   ra   rŒ   c            	       óZ   ‡ — e Zd ZdZd	dedededefˆ fd„Zdej                  fˆ fd„Z	ˆ xZ
S )
ÚGemma3TextScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scalec                 ó„   •— t         ‰| �  |||«       || _        | j                  dt	        j
                  |«      d¬«       y )Nr’   F©Ú
persistent)r�   Ú__init__Úscalar_embed_scaleÚregister_bufferÚtorchÚtensor)rP   r�   r�   r‘   r’   r‚   s        €rS   r–   z&Gemma3TextScaledWordEmbedding.__init__ñ   s;   ø€ Ü‰Ñ˜¨¸ÔDØ"-ˆÔØ×Ñ˜]¬E¯L©L¸Ó,EÐRWÐÕXra   Ú	input_idsc                 ó‚   •— t         ‰| �  |«      | j                  j                  | j                  j
                  «      z  S ©N)r�   Úforwardr’   ÚtoÚweightÚdtype)rP   r›   r‚   s     €rS   rž   z%Gemma3TextScaledWordEmbedding.forwardö   s2   ø€ Ü‰w‰˜yÓ)¨D×,<Ñ,<×,?Ñ,?ÀÇÁ×@QÑ@QÓ,RÑRÐRra   )ç      ð?)rb   rc   rd   re   rh   rl   r–   r™   ÚTensorrž   r…   r†   s   @rS   rŽ   rŽ   ì   sG   ø„ ññY sð Y¸3ð YÈSð YÐ_dõ Yð
S §¡÷ Sñ Sra   rŽ   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )Ú	Gemma3MLPÚconfigc                 ó$   •— t         ‰| �  |«       y r�   ©r�   r–   ©rP   r¦   r‚   s     €rS   r–   zGemma3MLP.__init__û   s   ø€ Ü‰Ñ˜Õ ra   )rb   rc   rd   r4   r–   r…   r†   s   @rS   r¥   r¥   ú   s   ø„ ð!Ð/÷ !ñ !ra   r¥   c                   ó*   ‡ — e Zd Zddedefˆ fd„Zˆ xZS )ÚGemma3RMSNormÚdimÚepsc                 ó(   •— t         ‰| �  ||¬«       y )N©r¬   r­   r¨   )rP   r¬   r­   r‚   s      €rS   r–   zGemma3RMSNorm.__init__   s   ø€ Ü‰Ñ˜S cÐÕ*ra   )g�íµ ÷Æ°>)rb   rc   rd   rh   rl   r–   r…   r†   s   @rS   r«   r«   ÿ   s   ø„ ñ+˜Cð + e÷ +ñ +ra   r«   c                   ó¤   — e Zd Zdefd„Ze	 	 	 	 ddedz  ded   dedz  dedz  de	d	e
f   f
d
„«       Z ej                  «       edd„«       «       Zy)ÚGemma3RotaryEmbeddingr¦   c                 ó”  — t         j                  j                  | «       |j                  | _        |j                  | _        || _        t        t        |j                  «      «      | _	        i | _
        | j                  D ]Ç  }| j                  j                  |   }|€Œ|d   | j                  |<   | j                  }| j                  |   dk7  rt        | j                  |      } || j                  |¬«      \  }}| j                  |› d�|d¬«       | j                  |› d�|j                  «       d¬«       t!        | |› d�|«       ŒÉ y )	NrV   rW   ©Ú
layer_typeÚ	_inv_freqFr”   Ú_original_inv_freqÚ_attention_scaling)ÚnnÚModuler–   r<   Úmax_seq_len_cachedÚoriginal_max_seq_lenr¦   rj   Úsetr=   rV   r@   Úcompute_default_rope_parametersr   r˜   ÚcloneÚsetattr)rP   r¦   r´   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalings          rS   r–   zGemma3RotaryEmbedding.__init__  s<  € Ü
�	‰	×Ñ˜4Ô Ø"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØˆŒÜ¤ F×$6Ñ$6Ó 7Ó8ˆÔØˆŒØ×*Ñ*ò 	UˆJØŸ+™+×5Ñ5°jÑAˆKØÐ"Øà)4°[Ñ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÑ)¨YÒ6Ü2°4·>±>À*Ñ3MÑN�Ù4@ÀÇÁÐYcÔ4dÑ1ˆMÐ1Ø× Ñ  J <¨yÐ!9¸=ÐUZÐ Ô[Ø× Ñ  J <Ð/AÐ!BÀM×DWÑDWÓDYÐfkÐ ÔlÜ�D˜Z˜LÐ(:Ð;Ð=SÕTñ	Ura   NÚdeviceztorch.deviceÚseq_lenr´   Úreturnztorch.Tensorc                 ó   — | j                   |   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a|  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        rX   Úhead_dimNr¢   r   r"   ©r¡   )rÄ   r¡   )	r@   ÚgetattrÚhidden_sizeÚnum_attention_headsr™   ÚarangeÚint64rŸ   rl   )r¦   rÄ   rÅ   r´   Úbaser¬   Úattention_factorÚinv_freqs           rS   r½   z5Gemma3RotaryEmbedding.compute_default_rope_parameters  s•   € ð2 ×%Ñ% jÑ1°,Ñ?ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)ra   c                 óN  — t        | |› d�«      }t        | |› d�«      }|d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬	«      5  |j                  «       |j                  «       z  j                  dd
«      }	t        j                  |	|	fd¬«      }
|
j                  «       |z  }|
j                  «       |z  }d d d «       j	                  |j                  ¬«      j	                  |j                  ¬«      fS # 1 sw Y   ŒAxY w)Nrµ   r·   r   éÿÿÿÿrC   ÚmpsÚcpuF)Údevice_typeÚenabledr"   ©r¬   rÉ   )rÊ   rl   ÚexpandÚshaperŸ   rÄ   r€   Útyperk   r!   Ú	transposer™   ÚcatÚcosÚsinr¡   )rP   ÚxÚposition_idsr´   rÑ   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedrÖ   ÚfreqsÚembrÞ   rß   s                rS   rž   zGemma3RotaryEmbedding.forward>  sl  € ô ˜4 J <¨yÐ!9Ó:ˆÜ# D¨Z¨LÐ8JÐ*KÓLÐà$ Tª1¨d ]Ñ3×9Ñ9Ó;×BÑBÀ<×CUÑCUÐVWÑCXÐZ\Ð^_Ó`×cÑcÐde×dlÑdlÓmÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	0Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)Ð/Ñ/ˆCØ—'‘'“)Ð/Ñ/ˆC÷		0ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	0ð 	0ús   Ã*A1FÆF$©NNNNr�   )rb   rc   rd   r4   r–   Ústaticmethodr   rh   rk   Útuplerl   r½   r™   Úno_gradr   rž   rH   ra   rS   r±   r±     s    „ ðUÐ/ó Uð* à*.Ø+/Ø"Ø!%ñ	!*Ø  4Ñ'ð!*à˜Ñ(ð!*ð �t‘ð!*ð ˜$‘Jð	!*ð
 
ˆ~˜uÐ$Ñ	%ò!*ó ð!*ðF €U‡]�]ƒ_Øò<ó ó ñ<ra   r±   c                   óø   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej                  dej                  dej                  dz  dedz  d	e	e
   d
eej                  ej                  dz  eej                     dz  f   fd„Zˆ xZS )ÚGemma3Attentionr¦   Ú	layer_idxc                 ób  •— t         ‰| �  ||«       | j                  dk(  r|j                  nd | _        | j                  dk(  | _        | j
                  j                   | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        y )NrF   r¯   )r�   r–   r´   rI   Ú
is_slidingr¦   rA   Ú	is_causalr«   rÈ   Úrms_norm_epsÚq_normÚk_norm©rP   r¦   rí   r‚   s      €rS   r–   zGemma3Attention.__init__S  s„   ø€ Ü‰Ñ˜ Ô+Ø7;·±ÐJ]Ò7]˜f×3Ò3ÐcgˆÔØŸ/™/Ð-@Ñ@ˆŒØ!Ÿ[™[×DÑDÐDˆŒä#¨¯©¸V×=PÑ=PÔQˆŒÜ#¨¯©¸V×=PÑ=PÔQˆ�ra   NÚhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valuesrQ   rÆ   c                 ód  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
| j                  |«      }| j                  |	«      }	|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t         «      } || ||	|
|f| j"                  r| j$                  nd| j&                  | j(                  dœ|¤Ž\  }} |j*                  g |¢d‘­Ž j-                  «       }| j/                  |«      }||fS )NrÓ   rC   r"   g        )ÚdropoutÚscalingrI   )rÚ   rÈ   Úq_projÚviewrÜ   Úk_projÚv_projrò   ró   r+   r[   rí   r   Úget_interfacer¦   Ú_attn_implementationr,   ÚtrainingÚattention_dropoutrû   rI   ÚreshapeÚ
contiguousÚo_proj)rP   rõ   rö   r÷   rø   rQ   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesrÞ   rß   Úattention_interfaceÚattn_outputÚattn_weightss                   rS   rž   zGemma3Attention.forward\  s¶  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà—{‘{ <Ó0ˆØ—[‘[ Ó,ˆ
à&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð /3¯mªm�D×*Ò*ÀØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(ra   )NNN)rb   rc   rd   r4   rh   r–   r™   r£   r	   r   r   ré   rž   r…   r†   s   @rS   rì   rì   R  s¦   ø„ ðRÐ/ð R¸Cõ Rð -1Ø.2Ø(,ñ*)à—|‘|ð*)ð #Ÿ\™\ð*)ð Ÿ™ tÑ+ð	*)ð
  ™ð*)ð Ð+Ñ,ð*)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷*)ra   rì   c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej                  dej                  dej                  dz  dej                  dz  d	e	dz  d
e
e   deej                  eej                  ej                  f   dz  f   fd„Zˆ xZS )ÚGemma3DecoderLayerr¦   rí   c                 óÔ  •— t         ‰| �  «        || _        |j                  | _        || _        t        ||¬«      | _        t        |«      | _        t        | j                  |j                  ¬«      | _        t        | j                  |j                  ¬«      | _        t        | j                  |j                  ¬«      | _        t        | j                  |j                  ¬«      | _        y )N)r¦   rí   ©r­   )r�   r–   r¦   rË   rí   rì   Ú	self_attnr¥   Úmlpr«   rñ   Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormrô   s      €rS   r–   zGemma3DecoderLayer.__init__Š  s°   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ"ˆŒÜ(°À)ÔLˆŒÜ˜VÓ$ˆŒÜ,¨T×-=Ñ-=À6×CVÑCVÔWˆÔÜ(5°d×6FÑ6FÈF×L_ÑL_Ô(`ˆÔ%Ü)6°t×7GÑ7GÈV×M`ÑM`Ô)aˆÔ&Ü*7¸×8HÑ8HÈf×NaÑNaÔ*bˆÕ'ra   Nrõ   rö   r÷   rá   rø   rQ   rÆ   c           	      ó  — |}| j                  |«      } | j                  d|||||dœ|¤Ž\  }}| j                  |«      }||z   }|}| j                  |«      }| j	                  |«      }| j                  |«      }||z   }|S )N)rõ   rö   r÷   rá   rø   rH   )r  r  r  r  r  r  )	rP   rõ   rö   r÷   rá   rø   rQ   ÚresidualÚ_s	            rS   rž   zGemma3DecoderLayer.forward–  s«   € ð !ˆà×,Ñ,¨]Ó;ˆà)˜4Ÿ>™>ð 
Ø'Ø 3Ø)Ø%Ø+ñ
ð ñ
Ñˆ�qð ×5Ñ5°mÓDˆØ  =Ñ0ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ  =Ñ0ˆàÐra   rç   )rb   rc   rd   r4   rh   r–   r™   r£   Ú
LongTensorr	   r   r   ré   ÚFloatTensorrž   r…   r†   s   @rS   r  r  ‰  sÃ   ø„ ð
cÐ/ð 
c¸Cõ 
cð -1Ø.2Ø04Ø(,ñà—|‘|ðð #Ÿ\™\ðð Ÿ™ tÑ+ð	ð
 ×&Ñ&¨Ñ-ðð  ™ðð Ð+Ñ,ðð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷ra   r  c                   óJ   — e Zd ZdZdZg d¢Z ej                  «       d„ «       Zy)ÚGemma3PreTrainedModelÚmodel)ÚimageÚtext)r  ÚSiglipVisionEmbeddingsÚSiglipEncoderLayerÚ#SiglipMultiheadAttentionPoolingHeadc                 óÜ  — t        j                  | |«       t        |t        «      r t	        j
                  |j                  «       y d|j                  j                  v r t	        j
                  |j                  «       y t        |t        «      r+t	        j                  |j                  |j                  «       y t        |t        «      r£|j                  D ]“  }|j                   }|j"                  |   dk7  rt$        |j"                  |      } ||j&                  |¬«      \  }}t	        j(                  t+        ||› d�«      |«       t	        j(                  t+        ||› d�«      |«       Œ• y y )NÚRMSNormrW   r³   rµ   r¶   )r   Ú_init_weightsr€   ÚGemma3MultiModalProjectorÚinitÚzeros_Úmm_input_projection_weightr‚   rb   r    rŽ   Ú	constant_r’   r—   r±   r=   r½   rV   r   r¦   Úcopy_rÊ   )rP   Úmoduler´   rÁ   rÂ   r  s         rS   r(  z#Gemma3PreTrainedModel._init_weightsÄ  s   € ä×%Ñ% d¨FÔ3Ü�fÔ7Ô8Ü�K‰K˜×9Ñ9Õ:à˜&×*Ñ*×3Ñ3Ñ3Ü�K‰K˜Ÿ™Õ&Ü˜Ô =Ô>Ü�N‰N˜6×-Ñ-¨v×/HÑ/HÕIÜ˜Ô 5Ô6Ø$×0Ñ0ò ^�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ò<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ô#UÑ �˜qÜ—
‘
œ7 6¨j¨\¸Ð+CÓDÀmÔTÜ—
‘
œ7 6¨j¨\Ð9KÐ+LÓMÈ}Õ]ñ^ð 7ra   N)	rb   rc   rd   Úbase_model_prefixÚinput_modalitiesÚ_no_split_modulesr™   rê   r(  rH   ra   rS   r  r  º  s4   „ ØÐØ(ÐòÐð €U‡]�]ƒ_ñ^ó ñ^ra   r  rI   rÆ   c           
      óP   ‡ — dt         dt         dt         dt         dt        f
ˆ fd„}|S )zA
    Enables a bidirectional mask within the sliding window.
    Ú	batch_idxÚhead_idxÚq_idxÚkv_idxrÆ   c                 ó&   •— t        ||z
  «      ‰k  S )z“A token can attend to any other token if their absolute distance is within
        the (exclusive) sliding window size (distance < sliding_window).)Úabs)r4  r5  r6  r7  rI   s       €rS   Ú
inner_maskz1_bidirectional_window_overlay.<locals>.inner_maskÝ  s   ø€ ô �5˜6‘>Ó" ^Ñ3Ð3ra   )rh   rN   )rI   r:  s   ` rS   Ú_bidirectional_window_overlayr;  Ø  s3   ø€ ð
4œcð 4¬Sð 4¼ð 4Äcð 4Ìdõ 4ð
 Ðra   c                   óä   ‡ — e Zd ZU eed<   dZdefˆ fd„Z	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de
dz  d	ej                  dz  d
edz  dee   defd„Zˆ xZS )ÚGemma3TextModelr¦   ©r"  c                 ó¶   •— t         ‰| �  |«       t        |j                  |j                  | j
                  | j                  j                  dz  ¬«      | _        y )Nç      à?)r’   )r�   r–   rŽ   r;   rË   r‘   r¦   Úembed_tokensr©   s     €rS   r–   zGemma3TextModel.__init__é  sM   ø€ Ü‰Ñ˜Ô ô :Ø×Ñ˜v×1Ñ1°4×3CÑ3CÐQU×Q\ÑQ\×QhÑQhÐjmÑQmô
ˆÕra   Nr›   r÷   rá   rø   Úinputs_embedsÚ	use_cacherQ   rÆ   c           	      óê  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      sw| j                  ||||dœ}
|
j                  «       }| j                  j                  r(d„ |
d<   t        | j                  j                  «      |d<   t!        di |
¤Žt#        di |¤Žd	œ}	|}i }t%        | j                  j&                  «      D ]  }| j)                  |||«      ||<   Œ t+        | j,                  d | j                  j.                   «      D ]G  \  }} ||f|	| j                  j&                  |      || j                  j&                  |      ||d
œ|¤Ž}ŒI | j1                  |«      }t3        ||¬«      S )Nú:You must specify exactly one of input_ids or inputs_embeds)r¦   r   rC   ©rÄ   ©r¦   rB  r÷   rø   rá   c                  óL   — t        j                  dt         j                  ¬«      S )NTrÉ   )r™   rš   rN   )Úargss    rS   ú<lambda>z)Gemma3TextModel.forward.<locals>.<lambda>  s   € ÄÇÁÈTÔY^×YcÑYcÔ@d€ ra   Úor_mask_function©rG   rF   )r÷   rö   rá   rø   )Úlast_hidden_staterø   rH   )Ú
ValueErrorrA  r
   r¦   Úget_seq_lengthr™   rÍ   rÚ   rÄ   Ú	unsqueezer€   rm   ÚcopyrA   r;  rI   r   r   r¼   r=   Ú
rotary_embÚ	enumerateÚlayersrM   Únormr   )rP   r›   r÷   rá   rø   rB  rC  rQ   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚsliding_mask_kwargsrõ   rö   r´   rR   Údecoder_layers                    rS   rž   zGemma3TextModel.forwardñ  s#  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-ÄÔFð Ÿ+™+Ø!.Ø"0Ø#2Ø ,ñˆKð #.×"2Ñ"2Ó"4Ðà�{‰{×6Ò6Ù2d�Ð.Ñ/Ü:WÐX\×XcÑXc×XrÑXrÓ:sÐ#Ð$6Ñ7ô #5Ñ"C°{Ñ"CÜ%FÑ%]ÐI\Ñ%]ñ#Ðð &ˆØ ÐÜ˜dŸk™k×5Ñ5Ó6ò 	gˆJØ.2¯o©o¸mÈ\Ð[eÓ.fÐ 
Ò+ð	gô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÓ Wò 	ÑˆAˆ}Ù)Øðà2°4·;±;×3JÑ3JÈ1Ñ3MÑNØ$7¸¿¹×8OÑ8OÐPQÑ8RÑ$SØ)Ø /ñð ñ‰Mð	ð Ÿ	™	 -Ó0ˆä&Ø+Ø+ô
ð 	
ra   )NNNNNN)rb   rc   rd   r4   ri   r1  r–   r™   r  r£   r	   r  rN   r   r   r   rž   r…   r†   s   @rS   r=  r=  å  sÄ   ø… ØÓØ Ðð
Ð/õ 
ð .2Ø.2Ø04Ø(,Ø26Ø!%ñC
à×#Ñ# dÑ*ðC
ð Ÿ™ tÑ+ðC
ð ×&Ñ&¨Ñ-ð	C
ð
  ™ðC
ð ×(Ñ(¨4Ñ/ðC
ð ˜$‘;ðC
ð Ð+Ñ,ðC
ð 
!÷C
ra   r=  c                   ó0   ‡ — e Zd ZU eed<   defˆ fd„Zˆ xZS )ÚGemma3ForCausalLMr¦   c                 óD   •— t         ‰| �  |«       t        |«      | _        y r�   )r�   r–   r=  r   r©   s     €rS   r–   zGemma3ForCausalLM.__init__:  s   ø€ Ü‰Ñ˜Ô Ü$ VÓ,ˆ�
ra   )rb   rc   rd   r4   ri   r–   r…   r†   s   @rS   r\  r\  7  s   ø… ØÓð-Ð/÷ -ñ -ra   r\  c                   óD   ‡ — e Zd Zdefˆ fd„Zdej                  fd„Zˆ xZS )r)  r¦   c                 óŽ  •— t         ‰| �  «        t        j                  t	        j
                  |j                  j                  |j                  j                  «      «      | _	        t        |j                  j                  |j                  j                  ¬«      | _        t        |j                  j                  |j                  j                  z  «      | _        t        |j"                  dz  «      | _        | j                   | j$                  z  | _        t        j(                  | j&                  | j&                  ¬«      | _        y )Nr  r@  )Úkernel_sizeÚstride)r�   r–   r¸   Ú	Parameterr™   Úzerosrx   rË   rw   r,  r«   Úlayer_norm_epsÚmm_soft_emb_normrh   Ú
image_sizeÚ
patch_sizeÚpatches_per_imagerz   Útokens_per_sider`  Ú	AvgPool2dÚavg_poolr©   s     €rS   r–   z"Gemma3MultiModalProjector.__init__@  sì   ø€ Ü‰ÑÔä*,¯,©,Ü�K‰K˜×,Ñ,×8Ñ8¸&×:LÑ:L×:XÑ:XÓYó+
ˆÔ'ô !.Ø× Ñ ×,Ñ,°&×2FÑ2F×2UÑ2Uô!
ˆÔô "% V×%9Ñ%9×%DÑ%DÈ×H\ÑH\×HgÑHgÑ%gÓ!hˆÔÜ" 6×#=Ñ#=¸sÑ#BÓCˆÔØ×1Ñ1°T×5IÑ5IÑIˆÔÜŸ™°×1AÑ1AÈ$×JZÑJZÔ[ˆ�ra   Úvision_outputsc                 ó¢  — |j                   \  }}}|j                  dd«      }|j                  ||| j                  | j                  «      }|j	                  «       }| j                  |«      }|j                  d«      }|j                  dd«      }| j                  |«      }t        j                  || j                  «      }|j                  |«      S )NrC   r"   )rÚ   rÜ   r  rh  r  rk  Úflattenre  r™   Úmatmulr,  Útype_as)	rP   rl  Ú
batch_sizer  rË   Úreshaped_vision_outputsÚpooled_vision_outputsÚnormed_vision_outputsÚprojected_vision_outputss	            rS   rž   z!Gemma3MultiModalProjector.forwardP  sÍ   € Ø%3×%9Ñ%9Ñ"ˆ
�A�{à"0×":Ñ":¸1¸aÓ"@ÐØ"9×"AÑ"AØ˜ T×%;Ñ%;¸T×=SÑ=Só#
Ðð #:×"DÑ"DÓ"FÐà $§¡Ð.EÓ FÐØ 5× =Ñ =¸aÓ @ÐØ 5× ?Ñ ?ÀÀ1Ó EÐà $× 5Ñ 5Ð6KÓ LÐä#(§<¡<Ð0EÀt×GfÑGfÓ#gÐ Ø'×/Ñ/°Ó?Ð?ra   )	rb   rc   rd   ro   r–   r™   r£   rž   r…   r†   s   @rS   r)  r)  ?  s#   ø„ ð\˜|õ \ð @ e§l¡l÷ @ra   r)  Útoken_type_idsrÄ   c                 ó  — | dk(  j                  |¬«      }t        j                  j                  |dd¬«      d d …d d…f   }|| z  }t	        j
                  |j                  «       d¬«      dz
  }t	        j                  ||d«      }|S )NrC   rF  )rC   r   r   )ÚvaluerÓ   rØ   )rŸ   r¸   Ú
functionalÚpadr™   Úcumsumrh   Úwhere)rv  rÄ   Úis_imageÚis_previous_imageÚnew_image_startÚ	group_idsÚblock_sequence_idss          rS   Úget_block_sequence_ids_for_maskr‚  c  sŠ   € ð  !Ñ#×'Ñ'¨vÐ'Ó6€HÜŸ™×)Ñ)¨(°FÀ!Ð)ÓDÂQÈÈÈÀVÑLÐØÐ"3Ð!3Ñ3€OÜ—‘˜_×0Ñ0Ó2¸Ô:¸QÑ>€IÜŸ™ X¨y¸"Ó=ÐØÐra   r¦   rB  r÷   rø   rá   r�  c                 óæ   — | ||||dœ}t        di |¤d|i¤Ž}t        di |¤ddi¤Ž\  }}	}	}	}
}	}|r|}nt        |||
|«      }t        di |¤t        |«      t	        | j
                  «      dœ¤Ž}||dœS )zãCreate full_attention and sliding_attention masks with correct composition.

    For global (full attention) layers:  OR(causal, blockwise)
    For local (sliding window) layers:  AND(sliding_window, OR(causal, blockwise))
    rG  r�  rí   r   )rK  Úand_mask_functionrL  rH   )r   r   r   r   r   rI   )r¦   rB  r÷   rø   rá   r�  rX  Ú	full_maskÚ
early_exitr  Ú	kv_lengthÚ	kv_offsetÚpadded_block_sequence_idsÚsliding_masks                 rS   Úcreate_masks_for_vision_modelr‹  n  s¼   € ð Ø&Ø(Ø*Ø$ñ€Kô #ÑX [ÑXÐEWÒX€Iô 4Nñ 4Ø
ñ4àò4Ñ0€J��1�a˜ A yñ Ø$6Ñ!ä$@Ø °	¸9ó%
Ð!ô &ñ Ø
ðä*Ð+DÓEÜ0°×1FÑ1FÓGó€Lð $Ø)ñð ra   c                   ó¨  ‡ — e Zd ZdZdefˆ fd„Ze ed¬«      dej                  de
e   deez  fd	„«       «       Zee	 	 	 	 	 	 	 	 	 ddej                  d
z  dej                  d
z  dej                   d
z  dej                  d
z  ded
z  dej                  d
z  dej                  d
z  dej                  d
z  ded
z  de
e   deez  fd„«       «       Zˆ xZS )ÚGemma3ModelFr¦   c                 ó(   •— t         ‰| �  |«       | `y r�   )r�   r–   Útext_config_dtyper©   s     €rS   r–   zGemma3Model.__init__¦  s   ø€ Ü‰Ñ˜Ô ØÑ"ra   zOProjects the last hidden state from the vision model into language model space.©Úcustom_introÚpixel_valuesrQ   rÆ   c                 ót   —  | j                   d|ddœ|¤Ž}|j                  }| j                  |«      |_        |S )NT)r’  Úreturn_dictrH   )Úvision_towerrM  Úmulti_modal_projectorÚpooler_output)rP   r’  rQ   rl  rM  s        rS   Úget_image_featureszGemma3Model.get_image_featuresª  sH   € ð
 +˜×*Ñ*Ða¸ÐRVÑaÐZ`ÑaˆØ*×<Ñ<ÐØ'+×'AÑ'AÐBSÓ'TˆÔ$àÐra   Nr›   r÷   rá   rø   rv  rB  ÚlabelsrC  Ú	lm_kwargsc
           
      óX  — |d u |d uz  rt        d«      ‚|�R| j                  j                  | j                  k\  r/|| j                  j                  k(  }|j	                  «       }d||<   n|}|€ | j                  «       |«      }|�i| j                  |d¬«      j                  }|j                  |j                  |j                  «      }| j                  |||¬«      }|j                  ||«      }t        |x}t        «      sR| j                  j                  «       ||||dœ}|�%t!        ||j                  ¬«      }t#        dd|i|¤Ž}nt%        di |¤Ž} | j&                  d|||||	dd	œ|
¤Ž}t)        |j*                  |j,                  |j.                  |j0                  |�¬
«      S d ¬
«      S )NrE  r   T)r”  )rB  Úimage_featuresrG  rF  r�  )r÷   rá   rø   rB  rC  r”  )rM  rø   rõ   Ú
attentionsÚimage_hidden_statesrH   )rN  r¦   rt   r;   r¾   Úget_input_embeddingsr˜  r—  rŸ   rÄ   r¡   Úget_placeholder_maskÚmasked_scatterr€   rm   Úget_text_configr‚  r‹  r   Úlanguage_modelrˆ   rM  rø   rõ   r�  )rP   r›   r’  r÷   rá   rø   rv  rB  r™  rC  rš  Úspecial_image_maskÚllm_input_idsrœ  rW  rX  r�  Úoutputss                     rS   rž   zGemma3Model.forwardµ  sú  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZð Ð  T§[¡[×%?Ñ%?À4Ç?Á?Ò%RØ!*¨d¯k©k×.HÑ.HÑ!HÐØ%ŸO™OÓ-ˆMØ01ˆMÐ,Ò-à%ˆMàÐ Ø7˜D×5Ñ5Ó7¸ÓFˆMð Ð#Ø!×4Ñ4°\ÈtÐ4ÓT×bÑbˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNØ!%×!:Ñ!:Ø¨À~ð ";ó "Ðð *×8Ñ8Ð9KÈ^Ó\ˆMô °Ð?Ð-ÄÔFàŸ+™+×5Ñ5Ó7Ø!.Ø"0Ø#2Ø ,ñˆKð Ð)Ü%DÀ^Ð\i×\pÑ\pÔ%qÐ"Ü&Cñ 'Ø'9ð'à!ñ'Ñ#ô
 '@Ñ&NÀ+Ñ&NÐ#à%�$×%Ñ%ð 
Ø.Ø%Ø+Ø'ØØñ
ð ñ
ˆô )Ø%×7Ñ7Ø#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø2>Ð2J ô
ð 	
ð
 QUô
ð 	
ra   )	NNNNNNNNN)rb   rc   rd   Úaccepts_loss_kwargsro   r–   r   r   r™   r  r   r   ré   r   r˜  r  r£   r	   rN   rˆ   rž   r…   r†   s   @rS   r�  r�  ¢  ss  ø„ àÐð#˜|õ #ð ÙÐ!rÔsðØ!×-Ñ-ðØ9?Ð@RÑ9Sðà	Ð+Ñ	+òó tó ðð Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.Ø!%ñG
à×#Ñ# dÑ*ðG
ð ×'Ñ'¨$Ñ.ðG
ð Ÿ™ tÑ+ð	G
ð
 ×&Ñ&¨Ñ-ðG
ð  ™ðG
ð ×(Ñ(¨4Ñ/ðG
ð ×(Ñ(¨4Ñ/ðG
ð × Ñ  4Ñ'ðG
ð ˜$‘;ðG
ð Ð.Ñ/ðG
ð 
Ð*Ñ	*òG
ó ó ôG
ra   r�  c                   ó  ‡ — e Zd ZdZee	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  de
dz  dej                  dz  d	ej                  dz  d
ej                  dz  dedz  deej                  z  dee   deez  fd„«       «       Z	 	 	 	 	 	 	 	 	 	 dˆ fd„	Z	 	 dded	ej                  dej                  dz  de
dz  dej                  dz  dej                  dz  dedz  defd„Zˆ xZS )ÚGemma3ForConditionalGenerationFNr›   r’  r÷   rá   rø   rv  rB  r™  rC  Úlogits_to_keeprš  rÆ   c                 óÔ  —  | j                   d	||||||||	|ddœ
|¤Ž}|d   }t        |
t        «      rt        |
 d«      n|
}| j	                  |dd…|dd…f   «      }d}|��O|j                  «       }|ddd…dd…f   }|ddd…f   }|�‘|dd…|j                  d    d…f   j                  |j                  «      }||j                  |j                  «      dk7     j                  «       }||j                  |j                  «      dk7     j                  «       }n |j                  «       }|j                  «       }t        j                  «       }|j                  d| j                  j                  j                  «      }|j                  d«      j                  |j                  «      } |||«      }t!        |||j"                  |j$                  |j&                  |j(                  ¬«      S )
a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

        >>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma-3-4b-it")
        >>> processor = AutoProcessor.from_pretrained("google/gemma-3-4b-it")

        >>> messages = [
        ...     {
        ...         "role": "system",
        ...         "content": [
        ...             {"type": "text", "text": "You are a helpful assistant."}
        ...         ]
        ...     },
        ...     {
        ...         "role": "user", "content": [
        ...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
        ...             {"type": "text", "text": "Where is the cat standing?"},
        ...         ]
        ...     },
        ... ]

        >>> inputs = processor.apply_chat_template(
        ...     messages,
        ...     tokenize=True,
        ...     return_dict=True,
        ...     return_tensors="pt",
        ...     add_generation_prompt=True
        ... )
        >>> # Generate
        >>> generate_ids = model.generate(**inputs)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
        ```
        T)
r›   r’  rv  r÷   rá   rø   rB  rC  r™  r”  r   N.rÓ   rC   )ÚlossÚlogitsrø   rõ   r�  rž  rH   )r   r€   rh   ÚsliceÚlm_headrl   rÚ   rŸ   rÄ   r  r¸   ÚCrossEntropyLossrý   r¦   rw   r;   rŒ   rø   rõ   r�  rž  )rP   r›   r’  r÷   rá   rø   rv  rB  r™  rC  rª  rš  r¦  rõ   Úslice_indicesr­  r¬  Úshift_logitsÚshift_labelsÚshift_attention_maskÚloss_fctÚflat_logitsÚflat_labelss                          rS   rž   z&Gemma3ForConditionalGeneration.forward  sú  € ðz �$—*‘*ð 
ØØ%Ø)Ø)Ø%Ø+Ø'ØØØñ
ð ñ
ˆð   ™
ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÑà—\‘\“^ˆFØ! # s¨ sªA +Ñ.ˆLØ! # q¡r '™?ˆLØÐ)ð (6²a¸,×:LÑ:LÈQÑ:OÐ9OÑ9QÐ6QÑ'R×'UÑ'UÐV\×VcÑVcÓ'dÐ$Ø+Ð,@×,CÑ,CÀFÇMÁMÓ,RÐVWÑ,WÑX×cÑcÓe�Ø+Ð,@×,CÑ,CÀL×DWÑDWÓ,XÐ\]Ñ,]Ñ^×iÑiÓk‘à+×6Ñ6Ó8�Ø+×6Ñ6Ó8�ä×*Ñ*Ó,ˆHà&×+Ñ+¨B°·±×0GÑ0G×0RÑ0RÓSˆKØ&×+Ñ+¨BÓ/×2Ñ2°<×3FÑ3FÓGˆKÙ˜K¨Ó5ˆDä+ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;ô
ð 	
ra   c                 óZ   •— t        ‰| �  |f||||||	||dœ|¤Ž}|s|s||d<   |S d |d<   |S )N)rø   rB  r÷   rá   rC  rª  rv  Úis_first_iterationr’  rv  )r�   Úprepare_inputs_for_generation)rP   r›   rø   rB  rá   r’  r÷   rv  rC  rª  r™  r¹  rQ   Úmodel_inputsr‚   s                 €rS   rº  z<Gemma3ForConditionalGeneration.prepare_inputs_for_generationu  sj   ø€ ô  ‘wÑ<Øð
à+Ø'Ø)Ø%ØØ)Ø)Ø1ñ
ð ñ
ˆñ" ¡YØ+7ˆL˜Ñ(ð
 Ðð .2ˆLÐ)Ñ*àÐra   r¦   r¹  c                 ó�   — | j                  «       ||||dœ}|�$t        ||j                  ¬«      }	t        dd|	i|¤ŽS t	        di |¤ŽS )NrG  rF  r�  rH   )r¢  r‚  rÄ   r‹  r   )
r¦   rB  r÷   rø   rá   rv  r¹  rQ   rX  r�  s
             rS   r   z8Gemma3ForConditionalGeneration.create_masks_for_generatež  sl   € ð ×,Ñ,Ó.Ø*Ø,Ø.Ø(ñ
ˆð Ð%Ü!@ÀÐXe×XlÑXlÔ!mÐÜ0ñ Ø#5ðàñð ô
 )Ñ7¨;Ñ7Ð7ra   )
NNNNNNNNNr   )
NNNNNNTNNF)NF)rb   rc   rd   r§  r   r   r™   r  r  r£   r	   rN   rh   r   r   ré   rŒ   rž   rº  r   rm   r   r…   r†   s   @rS   r©  r©    sØ  ø„ ð  ÐàØð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.Ø!%Ø-.ñk
à×#Ñ# dÑ*ðk
ð ×'Ñ'¨$Ñ.ðk
ð Ÿ™ tÑ+ð	k
ð
 ×&Ñ&¨Ñ-ðk
ð  ™ðk
ð ×(Ñ(¨4Ñ/ðk
ð ×(Ñ(¨4Ñ/ðk
ð × Ñ  4Ñ'ðk
ð ˜$‘;ðk
ð ˜eŸl™lÑ*ðk
ð Ð.Ñ/ðk
ð 
Ð-Ñ	-òk
ó ó ðk
ð` ØØØØØØØØØ õ'ð^ /3Ø*/ñ8Ø ð8à—|‘|ð8ð Ÿ™ tÑ+ð8ð  ™ð	8ð
 —l‘l TÑ)ð8ð Ÿ™ tÑ+ð8ð ! 4™Kð8ð 
÷8ra   r©  zÎ
Gemma3TextForSequenceClassification is a text-only sequence classification model that works with Gemma3TextConfig.
It uses the generic sequence classification implementation for efficiency and consistency.r�  c                   ó   — e Zd ZU eed<   dZy)Ú#Gemma3TextForSequenceClassificationr¦   r>  N)rb   rc   rd   r4   ri   r1  rH   ra   rS   r¾  r¾  º  s   … ð ÓØ Ñra   r¾  c                   ó  ‡ — e Zd Z	 	 	 	 	 	 	 	 ddej                  dz  dej
                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  dej
                  dz  d	ej                  dz  d
ee	   de
fˆ fd„Zˆ xZS )ÚGemma3ForSequenceClassificationNr›   r’  r÷   rá   rø   rv  rB  r™  rQ   rÆ   c	                 ó4   •— t        ‰
| �  d||||||||dœ|	¤ŽS )N)r›   r÷   rá   rø   rB  r’  rv  r™  rH   )r�   rž   )rP   r›   r’  r÷   rá   rø   rv  rB  r™  rQ   r‚   s             €rS   rž   z'Gemma3ForSequenceClassification.forwardÅ  s=   ø€ ô ‰w‰ð 

ØØ)Ø%Ø+Ø'Ø%Ø)Øñ

ð ñ

ð 
	
ra   )NNNNNNNN)rb   rc   rd   r™   r  r  r£   r	   r   r   r   rž   r…   r†   s   @rS   rÀ  rÀ  Ä  sÞ   ø„ ð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.ñ
à×#Ñ# dÑ*ð
ð ×'Ñ'¨$Ñ.ð
ð Ÿ™ tÑ+ð	
ð
 ×&Ñ&¨Ñ-ð
ð  ™ð
ð ×(Ñ(¨4Ñ/ð
ð ×(Ñ(¨4Ñ/ð
ð × Ñ  4Ñ'ð
ð Ð+Ñ,ð
ð 
*÷
ñ 
ra   rÀ  )	ro   r4   r  r=  r\  r©  r�  rÀ  r¾  r�   )dÚcollections.abcr   Útypingr   r   r™   Útorch.nnr¸   Úhuggingface_hub.dataclassesr   Ú r   r*  Úcache_utilsr	   r
   Úconfiguration_utilsr   Úmasking_utilsr   r   r   r   r   r   r   Úmodeling_layersr   r   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r    Úutils.genericr!   Úgemma2.configuration_gemma2r#   Úgemma2.modeling_gemma2r$   r%   r&   r'   r(   r)   r*   r+   r,   Úpaligemma.modeling_paligemmar-   r.   r/   r0   Úsiglipr1   Ú
get_loggerrb   r~   r4   ro   rˆ   rŒ   Ú	EmbeddingrŽ   r¥   r«   r¹   r±   rì   r  ÚGEMMA3_START_DOCSTRINGr  rh   rN   r;  r=  r\  r)  r£   rÄ   r‚  rm   r‹  r�  r©  r¾  rÀ  Ú__all__rH   ra   rS   ú<module>rÙ     s¹  ðõ %ß  ã Ý Ý .å &ß .Ý 3÷÷ ñ ÷ \ß uÑ u÷÷ GÝ &ß RÓ RÝ +Ý 6÷
÷ 
õ 
÷ó õ (ð 
ˆ×	Ñ	˜HÓ	%€ñ Ð1Ô2ØôW�|Ð%5ó Wó ó 3ðWñt Ð1Ô2Øô?(Ð#ó ?(ó ó 3ð?(ôD	Ð <ô 	ô	Ð#Bô 	ôS B§L¡Lô Sô!�	ô !ô
+�Mô +ô
J<Ð1°2·9±9ô J<ô\4)�oô 4)ôn+Ð3ô +ð\ Ð ô^Ð1ô ^ð<
°#ð 
¸(ÀCÈÈcÐSVÐCWÐY]ÐC]Ñ:^ó 
ôO
�kô O
ôd-Ð)ô -ô!@ §	¡	ô !@ñH°E·L±Lð È%Ï,É,ÐY]ÑJ]ð Ðin×iuÑiuó ð1Øð1à—<‘<ð1ð —L‘L 4Ñ'ð1ð ˜T‘\ð	1ð
 —,‘, Ñ%ð1ð Ÿ™ð1ð 
ó1ôh\
�.ô \
ô~v8Ð%Fô v8ñr ð^ôô
!Ð*JÐLaó !óð
!ô

Ð&FÐH]ô 
ò4
�ra   