Ë
    ÛÓ^j« ã                   ó~  — d dl Z d dlmZ d dlmZ d dlmZ d dlmZ d dl	Z	d dl	m
Z
 d dlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZmZmZmZmZm Z  ddl!m"Z" ddl#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4 ddl5m6Z6m7Z7m8Z8 ddl9m:Z:m;Z; ddl<m=Z= ddl>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZE ddlFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZO ddlPmQZQ ddlRmSZS ddlTmUZU ddlVmWZWmXZXmYZYmZZZ  e2«       r	  e3j¶                  e\«      Z]ded e	j¼                  d!e	j¼                  dz  d"edz  d#e	j¼                  dz  d$e	j¼                  d%e_fd&„Z` G d'„ d(eJ«      Za G d)„ d*eG«      Zbe G d+„ d,e$«      «       Zce0e G d-„ d.e%«      «       «       Zd G d/„ d0e
jÊ                  «      Zf G d1„ d2eM«      Zg G d3„ d4e
jÊ                  «      Zh G d5„ d6e
jÊ                  «      Zi G d7„ d8e
jÊ                  «      Zj G d9„ d:e
jÊ                  «      Zk G d;„ d<e
jÊ                  «      Zl G d=„ d>e
jÚ                  «      Zn G d?„ d@e
jÊ                  «      Zo G dA„ dBe
jÊ                  «      Zp G dC„ dDe
jÊ                  «      Zq G dE„ dFe
jÊ                  «      Zr G dG„ dHeB«      Zs	 d€dIe	j¼                  dJe	j¼                  dKe	j¼                  d#e	j¼                  dLetd%e	j¼                  fdM„Zu G dN„ dOeQ«      Zve8 G dP„ dQe?«      «       Zw G dR„ dSe@«      Zx G dT„ dUe
jÊ                  «      Zy G dV„ dWeB«      Zz G dX„ dYeC«      Z{ G dZ„ d[e
jÊ                  «      Z| G d\„ d]eS«      Z} G d^„ d_e
jÊ                  «      Z~ G d`„ dae@«      Z G db„ dceE«      Z€ G dd„ deeL«      Z� e0df¬g«       G dh„ dieD«      «       Z‚ e0dj¬g«       G dk„ dleA«      «       Zƒ G dm„ dne�«      Z„ G do„ dpe�«      Z… G dq„ dreK«      Z†dse	j¼                  dz  dte	j¼                  dz  d%edz  fdu„Z‡dve	j¼                  dwe	�j                  d%e	j¼                  fdx„Z‰ e0dy¬g«       G dz„ d{eI«      «       ZŠ e0d|¬g«       G d}„ d~eH«      «       Z‹g d¢ZŒy)�é    N)ÚUserDict)ÚCallable)Ú	dataclass)Úcached_property)Únn)Ú
functionalé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Ú_preprocess_mask_argumentsÚblockwise_overlayÚcreate_bidirectional_maskÚcreate_causal_maskÚcreate_masks_for_generateÚ!create_sliding_window_causal_maskÚmaybe_pad_block_sequence_idsÚsliding_window_overlay)ÚFlashAttentionKwargs)ÚBaseModelOutputWithPastÚBaseModelOutputWithPooling)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_accelerate_availableÚloggingÚtorch_compilable_check)Úmaybe_autocastÚmerge_with_config_defaultsÚno_inherit_decorator)ÚOutputRecorderÚcapture_outputsé   )Ú	AutoModel)ÚGemma3AttentionÚGemma3DecoderLayerÚGemma3ForCausalLMÚ	Gemma3MLPÚGemma3RotaryEmbeddingÚGemma3TextModelÚGemma3TextScaledWordEmbedding)	ÚGemma3nCausalLMOutputWithPastÚGemma3nForConditionalGenerationÚGemma3nModelÚGemma3nModelOutputWithPastÚGemma3nMultimodalEmbedderÚGemma3nPreTrainedModelÚGemma3nRMSNormÚapply_rotary_pos_embÚeager_attention_forward)ÚLlamaRotaryEmbedding)ÚMixtralExperts)Úsliding_window_mask_functioné   )ÚGemma4AudioConfigÚGemma4ConfigÚGemma4TextConfigÚGemma4VisionConfigÚconfigÚinputs_embedsÚattention_maskÚpast_key_valuesÚposition_idsÚblock_sequence_idsÚreturnc                 óÞ   — | ||||dœ}t        di |¤Ž}t        di |¤ddi¤Ž\  }}	}	}	}
}	}|r|}nt        |||
|«      }t        di |¤t        |«      t	        | j
                  «      dœ¤Ž}||dœS )a‹  Create full_attention and sliding_attention masks with correct composition.

    For global (full attention) layers:  causal only (no bidirectional)
    For local (sliding window) layers:  AND(sliding_window, OR(causal, blockwise))

    Unlike Gemma 3 (which applies bidirectional attention on all layers), Gemma 4
    explicitly disables bidirectional attention on global attention layers.
    ©rD   rE   rF   rG   rH   Ú	layer_idxr   )Úor_mask_functionÚand_mask_function©Úfull_attentionÚsliding_attention© )r   r   r   r   r   Úsliding_window)rD   rE   rF   rG   rH   rI   Úmask_kwargsÚ	full_maskÚ
early_exitÚ_Ú	kv_lengthÚ	kv_offsetÚpadded_block_sequence_idsÚsliding_masks                 út/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/gemma4/modular_gemma4.pyÚcreate_masks_for_vision_modelr^   X   s¶   € ð" Ø&Ø(Ø*Ø$ñ€Kô #Ñ1 [Ñ1€Iô 4Nñ 4Ø
ñ4àò4Ñ0€J��1�a˜ A yñ Ø$6Ñ!ä$@Ø °	¸9ó%
Ð!ô &ñ Ø
ðä*Ð+DÓEÜ0°×1FÑ1FÓGó€Lð $Ø)ñð ó    c                   ób   — e Zd ZU dZdZeeeej                  ej                  f   f   dz  e
d<   y)ÚGemma4ModelOutputWithPasta¯  
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    audio_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
    shared_kv_states (`dict`, *optional*):
        Dictionary mapping layer type strings to tuples of (key_states, value_states) tensors.
        Used to pass shared KV states between layers during KV sharing.
    NÚshared_kv_states©Ú__name__Ú
__module__Ú__qualname__Ú__doc__rb   ÚdictÚstrÚtupleÚtorchÚTensorÚ__annotations__rS   r_   r]   ra   ra   �   s7   … ñð" MQÐ�d˜3  e§l¡l°E·L±LÐ&@Ñ AÐAÑBÀTÑIÔPr_   ra   c                   ób   — e Zd ZU dZdZeeeej                  ej                  f   f   dz  e
d<   y)ÚGemma4CausalLMOutputWithPasta  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.text_config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
    audio_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
    shared_kv_states (`dict`, *optional*):
        Dictionary mapping layer type strings to tuples of (key_states, value_states) tensors.
        Used to pass shared KV states between layers during KV sharing.
    Nrb   rc   rS   r_   r]   ro   ro   ¤   s7   … ñð* MQÐ�d˜3  e§l¡l°E·L±LÐ&@Ñ AÐAÑBÀTÑIÔPr_   ro   c                   ób   — e Zd ZU dZdZeeeej                  ej                  f   f   dz  e
d<   y)ÚGemma4TextModelOutputWithPasta9  
    BaseModelOutputWithPast extended with shared_kv_states for KV sharing.

    Args:
        shared_kv_states (`dict`, *optional*):
            Dictionary mapping layer type strings to tuples of (key_states, value_states) tensors.
            Used to pass shared KV states between layers during KV sharing.
    Nrb   rc   rS   r_   r]   rq   rq   ½   s7   … ñð MQÐ�d˜3  e§l¡l°E·L±LÐ&@Ñ AÐAÑBÀTÑIÔPr_   rq   c                   ó:   — e Zd ZU dZdZej                  dz  ed<   y)ÚGemma4AudioModelOutputz§
    attention_mask (`torch.BoolTensor`, *optional*):
        A torch.BoolTensor of shape `(batch_size, num_frames)`. True for valid positions, False for padding.
    NrF   )rd   re   rf   rg   rF   rk   Ú
BoolTensorrm   rS   r_   r]   rs   rs   Ë   s   … ñð
 /3€N�E×$Ñ$ tÑ+Ô2r_   rs   c                   ón   ‡ — e Zd Zdeez  dededdfˆ fd„Zdej                  dej                  fd„Z	ˆ xZ
S )	ÚGemma4ClippableLinearrD   Úin_featuresÚout_featuresrJ   Nc                 ó  •— t         ‰| �  «        |j                  | _        t        j                  ||d¬«      | _        | j                  r»| j                  dt        j                  t        d«       «      «       | j                  dt        j                  t        d«      «      «       | j                  dt        j                  t        d«       «      «       | j                  dt        j                  t        d«      «      «       y y )NF©ÚbiasÚ	input_minÚinfÚ	input_maxÚ
output_minÚ
output_max)
ÚsuperÚ__init__Úuse_clipped_linearsr   ÚLinearÚlinearÚregister_bufferrk   ÚtensorÚfloat)ÚselfrD   rw   rx   Ú	__class__s       €r]   r‚   zGemma4ClippableLinear.__init__×   s¸   ø€ ô 	‰ÑÔØ#)×#=Ñ#=ˆÔ Ü—i‘i ¨\ÀÔFˆŒà×#Ò#Ø× Ñ  ¬e¯l©l¼EÀ%»L¸=Ó.IÔJØ× Ñ  ¬e¯l©l¼5À»<Ó.HÔIØ× Ñ  ¬u¯|©|¼UÀ5»\¸MÓ/JÔKØ× Ñ  ¬u¯|©|¼EÀ%»LÓ/IÕJð	 $r_   Úhidden_statesc                 ó  — | j                   r+t        j                  || j                  | j                  «      }| j                  |«      }| j                   r+t        j                  || j                  | j                  «      }|S ©N)rƒ   rk   Úclampr|   r~   r…   r   r€   )r‰   r‹   s     r]   ÚforwardzGemma4ClippableLinear.forwardç   s\   € Ø×#Ò#Ü!ŸK™K¨°t·~±~ÀtÇ~Á~ÓVˆMàŸ™ MÓ2ˆà×#Ò#Ü!ŸK™K¨°t·±ÈÏÉÓXˆMàÐr_   )rd   re   rf   rC   r@   Úintr‚   rk   rl   r�   Ú__classcell__©rŠ   s   @r]   rv   rv   Ö   sT   ø„ ðKà"Ð%6Ñ6ðKð ðKð ð	Kð
 
õKð 	 U§\¡\ð 	°e·l±l÷ 	r_   rv   c                   ó   — e Zd Zy)ÚGemma4RMSNormN©rd   re   rf   rS   r_   r]   r”   r”   ó   ó   „ Ør_   r”   c                   ó¨   ‡ — e Zd ZU dZej
                  ed<   defˆ fd„Z ej                  «       dej
                  dej
                  fd„«       Z
ˆ xZS )Ú Gemma4AudioRelPositionalEncodingzíSinusoidal relative positional encoding for the audio encoder.

    Produces position embeddings of shape [1, context_size // 2 + 1, hidden_size] with
    concatenated [sin..., cos...] layout matching the original Gemma4 convention.
    Úinv_timescalesrD   c                 óØ  •— t         ‰| �  «        |j                  | _        |j                  |j                  z   dz
  |j
                  z   | _        d}d}| j                  dz  }t        j                  ||z  «      t        |dz
  d«      z  }|t        j                  t        j                  |«      | z  «      z  }| j                  d|j                  d«      j                  d«      d¬«       y )	Nr?   ç      ð?ç     ˆÃ@r*   r™   r   F©Ú
persistent)r�   r‚   Úhidden_sizeÚattention_chunk_sizeÚattention_context_leftÚattention_context_rightÚcontext_sizeÚmathÚlogÚmaxrk   ÚexpÚaranger†   Ú	unsqueeze)r‰   rD   Úmin_timescaleÚmax_timescaleÚnum_timescalesÚlog_timescale_incrementr™   rŠ   s          €r]   r‚   z)Gemma4AudioRelPositionalEncoding.__init__   sß   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔà×'Ñ'¨&×*GÑ*GÑGÈ!ÑKÈf×NlÑNlÑlð 	Ôð ˆØˆØ×)Ñ)¨QÑ.ˆÜ"&§(¡(¨=¸=Ñ+HÓ"IÌCÐP^ÐabÑPbÐdeÓLfÑ"fÐØ&¬¯©´5·<±<ÀÓ3OÐSjÐRjÑ3jÓ)kÑkˆØ×ÑÐ-¨~×/GÑ/GÈÓ/J×/TÑ/TÐUVÓ/WÐdiÐÕjr_   r‹   rJ   c                 ót  — t        j                  | j                  dz  dd|j                  ¬«      }|d   }|| j                  j                  |j                  ¬«      z  }t        j                  t        j                  |«      t        j                  |«      gd¬«      }|j                  |j                  ¬«      S )Nr*   éÿÿÿÿ©Údevice©.N©Údim©Údtype)
rk   r¨   r£   r±   r™   ÚtoÚcatÚsinÚcosr¶   )r‰   r‹   rH   Úscaled_timeÚ	pos_embeds        r]   r�   z(Gemma4AudioRelPositionalEncoding.forward  s”   € ä—|‘| D×$5Ñ$5¸Ñ$:¸BÀÈ=×K_ÑK_Ô`ˆØ# IÑ.ˆØ" T×%8Ñ%8×%;Ñ%;À=×CWÑCWÐ%;Ó%XÑXˆÜ—I‘IœuŸy™y¨Ó5´u·y±yÀÓ7MÐNÐTVÔWˆ	Ø�|‰| -×"5Ñ"5ˆ|Ó6Ð6r_   )rd   re   rf   rg   rk   rl   rm   r@   r‚   Úno_gradr�   r‘   r’   s   @r]   r˜   r˜   ÷   sU   ø… ñð —L‘LÓ ðkÐ0õ kð €U‡]�]ƒ_ð7 U§\¡\ð 7°e·l±lò 7ó ô7r_   r˜   c                   óP  ‡ — e Zd ZdZdedefˆ fd„Zdej                  dej                  fd„Z	dej                  dej                  fd„Z
d	ej                  dej                  fd
„Z	 ddej                  dej                  dej                  dz  deej                  df   fd„Zˆ xZS )ÚGemma4AudioAttentionz3Chunked local attention with relative position biasrD   rM   c                 ó   •— t         ‰| �  «        || _        || _        |j                  | _        |j                  |j                  z  | _        |j                  | _	        | j                  dz  t        j                  d«      z  | _        t        j                  dt        j                  z   «      t        j                  d«      z  | _        |j                  | _        |j"                  dz
  | _        |j&                  | _        | j                   | j$                  z   | j(                  z   | _        t-        ||j                  | j                  | j                  z  «      | _        t-        ||j                  | j                  | j                  z  «      | _        t-        ||j                  | j                  | j                  z  «      | _        t-        ||j                  |j                  «      | _        t7        j8                  |j                  | j                  | j                  z  d¬«      | _        t7        j<                  t?        j@                  | j                  «      «      | _!        | jE                  dt?        jF                  | j
                  «      d¬«       y )Nç      à¿r*   r?   Frz   Úsoftcapr�   )$r�   r‚   rD   rM   Úattention_logit_capÚattention_logits_soft_caprŸ   Únum_attention_headsÚhead_dimÚ	num_headsr¤   r¥   Úq_scaleÚeÚk_scaler    Ú
chunk_sizer¡   Úmax_past_horizonr¢   Úmax_future_horizonr£   rv   Úq_projÚk_projÚv_projÚpostr   r„   Úrelative_k_projÚ	Parameterrk   ÚzerosÚper_dim_scaler†   r‡   ©r‰   rD   rM   rŠ   s      €r]   r‚   zGemma4AudioAttention.__init__  sÛ  ø€ Ü‰ÑÔØˆŒØ"ˆŒØ)/×)CÑ)CˆÔ&Ø×*Ñ*¨f×.HÑ.HÑHˆŒØ×3Ñ3ˆŒàŸ™ tÑ+¬t¯x©x¸«{Ñ:ˆŒÜ—x‘x ¤D§F¡F¡
Ó+¬d¯h©h°q«kÑ9ˆŒà ×5Ñ5ˆŒØ &× =Ñ =ÀÑ AˆÔØ"(×"@Ñ"@ˆÔØ ŸO™O¨d×.CÑ.CÑCÀd×F]ÑF]Ñ]ˆÔä+¨F°F×4FÑ4FÈÏÉÐY]×YfÑYfÑHfÓgˆŒÜ+¨F°F×4FÑ4FÈÏÉÐY]×YfÑYfÑHfÓgˆŒÜ+¨F°F×4FÑ4FÈÏÉÐY]×YfÑYfÑHfÓgˆŒÜ)¨&°&×2DÑ2DÀf×FXÑFXÓYˆŒ	ä!Ÿy™y¨×);Ñ);¸T¿^¹^ÈdÏmÉmÑ=[ÐbgÔhˆÔÜŸ\™\¬%¯+©+°d·m±mÓ*DÓEˆÔà×Ñ˜Y¬¯©°T×5SÑ5SÓ(TÐafÐÕgr_   r‹   rJ   c           	      ó  — |j                   \  }}}}|| j                  z   dz
  | j                  z  }|| j                  z  |z
  }t        j                  |ddddd|f«      }|j	                  ||| j                  ||«      j                  «       S )z€Splits a `(batch_size, seq_len, num_heads, head_dim)` tensor into non-overlapping blocks of `chunk_size` along the sequence dim.r?   r   )ÚshaperË   ÚFÚpadÚreshapeÚ
contiguous)r‰   r‹   Ú
batch_sizeÚseq_lenrÇ   rÆ   Ú
num_blocksrÚ   s           r]   Ú_convert_to_blockz&Gemma4AudioAttention._convert_to_block3  s‰   € à3@×3FÑ3FÑ0ˆ
�G˜Y¨Ø §¡Ñ/°!Ñ3¸¿¹ÑGˆ
Ø˜4Ÿ?™?Ñ*¨WÑ4ˆÜŸ™˜m¨a°°A°q¸!¸SÐ-AÓBˆØ×$Ñ$ Z°¸T¿_¹_ÈiÐYaÓb×mÑmÓoÐor_   c           
      ó@  — |j                   \  }}}}t        j                  |dddd| j                  | j                  | j
                  z   dz
  f«      }|j                  d| j                  | j
                  «      }t        j                  |dd«      }|j                  «       S )z`Extracts overlapping context windows of `context_size` for every block, strided by `chunk_size`.r   r?   r¯   r*   )rØ   rÙ   rÚ   rÌ   rÍ   rË   Úunfoldr£   rk   ÚmovedimrÜ   )r‰   r‹   rÝ   rÞ   rÇ   rÆ   s         r]   Ú_extract_block_contextz+Gemma4AudioAttention._extract_block_context;  s•   € à3@×3FÑ3FÑ0ˆ
�G˜Y¨ÜŸ™Ø˜A˜q ! Q¨×(=Ñ(=¸t×?VÑ?VÐY]×YhÑYhÑ?hÐklÑ?lÐmó
ˆð &×,Ñ,¨Q°×0AÑ0AÀ4Ç?Á?ÓSˆÜŸ™ m°R¸Ó;ˆØ×'Ñ'Ó)Ð)r_   Úxc                 óð   — |j                   \  }}}}}| j                  }t        j                  |d|dz   |z
  f«      }|j	                  |||||dz   z  «      }|dd||z  …f   }|j	                  |||||«      S )zjRelative position shift for blocked attention. See appendix B of https://huggingface.co/papers/1901.02860.r   r?   .N)rØ   r£   rÙ   rÚ   Úview)r‰   rå   rÝ   rÇ   rß   Ú
block_sizeÚposition_lengthr£   s           r]   Ú
_rel_shiftzGemma4AudioAttention._rel_shiftE  s�   € àIJÏÉÑFˆ
�I˜z¨:°Ø×(Ñ(ˆÜ�E‰E�!�a˜¨Ñ)¨OÑ;Ð<Ó=ˆØ�F‰F�:˜y¨*°jÀLÐSTÑDTÑ6UÓVˆØˆcÐ.�Z ,Ñ.Ð.Ð.Ñ/ˆØ�v‰v�j )¨Z¸À\ÓRÐRr_   NÚposition_embeddingsrF   c                 óÚ  — |j                   \  }}}||| j                  | j                  f}| j                  |«      j	                  «       j                  |«      }| j                  |«      j	                  «       j                  |«      }	| j                  |«      j	                  «       j                  |«      }
|| j                  z  t        j                  | j                  «      z  }|	| j                  z  }	| j                  |«      }| j                  |	«      }	| j                  |
«      }
|j                   d   }| j                  |«      }|j                  d| j                  | j                  «      }|j!                  |j"                  ¬«      }|j%                  ddddd«      }||	j%                  ddddd«      z  }|j'                  || j                  d| j                  «      }||j%                  ddd«      z  }|j'                  || j                  || j(                  d«      }| j+                  |«      }||z   }|| j,                  z  }t/        j0                  |«      }|| j,                  z  }|�4|j3                  |j5                  «       | j6                  j8                  «      }t        j:                  |dt.        j<                  ¬«      j!                  |
j"                  «      }||
j%                  ddddd«      z  }|j%                  ddddd«      j'                  ||| j(                  z  d«      }|d d …d |…f   j?                  «       }| jA                  |j!                  |j"                  «      «      }||fS )	Nr?   r¯   rµ   r   r	   r*   é   ©r´   r¶   )!rØ   rÇ   rÆ   rÎ   rˆ   rç   rÏ   rÐ   rÈ   rÙ   ÚsoftplusrÕ   rÊ   rà   rä   rÒ   r·   r¶   ÚpermuterÛ   rË   rê   rÂ   rk   ÚtanhÚmasked_fillÚlogical_notrD   Úattention_invalid_logits_valueÚsoftmaxÚfloat32rÜ   rÑ   )r‰   r‹   rë   rF   rÝ   Ú
seq_lengthrX   Úhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesrß   Úrelative_key_statesÚqueriesÚ	matrix_acÚqueries_flatÚ	matrix_bdÚattn_weightsÚattn_outputs                      r]   r�   zGemma4AudioAttention.forwardN  s  € ð %2×$7Ñ$7Ñ!ˆ
�J Ø" J°·±ÀÇÁÐNˆà—{‘{ =Ó1×7Ñ7Ó9×>Ñ>¸|ÓLˆØ—[‘[ Ó/×5Ñ5Ó7×<Ñ<¸\ÓJˆ
Ø—{‘{ =Ó1×7Ñ7Ó9×>Ñ>¸|ÓLˆà# d§l¡lÑ2´Q·Z±ZÀ×@RÑ@RÓ5SÑSˆØ $§,¡,Ñ.ˆ
à×-Ñ-¨lÓ;ˆØ×0Ñ0°Ó<ˆ
Ø×2Ñ2°<Ó@ˆØ!×'Ñ'¨Ñ*ˆ
à"×2Ñ2Ð3FÓGÐØ1×6Ñ6°r¸4¿>¹>È4Ï=É=ÓYÐØ1×4Ñ4¸<×;MÑ;MÐ4ÓNÐà×&Ñ& q¨!¨Q°°1Ó5ˆØ˜j×0Ñ0°°A°q¸!¸QÓ?Ñ?ˆ	à—‘ z°4·>±>À2ÀtÇ}Á}ÓUˆØ Ð#6×#>Ñ#>¸qÀ!ÀQÓ#GÑGˆ	Ø×%Ñ% j°$·.±.À*ÈdÏoÉoÐ_aÓbˆ	Ø—O‘O IÓ.ˆ	à  9Ñ,ˆØ# d§l¡lÑ2ˆÜ—z‘z ,Ó/ˆØ# d§l¡lÑ2ˆàÐ%Ø'×3Ñ3Ø×*Ñ*Ó,¨d¯k©k×.XÑ.XóˆLô —y‘y °2¼U¿]¹]ÔK×NÑNÈ|×OaÑOaÓbˆØ" \×%9Ñ%9¸!¸QÀÀ1ÀaÓ%HÑHˆØ!×)Ñ)¨!¨Q°°1°aÓ8×@Ñ@ÀÈZÐZ^×ZiÑZiÑMiÐkmÓnˆØ!¢! [ j [ .Ñ1×<Ñ<Ó>ˆØ—i‘i §¡¨}×/BÑ/BÓ CÓDˆà˜LÐ(Ð(r_   r�   )rd   re   rf   rg   r@   r�   r‚   rk   rl   rà   rä   rê   rt   rj   r�   r‘   r’   s   @r]   r¿   r¿     sÅ   ø„ Ù=ðhÐ0ð h¸Sõ hð4p¨u¯|©|ð pÀÇÁó pð*°E·L±Lð *ÀUÇ\Á\ó *ðS˜EŸL™Lð S¨U¯\©\ó Sð 37ñ	1)à—|‘|ð1)ð #Ÿ\™\ð1)ð ×(Ñ(¨4Ñ/ð	1)ð
 
ˆu�|‰|˜TÐ!Ñ	"÷1)r_   r¿   c                   ó^   ‡ — e Zd Zˆ fd„Zddej
                  dej
                  dz  fd„Zˆ xZS )Ú'Gemma4AudioSubSampleConvProjectionLayerc                 óÐ   •— t         ‰| �  «        t        j                  ||dddd¬«      | _        t        j
                  ||dd¬«      | _        t        j                  «       | _        y )N)r	   r	   )r*   r*   r?   F)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingr{   T)ÚepsÚelementwise_affiner{   )	r�   r‚   r   ÚConv2dÚconvÚ	LayerNormÚnormÚReLUÚact)r‰   r  r  Únorm_epsrŠ   s       €r]   r‚   z0Gemma4AudioSubSampleConvProjectionLayer.__init__ƒ  sW   ø€ Ü‰ÑÔÜ—I‘IØ#Ø%ØØØØô
ˆŒ	ô —L‘L °8ÐPTÐ[`ÔaˆŒ	Ü—7‘7“9ˆ�r_   Nr‹   Úmaskc           
      ó¢  — |�,|j                  |j                  ¬«      }||d d …d d d …d f   z  }| j                  |j                  | j                  j                  j                  «      «      }| j                  | j                  |j                  dddd«      «      j                  dddd«      j                  «       «      }|�|d d …d d d…f   }||fS )Nr°   r   r*   r	   r?   )	r·   r±   r  Úweightr¶   r  r  rð   rÜ   )r‰   r‹   r  s      r]   r�   z/Gemma4AudioSubSampleConvProjectionLayer.forward�  sÇ   € ØÐØ—7‘7 -×"6Ñ"6�7Ó7ˆDØ)¨D²°Dº!¸TÐ1AÑ,BÑBˆMàŸ	™	 -×"2Ñ"2°4·9±9×3CÑ3C×3IÑ3IÓ"JÓKˆØŸ™ §¡¨=×+@Ñ+@ÀÀAÀqÈ!Ó+LÓ!M×!UÑ!UÐVWÐYZÐ\]Ð_`Ó!a×!lÑ!lÓ!nÓoˆàÐØš™3˜Q˜3˜‘<ˆDà˜dÐ"Ð"r_   r�   )rd   re   rf   r‚   rk   rl   r�   r‘   r’   s   @r]   r  r  ‚  s(   ø„ ôñ# U§\¡\ð #¸¿¹ÈÑ9L÷ #r_   r  c            	       óœ   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dej                  dz  deej                  ej                  f   fd„Zˆ xZ	S )	Ú"Gemma4AudioSubSampleConvProjectionrD   c                 óv  •— t         ‰| �  «        t        d|j                  d   |j                  ¬«      | _        t        |j                  d   |j                  d   |j                  ¬«      | _        |j                  d   dz  |j                  d   z  }t        j                  ||j                  d¬«      | _
        y )Nr?   r   )r  r  r  rí   Frz   )r�   r‚   r  Úsubsampling_conv_channelsÚrms_norm_epsÚlayer0Úlayer1r   r„   rŸ   Úinput_proj_linear)r‰   rD   Úproj_input_dimrŠ   s      €r]   r‚   z+Gemma4AudioSubSampleConvProjection.__init__Ÿ  s«   ø€ Ü‰ÑÔÜ=ØØ×9Ñ9¸!Ñ<Ø×(Ñ(ô
ˆŒô
 >Ø×8Ñ8¸Ñ;Ø×9Ñ9¸!Ñ<Ø×(Ñ(ô
ˆŒð
 !×:Ñ:¸1Ñ=ÀÑBÀf×FfÑFfÐghÑFiÑiˆÜ!#§¡¨>¸6×;MÑ;MÐTYÔ!ZˆÕr_   NÚinput_featuresÚinput_features_maskrJ   c                 ó&  — |j                  d«      }| j                  ||«      \  }}| j                  ||«      \  }}|j                  \  }}}}|j	                  dddd«      j                  «       j                  ||d«      }| j                  |«      |fS )Nr?   r   r*   r	   r¯   )r©   r  r  rØ   rð   rÜ   rÛ   r  )r‰   r   r!  r‹   r  rÝ   rX   rÞ   s           r]   r�   z*Gemma4AudioSubSampleConvProjection.forward®  s™   € ð
 '×0Ñ0°Ó3ˆØ"Ÿk™k¨-Ð9LÓMÑˆ�tØ"Ÿk™k¨-¸Ó>Ñˆ�tà$1×$7Ñ$7Ñ!ˆ
�A�w Ø%×-Ñ-¨a°°A°qÓ9×DÑDÓF×NÑNÈzÐ[bÐdfÓgˆØ×%Ñ% mÓ4°dÐ:Ð:r_   r�   )
rd   re   rf   r@   r‚   rk   rl   rj   r�   r‘   r’   s   @r]   r  r  ž  sW   ø„ ð[Ð0õ [ð$ 48ñ;àŸ™ð;ð #Ÿ\™\¨DÑ0ð;ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷	;r_   r  c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚGemma4AudioFeedForwardrD   c                 ó°  •— t         ‰| �  «        || _        t        ||j                  |j                  dz  «      | _        t        ||j                  dz  |j                  «      | _        t        |j                  «      | _        t        |j                  «      | _	        t        |j                     | _        |j                  | _        |j                  | _        y )Nrí   )r�   r‚   rD   rv   rŸ   Úffw_layer_1Úffw_layer_2r”   Úpre_layer_normÚpost_layer_normr   Ú
hidden_actÚact_fnÚgradient_clippingÚresidual_weightÚpost_layer_scale©r‰   rD   rŠ   s     €r]   r‚   zGemma4AudioFeedForward.__init__½  s¯   ø€ Ü‰ÑÔØˆŒä0°¸×9KÑ9KÈV×M_ÑM_ÐbcÑMcÓdˆÔÜ0°¸×9KÑ9KÈaÑ9OÐQW×QcÑQcÓdˆÔä+¨F×,>Ñ,>Ó?ˆÔÜ,¨V×-?Ñ-?Ó@ˆÔÜ˜V×.Ñ.Ñ/ˆŒà!'×!9Ñ!9ˆÔØ &× 6Ñ 6ˆÕr_   r‹   rJ   c                 ó¶  — t        | j                  t        j                  |j                  «      j
                  «      }|}t        j                  || |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }t        j                  || |«      }| j                  |«      }|| j                  z  }||z  }|S r�   )Úminr,  rk   Úfinfor¶   r¦   rŽ   r(  r&  r+  r'  r)  r.  )r‰   r‹   r,  Úresiduals       r]   r�   zGemma4AudioFeedForward.forwardË  sË   € ä × 6Ñ 6¼¿¹ÀM×DWÑDWÓ8X×8\Ñ8\Ó]Ðà ˆÜŸ™ MÐ4EÐ3EÐGXÓYˆØ×+Ñ+¨MÓ:ˆà×(Ñ(¨Ó7ˆØŸ™ MÓ2ˆØ×(Ñ(¨Ó7ˆäŸ™ MÐ4EÐ3EÐGXÓYˆØ×,Ñ,¨]Ó;ˆØ˜×.Ñ.Ñ.ˆØ˜Ñ!ˆàÐr_   ©	rd   re   rf   r@   r‚   rk   rl   r�   r‘   r’   s   @r]   r$  r$  ¼  s+   ø„ ð7Ð0õ 7ð U§\¡\ð °e·l±l÷ r_   r$  c                   ó`   ‡ — e Zd Zed„ «       Zdej                  dej                  fˆ fd„Zˆ xZS )ÚGemma4AudioCausalConv1dc                 óp   — | j                   d   dz
  | j                  d   z  dz   }|| j                  d   z
  S )Nr   r?   )r  Údilationr	  )r‰   Úeffective_kernel_sizes     r]   Úleft_padz Gemma4AudioCausalConv1d.left_padî  s>   € à!%×!1Ñ!1°!Ñ!4°qÑ!8¸D¿M¹MÈ!Ñ<LÑ LÈqÑ PÐØ$ t§{¡{°1¡~Ñ5Ð5r_   rå   rJ   c                 óz   •— t         j                  j                  || j                  df«      }t        ‰| �  |«      S )Nr   )r   r   rÚ   r:  r�   r�   )r‰   rå   rŠ   s     €r]   r�   zGemma4AudioCausalConv1d.forwardó  s3   ø€ ô �M‰M×Ñ˜a $§-¡-°Ð!3Ó4ˆä‰w‰˜qÓ!Ð!r_   )	rd   re   rf   r   r:  rk   rl   r�   r‘   r’   s   @r]   r6  r6  à  s;   ø„ ð ñ6ó ð6ð"à�<‰<ð"ð 
�‰÷	"ñ "r_   r6  c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚGemma4AudioLightConv1drD   c                 ó6  •— t         ‰| �  «        || _        t        ||j                  |j                  dz  «      | _        t        ||j                  |j                  «      | _        t        |j                  |j                  |j                  |j                  d¬«      | _	        t        |j                  |j                  d¬«      | _        t        |j                  |j                  d¬«      | _        t        |j                     | _        |j"                  | _        y )Nr*   F)r  r  r  Úgroupsr{   T©r  Ú
with_scale)r�   r‚   rD   rv   rŸ   Úlinear_startÚ
linear_endr6  Úconv_kernel_sizeÚdepthwise_conv1dr”   r  r(  Ú	conv_normr   r*  r+  r,  r/  s     €r]   r‚   zGemma4AudioLightConv1d.__init__  så   ø€ Ü‰ÑÔØˆŒä1°&¸&×:LÑ:LÈf×N`ÑN`ÐcdÑNdÓeˆÔÜ/°¸×8JÑ8JÈF×L^ÑL^Ó_ˆŒÜ 7Ø×*Ñ*Ø×+Ñ+Ø×/Ñ/Ø×%Ñ%Øô!
ˆÔô ,¨F×,>Ñ,>ÀF×DWÑDWÐdhÔiˆÔÜ& v×'9Ñ'9¸v×?RÑ?RÐ_cÔdˆŒÜ˜V×.Ñ.Ñ/ˆŒà!'×!9Ñ!9ˆÕr_   r‹   rJ   c                 ó  — |}| j                  |«      }| j                  |«      }t        j                  j	                  |d¬«      }| j                  |j                  dd«      «      j                  dd«      }t        | j                  t        j                  |j                  «      j                  «      }t        j                  || |«      }| j                  |«      }| j                  |«      }| j!                  |«      }||z  }|S )Nr¯   r³   r?   r*   )r(  rB  r   r   ÚglurE  Ú	transposer1  r,  rk   r2  r¶   r¦   rŽ   rF  r+  rC  )r‰   r‹   r3  r,  s       r]   r�   zGemma4AudioLightConv1d.forward  sç   € Ø ˆà×+Ñ+¨MÓ:ˆØ×)Ñ)¨-Ó8ˆÜŸ™×)Ñ)¨-¸RÐ)Ó@ˆà×-Ñ-¨m×.EÑ.EÀaÈÓ.KÓL×VÑVÐWXÐZ[Ó\ˆô   × 6Ñ 6¼¿¹ÀM×DWÑDWÓ8X×8\Ñ8\Ó]ÐÜŸ™ MÐ4EÐ3EÐGXÓYˆØŸ™ }Ó5ˆàŸ™ MÓ2ˆØŸ™¨Ó6ˆØ˜Ñ!ˆØÐr_   r4  r’   s   @r]   r=  r=    s+   ø„ ð:Ð0õ :ð( U§\¡\ð °e·l±l÷ r_   r=  c            
       ó    ‡ — e Zd Zdedefˆ fd„Zdej                  dej                  dz  dej                  de	e
   d	ej                  f
d
„Zˆ xZS )ÚGemma4AudioLayerrD   rM   c                 óp  •— t         ‰| �  «        || _        t        |«      | _        t        |«      | _        t        ||«      | _        t        |«      | _	        t        |j                  «      | _        t        |j                  «      | _        t        |j                  «      | _        |j                  | _        y r�   )r�   r‚   rD   r$  Úfeed_forward1Úfeed_forward2r¿   Ú	self_attnr=  Úlconv1dr”   rŸ   Únorm_pre_attnÚnorm_post_attnÚnorm_outr,  rÖ   s      €r]   r‚   zGemma4AudioLayer.__init__+  s�   ø€ Ü‰ÑÔØˆŒä3°FÓ;ˆÔÜ3°FÓ;ˆÔÜ-¨f°iÓ@ˆŒÜ-¨fÓ5ˆŒä*¨6×+=Ñ+=Ó>ˆÔÜ+¨F×,>Ñ,>Ó?ˆÔÜ% f×&8Ñ&8Ó9ˆŒà!'×!9Ñ!9ˆÕr_   r‹   rF   Nrë   ÚkwargsrJ   c                 ó@  — t        | j                  t        j                  | j                  j
                  j                  «      j                  «      }| j                  |«      }|}t        j                  || |«      }| j	                  |«      }| j                  |||¬«      \  }}t        j                  || |«      }| j                  |«      }||z  }| j                  |«      }| j                  |«      }t        j                  || |«      }| j                  |«      }|S )N)r‹   rë   rF   )r1  r,  rk   r2  rQ  r  r¶   r¦   rM  rŽ   rO  rR  rP  rN  rS  )r‰   r‹   rF   rë   rT  r,  r3  rX   s           r]   r�   zGemma4AudioLayer.forward:  s  € ô   × 6Ñ 6¼¿¹ÀD×DVÑDV×D]ÑD]×DcÑDcÓ8d×8hÑ8hÓiÐà×*Ñ*¨=Ó9ˆØ ˆäŸ™ MÐ4EÐ3EÐGXÓYˆØ×*Ñ*¨=Ó9ˆàŸ>™>Ø'Ø 3Ø)ð *ó 
Ñˆ�qô Ÿ™ MÐ4EÐ3EÐGXÓYˆØ×+Ñ+¨MÓ:ˆØ˜Ñ!ˆàŸ™ ]Ó3ˆØ×*Ñ*¨=Ó9ˆäŸ™ MÐ4EÐ3EÐGXÓYˆØŸ™ mÓ4ˆàÐr_   )rd   re   rf   r@   r�   r‚   rk   rl   rt   r   r   r�   r‘   r’   s   @r]   rK  rK  *  si   ø„ ð:Ð0ð :¸Sõ :ð à—|‘|ð ð ×(Ñ(¨4Ñ/ð ð #Ÿ\™\ð	 ð
 Ð+Ñ,ð ð 
�‰÷ r_   rK  c                   óÜ   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  dej                  fd„Zdej                  dej                  dej                  dej                  fd„Zˆ xZ	S )	ÚGemma4VisionPatchEmbedderrD   c                 óŠ  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        |j
                  | _        t        j                  d| j                  dz  z  | j                  d¬«      | _        t        j                  t        j                  d| j
                  | j                  «      «      | _        y )Nr	   r*   Frz   )r�   r‚   rD   rŸ   Ú
patch_sizeÚposition_embedding_sizer   r„   Ú
input_projrÓ   rk   ÚonesÚposition_embedding_tabler/  s     €r]   r‚   z"Gemma4VisionPatchEmbedder.__init__a  s’   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ ×+Ñ+ˆŒØ'-×'EÑ'EˆÔ$äŸ)™) A¨¯©¸Ñ(:Ñ$:¸D×<LÑ<LÐSXÔYˆŒÜ(*¯©´U·Z±ZÀÀ4×C_ÑC_Ðae×aqÑaqÓ5rÓ(sˆÕ%r_   Úpixel_position_idsÚpadding_positionsrJ   c                 ó  — |j                  d¬«      }t        j                  |d   | j                  d   «      }t        j                  |d   | j                  d   «      }||z   }t	        j
                  |j                  d«      d|«      }|S )ak  Compute 2-D patch position embeddings via embedding lookup.

        ``pixel_position_ids`` has shape ``(batch, num_patches, 2)`` where the
        last dimension holds (x, y) indices into ``position_embedding_table``
        (shape ``(2, position_embedding_size, hidden_size)``).  The result is the
        sum of the x- and y-embeddings for each patch.
        r   ©r1  ©.r   ©.r?   r?   r¯   ç        )rŽ   rÙ   Ú	embeddingr]  rk   Úwherer©   )r‰   r^  r_  Úclamped_positionsÚx_embÚy_embrë   s          r]   Ú_position_embeddingsz.Gemma4VisionPatchEmbedder._position_embeddingsk  sŽ   € ð /×4Ñ4¸Ð4Ó;Ðô —‘Ð-¨fÑ5°t×7TÑ7TÐUVÑ7WÓXˆÜ—‘Ð-¨fÑ5°t×7TÑ7TÐUVÑ7WÓXˆØ# e™mÐÜ#Ÿk™kÐ*;×*EÑ*EÀbÓ*IÈ3ÐPcÓdÐØ"Ð"r_   Úpixel_valuesc                 óÜ   — d|dz
  z  }| j                   j                  j                  x}j                  r|j	                  |«      }| j                  |«      }| j                  ||«      }||z   S )Nr*   ç      à?)r[  r  r¶   Úis_floating_pointr·   rj  )r‰   rk  r^  r_  Útarget_dtyper‹   rë   s          r]   r�   z!Gemma4VisionPatchEmbedder.forward�  sn   € ð ˜L¨3Ñ.Ñ/ˆØ ŸO™O×2Ñ2×8Ñ8Ð8ˆL×KÒKØ'Ÿ?™?¨<Ó8ˆLØŸ™¨Ó5ˆØ"×7Ñ7Ð8JÐL]Ó^ÐØÐ2Ñ2Ð2r_   )
rd   re   rf   rC   r‚   rk   rl   rj  r�   r‘   r’   s   @r]   rW  rW  `  su   ø„ ðtÐ1õ tð#°u·|±|ð #ÐX]×XdÑXdð #Ðin×iuÑiuó #ð,	3Ø!ŸL™Lð	3Ø>C¿l¹lð	3Ø_d×_kÑ_kð	3à	�‰÷	3r_   rW  c                   ó.  ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  dede	ej                  ej                  f   fd„Z
	 ddej                  dej                  d
ej                  ded	z  de	ej                  ej                  f   f
d„Zˆ xZS )ÚGemma4VisionPoolera[  Spatial pooling and ``sqrt(hidden_size)`` scaling for vision encodings.

    The scaling expands the activation magnitude, which can exceed the float16 range, so it is
    computed in float32 and the pooled features are returned in float32. The caller
    (``Gemma4VisionModel.forward``) standardizes them and casts back to the working dtype.
    rD   c                 ól   •— t         ‰| �  «        |j                  | _        | j                  dz  | _        y )Nrm  )r�   r‚   rŸ   Úroot_hidden_sizer/  s     €r]   r‚   zGemma4VisionPooler.__init__•  s/   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ $× 0Ñ 0°#Ñ 5ˆÕr_   r‹   r^  ÚlengthrJ   c                 óŽ  — |j                   d   }t        ||z  dz  «      }|dz  }||z  |k7  r%t        d|j                   › d|› d|›d|›d|› d	�«      ‚|j                  d
¬«      }|d   j	                  dd¬«      d
   dz   }t        j                  ||d¬«      }	|	d   ||z  |	d   z  z   }	t        j                  |	j                  «       |«      j                  «       |z  }
|
j                  dd«      |j                  «       z  }t        j                  |
d
k(  j                  d¬«      «      }|j                  |j                  «      |fS )zÙ
        2D spatial pooling according to patch positions.
        Pools the input tokens by averaging patches within a `k^2` grid, where `k` is determined by the ratio between
        input and output lengths
        r?   rm  r*   zCannot pool z to z: k=z^2 times length=z	 must be ú.r   ra  rb  r¯   T©r´   ÚkeepdimÚfloor)Úrounding_moderc  r³   )rØ   r�   Ú
ValueErrorrŽ   r¦   rk   ÚdivrÙ   Úone_hotÚlongrˆ   rI  ró   Úallr·   r¶   )r‰   r‹   r^  rt  Úinput_seq_lenÚkÚ	k_squaredrg  Úmax_xÚkernel_idxsÚweightsÚoutputr  s                r]   Ú_avg_pool_by_positionsz)Gemma4VisionPooler._avg_pool_by_positionsš  sh  € ð &×+Ñ+¨AÑ.ˆÜ� &Ñ(¨SÑ0Ó1ˆØ�q‘Dˆ	Ø�vÑ Ò.ÜØ˜}×2Ñ2Ð3°4¸°x¸uÀ!ÀÐEVÈvÈiÐW`ÐanÐ`oÐopÐqóð ð /×4Ñ4¸Ð4Ó;ÐØ! &Ñ)×-Ñ-°"¸dÐ-ÓCÀAÑFÈÑJˆÜ—i‘iÐ 1°1ÀGÔLˆØ! &Ñ)¨U°a©Z¸;ÀvÑ;NÑ,NÑNˆÜ—)‘)˜K×,Ñ,Ó.°Ó7×=Ñ=Ó?À)ÑKˆØ×"Ñ" 1 aÓ(¨=×+>Ñ+>Ó+@Ñ@ˆÜ× Ñ  '¨Q¡,×!3Ñ!3¸Ð!3Ó!:Ó;ˆØ�y‰y˜×,Ñ,Ó-¨tÐ3Ð3r_   Nr_  Úoutput_lengthc                 ó8  — ||j                   d   kD  rt        d|› d|j                   d   › d�«      ‚|j                  |j                  d«      d«      }|j                   d   |k7  r| j	                  |||«      \  }}|j                  «       | j                  z  }||fS )Nr?   z*Cannot output more soft tokens (requested z) than there are patches (z9). Change the value of `num_soft_tokens` when processing.r¯   rd  )rØ   r{  rò   r©   r‡  rˆ   rs  )r‰   r‹   r^  r_  rˆ  s        r]   r�   zGemma4VisionPooler.forwardµ  sÄ   € ð ˜=×.Ñ.¨qÑ1Ò1ÜØ<¸]¸Oð LØ"×(Ñ(¨Ñ+Ð,Ð,eðgóð ð
 &×1Ñ1Ð2C×2MÑ2MÈbÓ2QÐSVÓWˆà×Ñ˜qÑ! ]Ò2Ø/3×/JÑ/JØÐ1°=ó0Ñ,ˆMÐ,ð &×+Ñ+Ó-°×0EÑ0EÑEˆØÐ/Ð/Ð/r_   r�   )rd   re   rf   rg   rC   r‚   rk   rl   r�   rj   r‡  r�   r‘   r’   s   @r]   rq  rq  �  s±   ø„ ñð6Ð1õ 6ð
4Ø"Ÿ\™\ð4Ø?D¿|¹|ð4ØUXð4à	ˆu�|‰|˜UŸ\™\Ð)Ñ	*ó4ð@ %)ñ0à—|‘|ð0ð "ŸL™Lð0ð !Ÿ<™<ð	0ð
 ˜T‘zð0ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷0r_   rq  c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚGemma4VisionMLPrD   c                 ó
  •— t         ‰| �  | |«       t        || j                  | j                  «      | _        t        || j                  | j                  «      | _        t        || j                  | j                  «      | _        y r�   )r�   r‚   rv   rŸ   Úintermediate_sizeÚ	gate_projÚup_projÚ	down_projr/  s     €r]   r‚   zGemma4VisionMLP.__init__Ñ  sf   ø€ Ü‰Ñ˜˜vÔ&Ü.¨v°t×7GÑ7GÈ×I_ÑI_Ó`ˆŒÜ,¨V°T×5EÑ5EÀt×G]ÑG]Ó^ˆŒÜ.¨v°t×7MÑ7MÈt×O_ÑO_Ó`ˆ�r_   )rd   re   rf   rC   r‚   r‘   r’   s   @r]   r‹  r‹  Ð  s   ø„ ðaÐ1÷ añ ar_   r‹  rå   rº   r¹   Úunsqueeze_dimc           	      ó¸  — |j                   d   }| j                   d   }d|d|z  z  z  }|dk  rt        d|› d|› d|› d�«      ‚|g|z  }t        j                  | |d¬«      }	t        j                  ||d¬«      }
t        j                  ||d¬«      }t	        |«      D �cg c]  }t        |	|   |
|   ||   |¬	«      ‘Œ }}t        j                  |d¬«      S c c}w )
ak  Applies multidimensional RoPE to inputs.

    Args:
        x (`torch.Tensor`): The tensor to embed.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            If position_ids.ndim + 2 == x.ndim, then this function passes through to `apply_rotary_pos_emb()`.
            Otherwise, position_ids is used to split the inputs, x, into multiple pieces, where each piece is fed to
            `apply_rotary_pos_emb()`, and then concatenated back together.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.

    Returns:
      Tensor of shape [B, L, N, H] with RoPE applied.
    r¯   r*   r   zEInvalid configuration: num_rotated_channels_per_dim must be > 0, got z (num_input_channels=z, ndim=ú)r³   )rå   rº   r¹   r‘  )rØ   r{  rk   ÚsplitÚranger:   r¸   )rå   rº   r¹   rH   r‘  ÚndimÚnum_input_channelsÚnum_rotated_channels_per_dimÚsplit_sizesÚx_partsÚ	cos_partsÚ	sin_partsr�  Úy_partss                 r]   Úapply_multidimensional_roperž  Ø  s  € ð8 ×Ñ˜bÑ!€DØŸ™ ™ÐØ#$Ð(:¸qÀ4¹xÑ(HÑ#IÐ à# qÒ(ÜðØ,Ð-Ð-BÐCUÐBVð WØ�F˜!ðó
ð 	
ð 0Ð0°4Ñ7€KÜ�k‰k˜!˜[¨bÔ1€GÜ—‘˜C °"Ô5€IÜ—‘˜C °"Ô5€Iô �t“öð ô 	Ø�a‰jØ˜!‘Ø˜!‘Ø'ö		
ð€Gð ô �9‰9�W "Ô%Ð%ùòs   ÂCc                   óž   — e Zd Ze	 	 	 d	dedz  dej                  dz  dedz  dede	f   fd„«       Z
 ej                  «       ed„ «       «       Zy)
ÚGemma4VisionRotaryEmbeddingNrD   r±   rÞ   rJ   ztorch.Tensorc                 ó$  — | j                   d   }t        | dd«      xs | j                  | j                  z  }|dz  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetarÆ   Nr*   r›   r   rµ   )r±   r¶   )	Úrope_parametersÚgetattrrŸ   rÅ   rk   r¨   Úint64r·   rˆ   )rD   r±   rÞ   Úbaser´   Úspatial_dimÚattention_factorÚinv_freqs           r]   Úcompute_default_rope_parametersz;Gemma4VisionRotaryEmbedding.compute_default_rope_parameters  s�   € ð& ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆð ˜Q‘hˆàÐØØÜ—‘˜Q ¨Q´e·k±kÔB×EÑEÈVÔ[`×[fÑ[fÐEÓgÐjuÑuñwñ
ˆð Ð)Ð)Ð)r_   c                 ó.  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}g g }}t        d«      D ]ä  }|d d …d d …|f   }|d d …d d d …f   j                  «       }	t        |d¬«      5  |j                  «       |	j                  «       z  j                  dd«      }
t        j                  |
|
fd¬	«      }|j                  «       | j                  z  }|j!                  «       | j                  z  }d d d «       |j#                  «       |j#                  «       Œæ t        j                  |d¬	«      j	                  |j$                  ¬
«      }t        j                  |d¬	«      j	                  |j$                  ¬
«      }||fS # 1 sw Y   Œ“xY w)Nr   r¯   r?   ÚmpsÚcpur*   F)Údevice_typeÚenabledr³   rµ   )r©  rˆ   ÚexpandrØ   r·   r±   Ú
isinstanceÚtyperi   r•  r%   rI  rk   r¸   rº   Úattention_scalingr¹   Úappendr¶   )r‰   rå   rH   Úinv_freq_expandedr®  Úall_cosÚall_sinÚiÚdim_position_idsÚdim_position_ids_expandedÚfreqsÚembrº   r¹   s                 r]   r�   z#Gemma4VisionRotaryEmbedding.forward4  s¿  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐÜ'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆð ˜r�ˆÜ�q“ò 
	 ˆAØ+ªAªq°!¨GÑ4ÐØ(8º¸DÂ!¸Ñ(D×(JÑ(JÓ(LÐ%ä¨KÀÔGñ 9Ø*×0Ñ0Ó2Ð5N×5TÑ5TÓ5VÑV×aÑaÐbcÐefÓg�Ü—i‘i ¨ °BÔ7�Ø—g‘g“i $×"8Ñ"8Ñ8�Ø—g‘g“i $×"8Ñ"8Ñ8�÷	9ð
 �N‰N˜3ÔØ�N‰N˜3Õð
	 ô �i‰i˜ RÔ(×+Ñ+°!·'±'Ð+Ó:ˆÜ�i‰i˜ RÔ(×+Ñ+°!·'±'Ð+Ó:ˆØ�Cˆxˆ÷9ð 9ús   Ã4BHÈH	©NNN)rd   re   rf   ÚstaticmethodrC   rk   r±   r�   rj   rˆ   rª  r½   r   r�   rS   r_   r]   r   r     s‚   „ Øà,0Ø&*Ø"ñ *Ø" TÑ)ð *à—‘˜tÑ#ð *ð �t‘ð *ð 
ˆ~˜uÐ$Ñ	%ò	 *ó ð *ðD €U‡]�]ƒ_Øñó ó ñr_   r   c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej                  dej                  dej                  dz  dej                  dz  d	e	e
   d
eej                  ej                  dz  eej                     dz  f   fd„Zˆ xZS )ÚGemma4VisionAttentionrD   rM   c                 ó6  •— t         ‰| �  | ||«       | `| `| `d| _        d| _        t        ||j                  |j                  | j                  z  «      | _        t        ||j                  |j                  | j                  z  «      | _        t        ||j                  |j                  | j                  z  «      | _        t        ||j                  | j                  z  |j                  «      | _        t!        | j                  |j"                  d¬«      | _        y )Nr›   Fr@  )r�   r‚   Úattn_logit_softcappingrT   Ú
is_slidingÚscalingÚ	is_causalrv   rŸ   Únum_key_value_headsrÆ   rÏ   rÅ   rÎ   rÐ   Úo_projr”   r  Úv_normrÖ   s      €r]   r‚   zGemma4VisionAttention.__init__O  sï   ø€ Ü‰Ñ˜˜v yÔ1ØÐ'ØÐØˆOØˆŒØˆŒÜ+¨F°F×4FÑ4FÈ×HbÑHbÐei×erÑerÑHrÓsˆŒÜ+¨F°F×4FÑ4FÈ×HbÑHbÐei×erÑerÑHrÓsˆŒÜ+¨F°F×4FÑ4FÈ×HbÑHbÐei×erÑerÑHrÓsˆŒÜ+¨F°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ð`f×`rÑ`rÓsˆŒÜ# D§M¡M°v×7JÑ7JÐW\Ô]ˆ�r_   Nr‹   rë   rF   rH   rT  rJ   c                 óN  — |j                   d d }g |¢d‘| j                  ‘­}|\  }}	| j                  |«      j                  |«      }
| j	                  |
«      }
t        |
||	|«      }
|
j                  dd«      }
| j                  |«      j                  |«      }| j                  |«      }t        |||	|«      }|j                  dd«      }| j                  |«      j                  |«      }| j                  |«      }|j                  dd«      }t        j                  | j                  j                  t        «      } || |
|||f| j                   r| j"                  nd| j$                  dœ|¤Ž\  }} |j&                  g |¢d‘­Ž j)                  «       }| j+                  |«      }||fS )Nr¯   r?   r*   rd  )ÚdropoutrÄ  )rØ   rÆ   rÎ   rç   Úq_normrž  rI  rÏ   Úk_normrÐ   rÈ  r   Úget_interfacerD   Ú_attn_implementationr;   ÚtrainingÚattention_dropoutrÄ  rÛ   rÜ   rÇ  )r‰   r‹   rë   rF   rH   rT  Úinput_shaperø   rº   r¹   rù   rú   rû   Úattention_interfacer  r  s                   r]   r�   zGemma4VisionAttention.forward\  sª  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà&‰ˆˆSà—{‘{ =Ó1×6Ñ6°|ÓDˆØ—{‘{ <Ó0ˆÜ2°<ÀÀcÈ<ÓXˆØ#×-Ñ-¨a°Ó3ˆà—[‘[ Ó/×4Ñ4°\ÓBˆ
Ø—[‘[ Ó,ˆ
Ü0°¸SÀ#À|ÓTˆ
Ø×)Ñ)¨!¨QÓ/ˆ
à—{‘{ =Ó1×6Ñ6°|ÓDˆØ—{‘{ <Ó0ˆØ#×-Ñ-¨a°Ó3ˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð /3¯mªm�D×*Ò*ÀØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r_   r½  )rd   re   rf   rC   r�   r‚   rk   rl   Ú
LongTensorr   r   rj   r�   r‘   r’   s   @r]   rÀ  rÀ  M  s­   ø„ ð^Ð1ð ^¸cõ ^ð  -1Ø.2Ø04ñ,)à—|‘|ð,)ð #Ÿ\™\ð,)ð Ÿ™ tÑ+ð	,)ð
 ×&Ñ&¨Ñ-ð,)ð Ð+Ñ,ð,)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷,)r_   rÀ  c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej                  dej                  dej                  dz  dej                  dz  d	e	e
   d
eej                  eej                  ej                  f   dz  f   fd„Zˆ xZS )ÚGemma4VisionEncoderLayerrD   rM   c                 ól   •— t         ‰| �  | ||«       t        ||¬«      | _        t	        |«      | _        y ©N©rD   rM   )r�   r‚   rÀ  rO  r‹  ÚmlprÖ   s      €r]   r‚   z!Gemma4VisionEncoderLayer.__init__�  s.   ø€ Ü‰Ñ˜˜v yÔ1Ü.°fÈ	ÔRˆŒÜ" 6Ó*ˆ�r_   Nr‹   rë   rF   rH   rT  rJ   c                 ó   — |}| j                  |«      } | j                  d||||dœ|¤Ž\  }}| j                  |«      }||z   }|}| j                  |«      }| j	                  |«      }| j                  |«      }||z   }|S )N)r‹   rë   rF   rH   rS   )Úinput_layernormrO  Úpost_attention_layernormÚpre_feedforward_layernormrÙ  Úpost_feedforward_layernorm)r‰   r‹   rë   rF   rH   rT  r3  rX   s           r]   r�   z Gemma4VisionEncoderLayer.forward’  s¨   € ð !ˆà×,Ñ,¨]Ó;ˆà)˜4Ÿ>™>ð 
Ø'Ø 3Ø)Ø%ñ	
ð
 ñ
Ñˆ�qð ×5Ñ5°mÓDˆØ  =Ñ0ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ  =Ñ0ˆàÐr_   r½  )rd   re   rf   rC   r�   r‚   rk   rl   rÓ  r   r   rj   ÚFloatTensorr�   r‘   r’   s   @r]   rÕ  rÕ  Œ  s¯   ø„ ð+Ð1ð +¸cõ +ð -1Ø.2Ø04ñà—|‘|ðð #Ÿ\™\ðð Ÿ™ tÑ+ð	ð
 ×&Ñ&¨Ñ-ðð Ð+Ñ,ðð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷r_   rÕ  c                   óŒ   ‡ — e Zd Zdefˆ fd„Z	 d
dej                  dej                  dej                  dz  dee	   de
f
d	„Zˆ xZS )ÚGemma4VisionEncoderrD   c           	      ó  •— t         ‰| �  «        || _        |j                  | _        t        |«      | _        t        j                  t        | j                  «      D �cg c]  }t        ||¬«      ‘Œ c}«      | _        y c c}w r×  )r�   r‚   rD   Únum_hidden_layersÚ
num_layersr   Ú
rotary_embr   Ú
ModuleListr•  rÕ  Úlayers)r‰   rD   r¸  rŠ   s      €r]   r‚   zGemma4VisionEncoder.__init__²  sc   ø€ Ü‰ÑÔØˆŒØ ×2Ñ2ˆŒÜ5°fÓ=ˆŒÜ—m‘mÜKPÐQU×Q`ÑQ`ÓKaÖbÀaÔ%¨V¸qÖAÒbó
ˆ�ùÚbs   ÁA?NrE   rF   r^  rT  rJ   c                 óÜ   — t        | j                  ||¬«      }|}| j                  ||«      }| j                  d| j                  j                   D ]  } ||f|||dœ|¤Ž}Œ t        |¬«      S )z�
        pixel_position_ids (torch.Tensor):
            Patch positions as (x, y) coordinates in the image as [batch, num_patches, 2].
        )rD   rE   rF   N)rF   rë   rH   ©Úlast_hidden_state)r   rD   rå  rç  rã  r   )r‰   rE   rF   r^  rT  r‹   rë   Údecoder_layers           r]   r�   zGemma4VisionEncoder.forward»  s�   € ô 3Ø—;‘;Ø'Ø)ô
ˆð &ˆØ"Ÿo™o¨mÐ=OÓPÐð "Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÐIò 	ˆMÙ)Øðà-Ø$7Ø/ñ	ð
 ñ‰Mð	ô '¸ÔGÐGr_   r�   )rd   re   rf   rC   r‚   rk   rl   rÓ  r   r   r   r�   r‘   r’   s   @r]   rá  rá  ±  si   ø„ ð
Ð1õ 
ð 7;ñ	Hà—|‘|ðHð Ÿ™ðHð "×,Ñ,¨tÑ3ð	Hð
 Ð+Ñ,ðHð 
!÷Hr_   rá  c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚGemma4TextMLPrD   rM   c                 óÔ   •— |j                   |j                  z
  }||cxk\  xr dkD  nc }|j                  xr |}t        ‰| �  «        |j
                  |r
dz  | _        y dz  | _        y )Nr   r*   r?   )rã  Únum_kv_shared_layersÚuse_double_wide_mlpr�   r‚   r�  )r‰   rD   rM   Úfirst_kv_shared_layer_idxÚis_kv_shared_layerrð  rŠ   s         €r]   r‚   zGemma4TextMLP.__init__á  si   ø€ Ø$*×$<Ñ$<¸v×?ZÑ?ZÑ$ZÐ!Ø&Ð*CÖGÀaÔGÐØ$×8Ñ8ÒOÐ=OÐÜ‰ÑÔØ!'×!9Ñ!9ÑBU¸QÑ!]ˆÕÐ[\Ñ!]ˆÕr_   )rd   re   rf   rB   r�   r‚   r‘   r’   s   @r]   rí  rí  à  s    ø„ ð^Ð/ð ^¸C÷ ^ñ ^r_   rí  c                   ó   — e Zd Zddefd„Zy)ÚGemma4TextRotaryEmbeddingNrD   c                 ó¦  — t         j                  j                  | «       |j                  | _        |j                  | _        || _        t        |j                  «      | _        i | _	        i | _
        | j                  D ]Ò  }| j                  j                  |   }|€Œ|d   x}dk7  r
t        |   }n| j                  }|| j                  |<   || j                  |<   ||dœ}|dk(  r
|dk(  rd|d<    || j                  fi |¤Ž\  }}	| j                  |› d�|d	¬
«       | j                  |› d�|j                  «       d	¬
«       t!        | |› d�|	«       ŒÔ y )NÚ	rope_typeÚdefault)r±   Ú
layer_typerQ   ÚproportionalÚglobal_head_dimÚhead_dim_keyÚ	_inv_freqFr�   Ú_original_inv_freqÚ_attention_scaling)r   ÚModuler‚   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrD   ÚsetÚlayer_typesÚrope_init_fnsrö  r£  r   rª  r†   ÚcloneÚsetattr)
r‰   rD   r±   rø  Úrope_paramsrö  Úrope_init_fnÚrope_init_fn_kwargsÚcurr_inv_freqÚcurr_attention_scalings
             r]   r‚   z"Gemma4TextRotaryEmbedding.__init__ê  sk  € Ü
�	‰	×Ñ˜4Ô Ø"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àˆŒÜ˜v×1Ñ1Ó2ˆÔØSUˆÔØ)+ˆŒà×*Ñ*ò 	UˆJØŸ+™+×5Ñ5°jÑAˆKØÐ"Øà(¨Ñ5Ð5�	¸)ÒCÜ2°9Ñ=‘à#×CÑC�à-9ˆD×Ñ˜zÑ*Ø)2ˆD�N‰N˜:Ñ&à-3À:Ñ"NÐØÐ-Ò-°)¸~Ò2MØ6GÐ# NÑ3á4@ÀÇÁÑ4dÐPcÑ4dÑ1ˆMÐ1Ø× Ñ  J <¨yÐ!9¸=ÐUZÐ Ô[Ø× Ñ  J <Ð/AÐ!BÀM×DWÑDWÓDYÐfkÐ ÔlÜ�D˜Z˜LÐ(:Ð;Ð=SÕTñ)	Ur_   ©NN)rd   re   rf   rB   r‚   rS   r_   r]   rô  rô  é  s   „ ñUÐ/ô Ur_   rô  c                   ó  ‡ — e Zd ZdZdedefˆ fd„Z	 ddej                  dej                  dej                  dz  d	e	e
eej                  ej                  f   f   d
edz  dee   deej                  ej                  dz  f   fd„Zˆ xZS )ÚGemma4TextAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrD   rM   c                 óØ  •— t         ‰| �  «        t        |d«      r|j                  |   nd | _        || _        || _        | j                  dk(  | _        | j                  r|j                  nd | _        | j                  s|j                  r|j                  n|j                  | _
        |j                  xr | j                   | _        | j                  r|j                  n|j                  }|j                  |z  | _        d| _        | j
                  j$                  | _        |j&                  dk7  | _        | j
                  j*                  t-        | j
                  dd«      z
  }||cxk\  xr dk\  nc | _        |j                  d | }| j.                   xr6 |t1        |«      dz
  |d d d…   j3                  |j                  |   «      z
  k(  | _        t7        j8                  |j:                  |j                  | j                  z  |j<                  ¬	«      | _        tA        | j                  |jB                  ¬
«      | _"        | j.                  s×tA        | j                  |jB                  ¬
«      | _#        tA        | j                  |jB                  d¬«      | _$        t7        j8                  |j:                  || j                  z  |j<                  ¬	«      | _%        | j                  s9t7        j8                  |j:                  || j                  z  |j<                  ¬	«      nd | _&        t7        j8                  |j                  | j                  z  |j:                  |j<                  ¬	«      | _'        y )Nr  rR   r›   r  rï  r   r?   r¯   rz   )r´   r  Fr@  )(r�   r‚   Úhasattrr  rø  rD   rM   rÃ  rT   rú  rÆ   Úattention_k_eq_vÚuse_alternative_attentionÚnum_global_key_value_headsrÆ  rÅ   Únum_key_value_groupsrÄ  rÐ  Úuse_bidirectional_attentionrÅ  rã  r¤  rò  ÚlenÚindexÚstore_full_length_kvr   r„   rŸ   Úattention_biasrÎ   r”   r  rË  rÌ  rÈ  rÏ   rÐ   rÇ  )r‰   rD   rM   rÆ  rñ  Úprev_layersrŠ   s         €r]   r‚   zGemma4TextAttention.__init__  sÑ  ø€ Ü‰ÑÔÜ;BÀ6È=Ô;Y˜&×,Ñ,¨YÒ7Ð_cˆŒØˆŒØ"ˆŒØŸ/™/Ð-@Ñ@ˆŒØ7;·²˜f×3Ò3ÈDˆÔà6:·o²oÈ&×J`ÒJ`˜×.Ò.Ðfl×fuÑfuˆŒØ)/×)@Ñ)@Ò)XÈÏÉÐEXˆÔ&à15×1OÒ1OˆF×-Ò-ÐU[×UoÑUoð 	ð %+×$>Ñ$>ÐBUÑ$UˆÔ!ØˆŒØ!%§¡×!>Ñ!>ˆÔØ×;Ñ;¸uÑDˆŒð %)§K¡K×$AÑ$AÄGÈDÏKÉKÐYoÐqrÓDsÑ$sÐ!Ø"+Ð/HÖ"MÈAÔ"MˆÔØ×(Ñ(Ð)CÐ*CÐDˆØ(,×(?Ñ(?Ð$?ò %/ÀIÔQTÐU`ÓQaÐdeÑQeÐhsÙˆbˆDñi
ç
‰%�×"Ñ" 9Ñ-Ó
.ñR/ñ E/ˆÔ!ô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô $¨¯©¸6×;NÑ;NÔOˆŒð ×&Ò&Ü'¨D¯M©M¸v×?RÑ?RÔSˆDŒKÜ'¨¯©¸6×;NÑ;NÐ[`ÔaˆDŒKäŸ)™)Ø×"Ñ"Ð$7¸$¿-¹-Ñ$GÈf×NcÑNcôˆDŒKð
 ×5Ò5ô —	‘	˜&×,Ñ,Ð.AÀDÇMÁMÑ.QÐX^×XmÑXmÕnàð ŒKô —i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆ�r_   Nr‹   rë   rF   rb   rG   rT  rJ   c                 óÄ  — |j                   d d }g |¢d‘| j                  ‘­}|\  }	}
| j                  |«      j                  |«      }| j	                  |«      }t        ||	|
d¬«      }|j                  dd«      }| j                  rI|| j                     \  }}|j                  |j                  «      }|j                  |j                  «      }n£| j                  |«      j                  |«      }| j                  � | j                  |«      j                  |«      n|}| j                  |«      }t        ||	|
d¬«      }|j                  dd«      }| j                  |«      }|j                  dd«      }|�,| j                  s |j                  ||| j                   «      \  }}| j"                  r||f|| j                  <   t%        j&                  | j(                  j*                  t,        «      } || ||||f| j.                  r| j0                  nd| j2                  | j4                  dœ|¤Ž\  }} |j6                  g |¢d‘­Ž j9                  «       }| j;                  |«      }||fS )Nr¯   r*   )r‘  r?   rd  )rÊ  rÄ  rT   )rØ   rÆ   rÎ   rç   rË  r:   rI  rò  rø  r·   r±   rÏ   rÐ   rÌ  rÈ  ÚupdaterM   r  r   rÍ  rD   rÎ  r;   rÏ  rÐ  rÄ  rT   rÛ   rÜ   rÇ  )r‰   r‹   rë   rF   rb   rG   rT  rÑ  rø   rº   r¹   rù   rú   rû   rÒ  r  r  s                    r]   r�   zGemma4TextAttention.forward?  sR  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà&‰ˆˆSà—{‘{ =Ó1×6Ñ6°|ÓDˆØ—{‘{ <Ó0ˆÜ+¨L¸#¸sÐRSÔTˆØ#×-Ñ-¨a°Ó3ˆð
 ×"Ò"Ø'7¸¿¹Ñ'HÑ$ˆJ˜à#Ÿ™ |×':Ñ':Ó;ˆJØ'Ÿ?™?¨<×+>Ñ+>Ó?‰LàŸ™ ]Ó3×8Ñ8¸ÓFˆJØLPÏKÉKÐLc˜4Ÿ;™; }Ó5×:Ñ:¸<ÔHÐisˆLàŸ™ ZÓ0ˆJÜ-¨j¸#¸sÐRSÔTˆJØ#×-Ñ-¨a°Ó3ˆJàŸ;™; |Ó4ˆLØ'×1Ñ1°!°QÓ7ˆLàÐ&¨t×/FÒ/FØ'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜Ø×$Ò$Ø0:¸LÐ0HÐ˜TŸ_™_Ñ-ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð /3¯mªm�D×*Ò*ÀØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r_   r�   )rd   re   rf   rg   rB   r�   r‚   rk   rl   rh   ri   rj   r   r   r   r�   r‘   r’   s   @r]   r  r    s·   ø„ ÙGð/
Ð/ð /
¸Cõ /
ðn )-ñ=)à—|‘|ð=)ð #Ÿ\™\ð=)ð Ÿ™ tÑ+ð	=)ð
 ˜s E¨%¯,©,¸¿¹Ð*DÑ$EÐEÑFð=)ð  ™ð=)ð Ð-Ñ.ð=)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷=)r_   r  c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚGemma4TextExpertsrD   c                 ó–   •— t         ‰| �  «        |j                  | _        |j                  | _        t
        |j                     | _        y r�   )r�   r‚   Únum_expertsÚmoe_intermediate_sizeÚintermediate_dimr   Úhidden_activationr+  r/  s     €r]   r‚   zGemma4TextExperts.__init__€  s<   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ &× <Ñ <ˆÔÜ˜V×5Ñ5Ñ6ˆ�r_   )rd   re   rf   rB   r‚   r‘   r’   s   @r]   r  r    s   ø„ ð7Ð/÷ 7ñ 7r_   r  c                   óz   ‡ — e Zd Zdefˆ fd„Zdej                  deej                  ej                  f   fd„Zˆ xZ	S )ÚGemma4TextRouterrD   c                 ó(  •— t         ‰| �  «        || _        |j                  | _        | j                  dz  | _        |j
                  | _        t        | j                  | j                  d¬«      | _        t        j                  |j                  |j                  d¬«      | _        t        j                  t        j                  | j                  «      «      | _        t        j                  t        j                  |j                  «      «      | _        y )NrÁ   Fr@  rz   )r�   r‚   rD   rŸ   Úscalar_root_sizer  r  r”   r  r   r„   r!  ÚprojrÓ   rk   r\  ÚscaleÚper_expert_scaler/  s     €r]   r‚   zGemma4TextRouter.__init__ˆ  s¼   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ $× 0Ñ 0°$Ñ 6ˆÔØ×&Ñ&ˆŒä! $×"2Ñ"2¸¿¹ÈUÔSˆŒ	Ü—I‘I˜f×0Ñ0°&×2DÑ2DÈ5ÔQˆŒ	Ü—\‘\¤%§*¡*¨T×-=Ñ-=Ó">Ó?ˆŒ
Ü "§¡¬U¯Z©Z¸×8JÑ8JÓ-KÓ LˆÕr_   r‹   rJ   c                 ó–  — | j                  |«      }|| j                  z  | j                  z  }| j                  |«      }t        j
                  j                  |dt        j                  ¬«      }t        j                  || j                  j                  d¬«      \  }}||j                  dd¬«      z  }|| j                  |   z  }|||fS )Nr¯   rî   )r�  r´   Trw  )r  r*  r(  r)  r   r   rõ   rk   rö   ÚtopkrD   Útop_k_expertsÚsumr+  )r‰   r‹   Úexpert_scoresÚrouter_probabilitiesÚtop_k_weightsÚtop_k_indexs         r]   r�   zGemma4TextRouter.forward”  sÂ   € ØŸ	™	 -Ó0ˆØ%¨¯
©
Ñ2°T×5JÑ5JÑJˆàŸ	™	 -Ó0ˆä!Ÿ}™}×4Ñ4°]ÈÔRW×R_ÑR_Ð4Ó`Ðô &+§Z¡ZØ Ø�k‰k×'Ñ'Øô&
Ñ"ˆ�{ð 	˜×*Ñ*¨r¸4Ð*Ó@Ñ@ˆð &¨×(=Ñ(=¸kÑ(JÑJˆà# ]°KÐ?Ð?r_   )
rd   re   rf   rB   r‚   rk   rl   rj   r�   r‘   r’   s   @r]   r&  r&  ‡  s>   ø„ ð
MÐ/õ 
Mð@ U§\¡\ð @°e¸E¿L¹LÈ%Ï,É,Ð<VÑ6W÷ @r_   r&  c                   ó0  ‡ — e Zd Zdeez  defˆ fd„Z	 	 	 	 	 	 ddej                  dej                  de	e
eej                  ej                  f   f   dz  dej                  d	ej                  dz  d
ej                  dz  dedz  dej                  fd„Zˆ xZS )ÚGemma4TextDecoderLayerrD   rM   c                 ó�  •— t         ‰| �  ||«       t        ||¬«      | _        t	        ||«      | _        | j                  dt        j                  d«      «       |j                  | _	        | j                  r t        |j                     | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        t%        | j                  |j&                  ¬«      | _        |j*                  | _        | j*                  r“t-        |«      | _        t1        |«      | _        t%        | j                  |j&                  ¬«      | _        t%        | j                  |j&                  ¬«      | _        t%        | j                  |j&                  ¬«      | _        y y )NrØ  Úlayer_scalarr?   Frz   ©r  )r�   r‚   r  rO  rí  rÙ  r†   rk   r\  Úhidden_size_per_layer_inputr   r$  r+  r   r„   rŸ   Úper_layer_input_gateÚper_layer_projectionr”   r  Úpost_per_layer_input_normÚenable_moe_blockr&  Úrouterr  ÚexpertsÚpost_feedforward_layernorm_1Úpost_feedforward_layernorm_2Úpre_feedforward_layernorm_2rÖ   s      €r]   r‚   zGemma4TextDecoderLayer.__init__­  sX  ø€ Ü‰Ñ˜ Ô+Ü,°FÀiÔPˆŒÜ  ¨Ó3ˆŒØ×Ñ˜^¬U¯Z©Z¸«]Ô;à+1×+MÑ+MˆÔ(Ø×+Ò+Ü  ×!9Ñ!9Ñ:ˆDŒKÜ(*¯	©	°$×2BÑ2BÀD×DdÑDdÐkpÔ(qˆDÔ%Ü(*¯	©	°$×2RÑ2RÐTX×TdÑTdÐkpÔ(qˆDÔ%Ü-:¸4×;KÑ;KÐQW×QdÑQdÔ-eˆDÔ*à &× 7Ñ 7ˆÔØ× Ò Ü*¨6Ó2ˆDŒKÜ,¨VÓ4ˆDŒLÜ0=¸d×>NÑ>NÐTZ×TgÑTgÔ0hˆDÔ-Ü0=¸d×>NÑ>NÐTZ×TgÑTgÔ0hˆDÔ-Ü/<¸T×=MÑ=MÐSY×SfÑSfÔ/gˆDÕ,ð !r_   Nr‹   Úper_layer_inputrb   rë   rF   rH   rG   rJ   c           
      ó&  — |}	| j                  |«      } | j                  d||||||dœ|¤Ž\  }}
| j                  |«      }|	|z   }|}	| j                  |«      }| j	                  |«      }| j
                  rš| j                  |«      }|	j                  d|	j                  d   «      }| j                  |«      \  }
}}| j                  |«      }| j                  |||«      }|j                  |	j                  «      }| j                  |«      }||z   }| j                  |«      }|	|z   }| j                  rP|}	| j                  |«      }| j!                  |«      }||z  }| j#                  |«      }| j%                  |«      }|	|z   }|| j&                  z  }|S )N)r‹   rë   rF   rb   rH   rG   r¯   rS   )rÛ  rO  rÜ  rÝ  rÙ  r=  r@  rÛ   rØ   r>  rB  r?  rA  rÞ  r9  r:  r+  r;  r<  r7  )r‰   r‹   rC  rb   rë   rF   rH   rG   rT  r3  rX   Úhidden_states_1Úhidden_states_flatr2  r3  Úhidden_states_2s                   r]   r�   zGemma4TextDecoderLayer.forwardÂ  s¾  € ð !ˆà×,Ñ,¨]Ó;ˆØ)˜4Ÿ>™>ð 
Ø'Ø 3Ø)Ø-Ø%Ø+ñ
ð ñ
Ñˆ�qð ×5Ñ5°mÓDˆØ  =Ñ0ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆà× Ò Ø"×?Ñ?ÀÓNˆOð "*×!1Ñ!1°"°h·n±nÀRÑ6HÓ!IÐØ,0¯K©KÐ8JÓ,KÑ)ˆAˆ}˜kØ"×>Ñ>Ð?QÓRˆOØ"Ÿl™l¨?¸KÈÓWˆOØ-×5Ñ5°h·n±nÓEˆOØ"×?Ñ?ÀÓPˆOð ,¨oÑ=ˆMà×7Ñ7¸ÓFˆØ  =Ñ0ˆà×+Ò+Ø$ˆHØ ×5Ñ5°mÓDˆMØ ŸK™K¨Ó6ˆMØ)¨OÑ;ˆMØ ×5Ñ5°mÓDˆMØ ×:Ñ:¸=ÓIˆMØ$ }Ñ4ˆMà˜×*Ñ*Ñ*ˆØÐr_   )NNNNNN)rd   re   rf   rB   rC   r�   r‚   rk   rl   rh   ri   rj   rÓ  r   r�   r‘   r’   s   @r]   r5  r5  ¬  sÑ   ø„ ðhÐ/Ð2DÑDð hÐQTõ hð0 )-ØPTØ,0Ø.2Ø04Ø(,ñ9à—|‘|ð9ð Ÿ™ð9ð ˜s E¨%¯,©,¸¿¹Ð*DÑ$EÐEÑFÈÑMð	9ð
 #Ÿ\™\ð9ð Ÿ™ tÑ+ð9ð ×&Ñ&¨Ñ-ð9ð  ™ð9ð 
�‰÷9r_   r5  c                   ó   — e Zd Zy)ÚGemma4TextScaledWordEmbeddingNr•   rS   r_   r]   rI  rI  þ  r–   r_   rI  c                   óJ   — e Zd Zg d¢ZdZdZ ej                  «       d„ «       Zy)ÚGemma4PreTrainedModel)r5  rÕ  rW  rK  )ÚimageÚtextÚvideoÚaudioNc                 óÖ	  — t        j                  |«       t        |t        «      r t	        j
                  |j                  «       y t        |t        «      r©d}d}|j                  dz  }t        j                  ||z  «      t        |dz
  d«      z  }|t        j                  t        j                  |«      | z  «      z  }t	        j                  |j                   |j#                  d«      j#                  d«      «       y t        |t$        «      rJt	        j&                  |j(                  |j*                  «       t	        j,                  |j.                  «       y t        |t0        «      rŸ|j2                  j5                  «       D ]�  \  }}d|i}	|dk(  r|j6                  |   dk(  rd	|	d
<    ||j8                  fi |	¤Ž\  }
}t	        j                  t;        ||› d�«      |
«       t	        j                  t;        ||› d�«      |
«       Œƒ y t        |t<        «      r„|j6                  dk7  rt>        |j6                     n|j@                  } ||j8                  «      \  }}t	        j                  |jB                  |«       t	        j                  |jD                  |«       y t        |tF        «      r+t	        j&                  |jH                  |jJ                  «       y t        |tL        «      r?t	        j
                  |jN                  «       t	        j
                  |jP                  «       y t        |tR        «      r[| j8                  jT                  }t	        jV                  |jX                  d|¬«       t	        jV                  |jZ                  d|¬«       y t        |t\        «      r t	        j
                  |j^                  «       y t        |t`        «      r³|jb                  r§t	        j&                  |jd                  tg        d«       «       t	        j&                  |jh                  tg        d«      «       t	        j&                  |jj                  tg        d«       «       t	        j&                  |jl                  tg        d«      «       y t        |tn        «      rV|j8                  jp                  r?t	        j,                  |jr                  «       t	        j
                  |jt                  «       y y y )Nr›   rœ   r*   r?   r   rø  rQ   rù  rú  rû  rü  rý  r÷  rd  )ÚmeanÚstdr}   );r   Ú_init_weightsr±  rW  ÚinitÚones_r]  r˜   rŸ   r¤   r¥   r¦   rk   r§   r¨   Úcopy_r™   r©   r¿   Ú	constant_rÂ   rÄ   Úzeros_rÕ   rô  r  Úitemsrö  rD   r¤  r   r   rª  r©  Úoriginal_inv_freqrI  Úembed_scaleÚscalar_embed_scaler&  r*  r+  r  Úinitializer_rangeÚnormal_Úgate_up_projr�  r5  r7  rv   rƒ   r|   rˆ   r~   r   r€   ÚGemma4VisionModelÚstandardizeÚstd_biasÚ	std_scale)r‰   Úmodulerª   r«   r¬   r­   r™   rø  r	  r
  r  rX   Úrope_fnÚbuffer_valuerR  s                  r]   rS  z#Gemma4PreTrainedModel._init_weights  s—  € ä×%Ñ% fÔ-Ü�fÔ7Ô8Ü�J‰J�v×6Ñ6Õ7Ü˜Ô @ÔAØˆMØ#ˆMØ#×/Ñ/°1Ñ4ˆNÜ&*§h¡h¨}¸}Ñ/LÓ&MÔPSÐTbÐefÑTfÐhiÓPjÑ&jÐ#Ø*¬U¯Y©Y´u·|±|ÀNÓ7SÐWnÐVnÑ7nÓ-oÑoˆNÜ�J‰J�v×,Ñ,¨n×.FÑ.FÀqÓ.I×.SÑ.SÐTUÓ.VÕWÜ˜Ô 4Ô5Ü�N‰N˜6Ÿ>™>¨6×+KÑ+KÔLÜ�K‰K˜×,Ñ,Õ-Ü˜Ô 9Ô:Ø,2×,@Ñ,@×,FÑ,FÓ,Hò ^Ñ(�
˜LØ'3°ZÐ&@Ð#ØÐ!1Ò1°f×6FÑ6FÀzÑ6RÐVdÒ6dØ:KÐ'¨Ñ7á#/°·±Ñ#UÐATÑ#UÑ �˜qÜ—
‘
œ7 6¨j¨\¸Ð+CÓDÀmÔTÜ—
‘
œ7 6¨j¨\Ð9KÐ+LÓMÈ}Õ]ñ^ô ˜Ô ;Ô<ð ×#Ñ# yÒ0ô $ F×$4Ñ$4Ò5à×;Ñ;ð ñ
 & f§m¡mÓ4‰OˆL˜!Ü�J‰J�v—‘¨Ô5Ü�J‰J�v×/Ñ/°Õ>Ü˜Ô =Ô>Ü�N‰N˜6×-Ñ-¨v×/HÑ/HÕIÜ˜Ô 0Ô1Ü�J‰J�v—|‘|Ô$Ü�J‰J�v×.Ñ.Õ/Ü˜Ô 1Ô2Ø—+‘+×/Ñ/ˆCÜ�L‰L˜×,Ñ,°3¸CÕ@Ü�L‰L˜×)Ñ)°¸Ö=Ü˜Ô 6Ô7Ü�J‰J�v×*Ñ*Õ+Ü˜Ô 5Ô6¸6×;UÒ;UÜ�N‰N˜6×+Ñ+¬e°E«l¨]Ô;Ü�N‰N˜6×+Ñ+¬U°5«\Ô:Ü�N‰N˜6×,Ñ,¬u°U«|¨mÔ<Ü�N‰N˜6×,Ñ,¬e°E«lÕ;Ü˜Ô 1Ô2°v·}±}×7PÒ7PÜ�K‰K˜Ÿ™Ô(Ü�J‰J�v×'Ñ'Õ(ð 8QÐ2r_   )	rd   re   rf   Ú_no_split_modulesÚinput_modalitiesÚ_can_record_outputsrk   r½   rS  rS   r_   r]   rK  rK    s2   „ òÐð ;ÐØÐà€U‡]�]ƒ_ñ2)ó ñ2)r_   rK  zAThe base Gemma 4 language model without a language modeling head.©Úcustom_introc                   óð  ‡ — e Zd ZU eed<    eed¬«      eedœZ	defˆ fd„Z
dej                  dz  dej                  dz  d	ej                  fd
„Z	 ddej                  dej                  dz  d	ej                  fd„Zeee	 	 	 	 	 	 	 ddej$                  dz  dej                  dz  dej$                  dz  dedz  dej(                  dz  dej                  dz  dedz  dee   d	efd„«       «       «       Zˆ xZS )ÚGemma4TextModelrD   r   )r  )Úrouter_logitsr‹   Ú
attentionsc           
      óÎ  •— t         ‰| �  |«       t        j                  t	        |j
                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |«      | _	        t        | j                  j                  «      | _        |j                  | _        | j                  rËt        |j                   |j
                  |j                  z  | j"                  |j                  dz  ¬«      | _        d| _        t        j(                  |j*                  |j
                  |j                  z  d¬«      | _        |j*                  dz  | _        t1        |j                  |j2                  ¬«      | _        g | _        t9        | j                  «      D ]K  \  }}|j:                  j<                  sŒ| j6                  j?                  dD �cg c]
  }d	|› d
|› �‘Œ c}«       ŒM y c c}w c c}w )Nrm  )r[  gÍ;fž æ?Frz   rÁ   r8  )rÏ   rÐ   rÌ  rÈ  zlayers.z.self_attn.) r�   r‚   r   ræ  r•  rã  r5  rç  rô  rå  r  rD   r  Úunique_layer_typesr9  rI  Úvocab_size_per_layer_inputÚpadding_idxÚembed_tokens_per_layerÚper_layer_input_scaler„   rŸ   Úper_layer_model_projectionÚ per_layer_model_projection_scaler”   r  Úper_layer_projection_normÚ"_keys_to_ignore_on_load_unexpectedÚ	enumeraterO  rò  Úextend)r‰   rD   rM   r¸  ÚlayerÚnamerŠ   s         €r]   r‚   zGemma4TextModel.__init__N  sš  ø€ Ü‰Ñ˜Ô Ü—m‘mÜHMÈf×NfÑNfÓHgÖh¸9Ô# F¨IÕ6Òhó
ˆŒô 4°FÓ;ˆŒÜ"% d§k¡k×&=Ñ&=Ó">ˆÔð
 ,2×+MÑ+MˆÔ(Ø×+Ò+Ü*GØ×1Ñ1Ø×(Ñ(¨6×+MÑ+MÑMØ× Ñ Ø"×>Ñ>ÀÑCô	+ˆDÔ'ð *3ˆDÔ&Ü.0¯i©iØ×"Ñ"Ø×(Ñ(¨6×+MÑ+MÑMØô/ˆDÔ+ð
 5;×4FÑ4FÈÑ4LˆDÔ1Ü-:¸6×;]Ñ;]Ðci×cvÑcvÔ-wˆDÔ*ð 35ˆÔ/Ü! $§+¡+Ó.ò 	‰HˆAˆuØ�‰×1Ó1Ø×7Ñ7×>Ñ>Ø@hÖi¸�w˜q˜c ¨T¨FÒ3Òiõñ	ùò7 iùò< js   ·GÇG"
Ú	input_idsNrE   rJ   c                 óŠ  — | j                   st        d| j                  › �«      ‚|€­t        j                  «       5  |dd…dd…ddd…f   | j
                  j                  dddd…dd…f   | j                  j                  dz  z  k(  j                  d¬«      j                  «       dd…df   }	 |j                  |j                  dd «      }	 ddd«        | j                  |«      j                  g |j                  ¢| j                  j                  ‘| j                   ‘­Ž S # t        $ r t        d«      ‚w xY w# 1 sw Y   ŒnxY w)a  Compute the token-identity component of Per-Layer Embeddings (PLE).

        Looks up `input_ids` in `embed_tokens_per_layer` (a scaled embedding that multiplies
        by `sqrt(hidden_size_per_layer_input)`) and reshapes the packed output from
        `[batch, seq, num_hidden_layers * hidden_size_per_layer_input]` to
        `[batch, seq, num_hidden_layers, hidden_size_per_layer_input]`.

        If only `inputs_embeds` is provided (no `input_ids`), reverses the main embedding
        to recover `input_ids` for the PLE lookup.
        z}Attempting to call get_per_layer_inputs() from a model initialized with a config that does not support per-layer embeddings. Nrm  r	   r³   r*   a)  It seems like you tried to call `forward` from `inputs_embeds` without providing `input_ids`, and that the `inputs_embeds` you provided do not exactly match the embedding weights. Since Gemma4 needs to reverse the embedding to compute another embedding, make sure you provide exact `inputs_embeds`)r9  ÚRuntimeErrorrD   rk   r½   Úembed_tokensr  rŸ   r  Únonzerorç   rØ   rt  rÛ   rã  )r‰   r~  rE   s      r]   Úget_per_layer_inputsz$Gemma4TextModel.get_per_layer_inputsr  sU  € ð ×/Ò/Üð*Ø*.¯+©+¨ð8óð ð ÐÜ—‘“ñ ð &¢aª¨D²! mÑ4Ø×,Ñ,×3Ñ3°D¸$ÂÂ1Ð4DÑEÈÏÉ×H_ÑH_ÐadÑHdÑdñe÷ ‘S˜Q�S“Zß‘W“Yšq !˜tñ%ð ðØ )§¡¨}×/BÑ/BÀ2ÀAÐ/FÓ G‘I÷ð$ >ˆt×*Ñ*¨9Ó5×=Ñ=ð 
Ø�_‰_ð
à�K‰K×)Ñ)ð
ð ×,Ñ,ò
ð 	
øô $ò Ü&ðróð ðú÷ð ús   »A1D9Â-D!Ä!D6Ä6D9Ä9EÚper_layer_inputsc                 óV  — | j                   st        d| j                  › �«      ‚| j                  |«      | j                  z  } |j
                  g |j                  dd ¢| j                  j                  ‘| j                   ‘­Ž }| j                  |«      }|€|S ||z   | j                  z  S )aŒ  Compute the context-aware component of PLE and combine with token-identity.

        Projects `inputs_embeds` through `per_layer_model_projection` (Linear), scales by
        `1/sqrt(hidden_size)`, reshapes to `[batch, seq, num_layers, ple_dim]`, and normalizes
        with `per_layer_projection_norm` (RMSNorm).

        If `per_layer_inputs` (the token-identity component from `get_per_layer_inputs()`)
        is provided, combines both: `(context_projection + token_identity) * (1/sqrt(2))`.
        If `per_layer_inputs` is None (e.g. for multimodal inputs where input_ids are not
        available), returns just the context projection.
        z�Attempting to call project_per_layer_inputs() from a model initialized with a config that does not support per-layer embeddings. Nr¯   )
r9  r€  rD   rv  rw  rÛ   rØ   rã  rx  ru  )r‰   rE   r„  r;  s       r]   Úproject_per_layer_inputsz(Gemma4TextModel.project_per_layer_inputsž  sÏ   € ð  ×/Ò/Üð2Ø26·+±+°ð@óð ð
  $×>Ñ>¸}ÓMÐPT×PuÑPuÑuÐØ;Ð3×;Ñ;ð  
Ø× Ñ   "Ð%ð 
à�K‰K×)Ñ)ð 
ð ×,Ñ,ò 
Ðð
  $×=Ñ=Ð>RÓSÐàÐ#Ø'Ð'à$Ð'7Ñ7¸4×;UÑ;UÑUÐUr_   rF   rH   rG   Ú	use_cacherT  c           
      ó<  — |du |duz  rt        d«      ‚|�|�t        d«      ‚|�| j                  |«      }| j                  r&|€| j                  ||«      }| j	                  ||«      }|r|€t        | j                  ¬«      }|€V|�|j                  «       nd}	t        j                  |j                  d   |j                  ¬«      |	z   }|j                  d«      }t        |x}
t        «      s)| j                  ||||dœ}t        di |¤Žt!        di |¤Žd	œ}
|}i }| j"                  D ]  }| j%                  |||«      ||<   Œ |j'                  d
t)        «       «      }t+        | j,                  d| j                  j.                   «      D ]\  \  }}|�|dd…dd…|dd…f   nd} |||f||| j                  j0                  |      |
| j                  j0                  |      ||dœ|¤Ž}Œ^ | j3                  |«      }t5        |||j7                  dd«      r|¬«      S d¬«      S )ak  
        per_layer_inputs (`torch.Tensor`, *optional*):
            Pre-computed per-layer input text embeddings of shape `(batch_size, sequence_length, num_hidden_layers,
            hidden_size_per_layer_input)`. When provided, these are used directly instead of being computed from `input_ids`
            via `get_per_layer_inputs()` in the text model. If calling the `forward` with `inputs_embeds` instead of `input_ids`,
            you should probably precompute them and forward them along `inputs_embeds`, otherwise recomputing them needs
            to reverse the main embedding, which is expensive.
        Nú:You must specify exactly one of input_ids or inputs_embedsú<You cannot specify per_layer_inputs if input_ids is provided)rD   r   r?   r°   rL   rP   rb   )rb   rë   rF   rH   rG   Úreturn_shared_kv_statesF)rê  rG   rb   rS   )r{  r�  r9  rƒ  r†  r   rD   Úget_seq_lengthrk   r¨   rØ   r±   r©   r±  rh   r   r   rq  rå  Úpopr   rz  rç  rã  r  r  rq   Úget)r‰   r~  rF   rH   rG   rE   r„  r‡  rT  Úpast_seen_tokensÚcausal_mask_mappingrU   r‹   rë   rø  rb   r¸  rë  rC  s                      r]   r�   zGemma4TextModel.forwardÁ  s}  € ð, ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ð%5Ð%AÜÐ[Ó\Ð\àÐ Ø ×-Ñ-¨iÓ8ˆMà×+Ò+ØÐ'Ø#'×#<Ñ#<¸YÈÓ#VÐ Ø#×<Ñ<¸]ÐL\Ó]Ðá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-ÄÔFð Ÿ+™+Ø!.Ø"0Ø#2Ø ,ñˆKô #5Ñ"C°{Ñ"CÜ%FÑ%UÈÑ%Uñ#Ðð &ˆØ ÐØ×1Ñ1ò 	gˆJØ.2¯o©o¸mÈ\Ð[eÓ.fÐ 
Ò+ð	gð "Ÿ:™:Ð&8¼(»*ÓEÐô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÓ Wò 	ÑˆAˆ}Ø>NÐ>ZÐ.ªq²!°Qº¨zÒ:Ð`dˆOá)ØØð	ð "2Ø$7¸¿¹×8OÑ8OÐPQÑ8RÑ$SØ2°4·;±;×3JÑ3JÈ1Ñ3MÑNØ)Ø /ñ	ð ñ	‰Mð	ð Ÿ	™	 -Ó0ˆä,Ø+Ø+Ø17·±Ð<UÐW\Ô1]Ð-ô
ð 	
ð dhô
ð 	
r_   r�   )NNNNNNN)rd   re   rf   rB   rm   r(   r&  r5  r  ri  r‚   rk   rl   rƒ  r†  r&   r)   r    rÓ  r   rß  Úboolr   r   rq   r�   r‘   r’   s   @r]   rm  rm  E  sƒ  ø… àÓá'Ð(8ÀÔBØ/Ø)ñÐð"Ð/õ "ðH*
¨e¯l©l¸TÑ.Að *
ÐRW×R^ÑR^ÐaeÑReð *
Ðjo×jvÑjvó *
ð^ 15ñ!Và—|‘|ð!Vð  Ÿ,™,¨Ñ-ð!Vð 
�‰ó	!VðF  ØØð .2Ø.2Ø04Ø(,Ø26Ø04Ø!%ñY
à×#Ñ# dÑ*ðY
ð Ÿ™ tÑ+ðY
ð ×&Ñ&¨Ñ-ð	Y
ð
  ™ðY
ð ×(Ñ(¨4Ñ/ðY
ð  Ÿ,™,¨Ñ-ðY
ð ˜$‘;ðY
ð Ð+Ñ,ðY
ð 
'òY
ó ó ó  ôY
r_   rm  z>The base Gemma 4 language model with a language modeling head.c                   ó4  — e Zd ZdZee	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de	dz  dej                  dz  dej                  dz  d	edz  d
eej                  z  dej                  dz  dee   defd„«       «       Zy)ÚGemma4ForCausalLMÚmodelNr~  rF   rH   rG   rE   Úlabelsr‡  Úlogits_to_keepr„  rT  rJ   c
                 ó4  —  | j                   d||||||	|dœ|
¤Ž}|j                  }t        |t        «      rt	        | d«      n|}| j                  |dd…|dd…f   «      }| j                  j                  �G|| j                  j                  z  }t        j                  |«      }|| j                  j                  z  }d}|� | j                  ||| j                  fi |
¤Ž}t        |||j                  |j                  |j                  |j                   ¬«      S )aí  
        per_layer_inputs (`torch.Tensor`, *optional*):
            Pre-computed per-layer input text embeddings of shape `(batch_size, sequence_length, num_hidden_layers,
            hidden_size_per_layer_input)`. When provided, these are used directly instead of being computed from `input_ids`
            via `get_per_layer_inputs()` in the text model. If calling the `forward` with `inputs_embeds` instead of `input_ids`,
            you should probably precompute them and forward them along `inputs_embeds`, otherwise recomputing them needs
            to reverse the main embedding, which is expensive.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Gemma4ForCausalLM

        >>> model = Gemma4ForCausalLM.from_pretrained("google/gemma-4-E2B-it")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-4-E2B-it")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```)r~  rF   rH   rG   rE   r„  r‡  N)ÚlossÚlogitsrG   r‹   ro  rb   rS   )r”  rê  r±  r�   ÚsliceÚlm_headrD   Úfinal_logit_softcappingrk   rñ   Úloss_functionÚ
vocab_sizero   rG   r‹   ro  rb   )r‰   r~  rF   rH   rG   rE   r•  r‡  r–  r„  rT  Úoutputsr‹   Úslice_indicesr™  r˜  s                   r]   r�   zGemma4ForCausalLM.forward$  s"  € ðP 2<°·±ð 	2
ØØ)Ø%Ø+Ø'Ø-Øñ	2
ð ñ	2
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆØ�;‰;×.Ñ.Ð:Ø˜dŸk™k×AÑAÑAˆFÜ—Z‘Z Ó'ˆFØ˜dŸk™k×AÑAÑAˆFàˆØÐØ%�4×%Ñ% f¨f°d·o±oÑPÈÑPˆDä+ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô
ð 	
r_   )	NNNNNNNr   N)rd   re   rf   Úbase_model_prefixr!   r    rk   rÓ  rl   r   rß  r‘  r�   r   r   ro   r�   rS   r_   r]   r“  r“     s  „ àÐàØð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.Ø04ñE
à×#Ñ# dÑ*ðE
ð Ÿ™ tÑ+ðE
ð ×&Ñ&¨Ñ-ð	E
ð
  ™ðE
ð ×(Ñ(¨4Ñ/ðE
ð × Ñ  4Ñ'ðE
ð ˜$‘;ðE
ð ˜eŸl™lÑ*ðE
ð  Ÿ,™,¨Ñ-ðE
ð Ð+Ñ,ðE
ð 
&òE
ó ó ñE
r_   r“  c                   ó,  ‡ — e Zd ZU dZeed<   dZdZee	dœZ
defˆ fd„Zdej                  dej                  fd	„Zee ed
¬«      	 ddej                  dej                  dz  dee   deej                  ej*                  f   fd„«       «       «       Zˆ xZS )ÚGemma4AudioModelznAn audio encoder based on the [Universal Speech Model](https://huggingface.co/papers/2303.01037) architecture.rD   r   zmodel.audio_tower©r‹   ro  c           	      óˆ  •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        j                  |j                  |j                  d¬«      | _        | j#                  «        y c c}w )NTrz   )r�   r‚   rD   r  Úsubsample_conv_projectionr˜   Úrel_pos_encr   ræ  r•  rã  rK  rç  r„   rŸ   Úoutput_proj_dimsÚoutput_projÚ	post_initrÖ   s      €r]   r‚   zGemma4AudioModel.__init__y  s–   ø€ Ü‰Ñ˜Ô ØˆŒä)KÈFÓ)SˆÔ&Ü;¸FÓCˆÔÜ—m‘mÜBGÈ×H`ÑH`ÓBaÖb°YÔ˜f iÕ0Òbó
ˆŒô Ÿ9™9 V×%7Ñ%7¸×9PÑ9PÐW[Ô\ˆÔà�‰Õùò	 cs   ÁB?Úmask_4drJ   c                 ó€  — |j                   \  }}}}|j                  }| j                  j                  }| j                  j                  dz
  }| j                  j
                  }||z   dz
  |z  }	|	|z  }
|
|z
  }t        j                  |d|d|fd¬«      }|j                  |d|	||
«      }t        j                  |||fd¬«      }t        j                  |	|¬«      |z  }t        j                  ||z   |z   |¬«      }|dd…df   |ddd…f   z   }|dddd…ddd…f   j                  |dd|d«      }|j                  d|«      S )zÜ
        Convert a standard 4D attention mask `[batch_size, 1, seq_len, seq_len]` to the 5D blocked format
        `[batch_size, 1, num_blocks, chunk_size, context_size]` expected by the chunked local attention,
        r?   r   F)Úvaluer°   Nr¯   )rØ   r±   rD   r    r¡   r¢   rÙ   rÚ   rÛ   rk   r¨   r°  Úgather)r‰   r«  rÝ   rX   rÞ   r±   rË   rÌ   rÍ   rß   Úpadded_seq_lenÚ
pad_amountÚmask_5dÚblock_startsÚoffsetsÚ
kv_indicess                   r]   Ú_convert_4d_mask_to_blocked_5dz/Gemma4AudioModel._convert_4d_mask_to_blocked_5d†  sN  € ð
 %,§M¡MÑ!ˆ
�A�w Ø—‘ˆà—[‘[×5Ñ5ˆ
ØŸ;™;×=Ñ=ÀÑAÐØ!Ÿ[™[×@Ñ@Ðà 
Ñ*¨QÑ.°:Ñ=ˆ
Ø# jÑ0ˆØ# gÑ-ˆ
ä—%‘%˜ ! Z°°JÐ!?ÀuÔMˆØ—/‘/ *¨a°¸ZÈÓXˆÜ—%‘%˜Ð"2Ð4FÐ!GÈuÔUˆä—|‘| J°vÔ>ÀÑKˆÜ—,‘,˜zÐ,<Ñ<Ð?QÑQÐZ`ÔaˆØ!¢! T 'Ñ*¨W°Tº1°WÑ-=Ñ=ˆ
Ø  dªA¨t²QÐ 6Ñ7×>Ñ>¸zÈ1ÈbÐR\Ð^`Óaˆ
à�~‰~˜b *Ó-Ð-r_   z&Encodes audio features to soft tokens.rj  NrF   rT  c           	      ó¶  — | j                  ||«      \  }}| j                  |«      }t        | j                  ||t	        | j                  j
                  dz
  | j                  j                  f«      ¬«      }|�| j                  |«      }| j                  d | j                  j                   D ]  } ||f||dœ|¤Ž}Œ | j                  |«      }t        ||¬«      S )Nr?   )rD   rE   rF   rO   )rF   rë   )rê  rF   )r¦  r§  r   rD   r>   r¡   r¢   rµ  rç  rã  r©  rs   )r‰   r   rF   rT  r‹   Úoutput_maskrë   Úencoder_layers           r]   r�   zGemma4AudioModel.forward¡  sì   € ð &*×%CÑ%CÀNÐTbÓ%cÑ"ˆ�{Ø"×.Ñ.¨}Ó=Ðä2Ø—;‘;Ø'Ø&Ü:Ø—‘×3Ñ3°aÑ7¸¿¹×9\Ñ9\Ð]óô	
ˆð Ð%Ø!×@Ñ@ÀÓPˆNà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÐIò 	ˆMÙ)Øðà-Ø$7ñð ñ	‰Mð	ð ×(Ñ(¨Ó7ˆÜ%¸ÐVaÔbÐbr_   r�   )rd   re   rf   rg   r@   rm   Úmain_input_namer¡  rK  r¿   ri  r‚   rk   rl   rµ  r&   r)   r    r   r   rj   rt   r�   r‘   r’   s   @r]   r£  r£  n  sÐ   ø… ÙxàÓØ&€OØ+Ðà)Ø*ñÐð
Ð0õ ð.°e·l±lð .ÀuÇ|Á|ó .ð6  ØÙÐ!IÔJð /3ñcàŸ™ðcð Ÿ™ tÑ+ðcð Ð+Ñ,ð	cð
 
ˆu�|‰|˜U×-Ñ-Ð-Ñ	.òcó Kó ó  ôcr_   r£  c                   ó¨   ‡ — e Zd ZdZeZeedœZdefˆ fd„Z	e
e ed¬«      dej                  dej                  d	ee   d
efd„«       «       «       Zˆ xZS )r`  zThe Gemma 4 Vision Encoder.r¤  rD   c                 ó´  •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        t        |«      | _        | j                  j                  rr| j                  dt        j                  | j                  j                  «      «       | j                  dt        j                  | j                  j                  «      «       | j                  «        y )Nrb  rc  )r�   r‚   rW  Úpatch_embedderrá  Úencoderrq  ÚpoolerrD   ra  r†   rk   ÚemptyrŸ   rª  r/  s     €r]   r‚   zGemma4VisionModel.__init__Í  s’   ø€ Ü‰Ñ˜Ô Ü7¸Ó?ˆÔÜ*¨6Ó2ˆŒÜ(¨Ó0ˆŒà�;‰;×"Ò"Ø× Ñ  ¬U¯[©[¸¿¹×9PÑ9PÓ-QÔRØ× Ñ  ¬e¯k©k¸$¿+¹+×:QÑ:QÓ.RÔSà�‰Õr_   z1Encodes image pixels to soft tokens from patches.rj  rk  r^  rT  rJ   c                 ó  — | j                   j                  }|j                  d   ||z  z  }|dk(  j                  d¬«      }| j	                  |||«      } | j
                  d|| |dœ|¤Ž}| j                  |j                  |||¬«      \  }	}
|	|
   }	| j                   j                  r8|	| j                  j                  «       z
  | j                  j                  «       z  }	|	j                  |j                  «      }	t        |	¬«      S )aà  
        pixel_values (`torch.FloatTensor` or `list[torch.FloatTensor]`):
            The images to encode. Either a single `[batch, channels, height, width]` tensor
            (all images same size) or a list of `[1, channels, height, width]` tensors (different sizes).
        pixel_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`):
            The patch positions as (x, y) coordinates in the image. Padding patches are indicated by (-1, -1).
        éþÿÿÿr¯   r³   )rE   rF   r^  )r‹   r^  r_  rˆ  ré  rS   )rD   Úpooling_kernel_sizerØ   r  r¼  r½  r¾  rê  ra  rb  rˆ   rc  r·   r¶   r   )r‰   rk  r^  rT  rÂ  rˆ  r_  rE   r†  r‹   Úpooler_masks              r]   r�   zGemma4VisionModel.forwardÙ  s  € ð  #Ÿk™k×=Ñ=ÐØ$×*Ñ*¨2Ñ.Ð3FÐI\Ñ3\Ñ]ˆà/°2Ñ5×:Ñ:¸rÐ:ÓBÐØ×+Ñ+¨LÐ:LÐN_Ó`ˆØ�—‘ð 
Ø'Ø-Ð-Ø1ñ
ð ñ	
ˆð &*§[¡[Ø ×2Ñ2Ø1Ø/Ø'ð	 &1ó &
Ñ"ˆ�{ð & kÑ2ˆð �;‰;×"Ò"Ø*¨T¯]©]×-@Ñ-@Ó-BÑBÀdÇnÁn×FZÑFZÓF\Ñ\ˆMØ%×(Ñ(¨×)<Ñ)<Ó=ˆä&¸ÔGÐGr_   )rd   re   rf   rg   rC   rD   rÕ  rÀ  ri  r‚   r&   r)   r    rk   rß  rÓ  r   r   r   r�   r‘   r’   s   @r]   r`  r`  Ä  s�   ø„ Ù%à€Fà1Ø+ñÐð

Ð1õ 
ð  ØÙÐ!TÔUð)Hà×'Ñ'ð)Hð "×,Ñ,ð)Hð Ð+Ñ,ð	)Hð
 
!ò)Hó Vó ó  ô)Hr_   r`  c                   óf   ‡ — e Zd Zdeez  defˆ fd„Zdej                  dej                  fd„Z	ˆ xZ
S )ÚGemma4MultimodalEmbedderÚmultimodal_configÚtext_configc                 óÄ   •— t         ‰| �  ||«       | `| `| `| `| `| `t        |d|j                  «      | _
        t        | j                  | j                  d¬«      | _        y )Nr¨  Fr@  )r�   r‚   re  Úhard_embedding_normÚsoft_embedding_normÚvocab_offsetrž  Úembedding_post_projection_normr¤  rŸ   Úmultimodal_hidden_sizer”   r  Úembedding_pre_projection_norm)r‰   rÆ  rÇ  rŠ   s      €r]   r‚   z!Gemma4MultimodalEmbedder.__init__	  sp   ø€ ô 	‰ÑÐ*¨KÔ8ØˆNØÐ$ØÐ$ØÐØˆOØÐ/ä&-Ð.?ÐASÐUf×UrÑUrÓ&sˆÔ#Ü-:¸4×;VÑ;VÐ\`×\dÑ\dÐqvÔ-wˆÕ*r_   rE   rJ   c                 óF   — | j                  |«      }| j                  |«      S )a:  Embeds token ids or soft tokens for multimodal content into language model space.
        Args:
            inputs_embeds: A torch.Tensor containing the soft tokens to embed.
        Returns:
            A torch.Tensor of embeddings with shape `[batch_size, seq_len, self.config.text_config.hidden_size]`.
        )rÎ  Úembedding_projection)r‰   rE   Úembs_normeds      r]   r�   z Gemma4MultimodalEmbedder.forward  s%   € ð ×8Ñ8¸ÓGˆØ×(Ñ(¨Ó5Ð5r_   )rd   re   rf   r@   rC   rB   r‚   rk   rl   r�   r‘   r’   s   @r]   rÅ  rÅ    s>   ø„ ðxà,Ð/AÑAðxð &õxð$6 U§\¡\ð 6°e·l±l÷ 6r_   rÅ  Útoken_type_idsÚimage_group_idsc           
      óV   ‡— | €ydt         dt         dt         dt         dt        f
ˆfd„}|S )z•
    This function adds the correct offsets to the `q_idx` and `kv_idx` as the torch API can only accept lengths,
    not start and end indices.
    NÚ	batch_idxÚhead_idxÚq_idxÚkv_idxrJ   c                 ó  •— ‰	j                   d   }|j                  |dz
  ¬«      }|j                  |dz
  ¬«      }‰	| |f   }‰	| |f   }t        j                  ||k  |d«      }t        j                  ||k  |d«      }||k(  |dk\  z  S )Nr¯   r?   )r¦   r   )rØ   rŽ   rk   rf  )
rÕ  rÖ  r×  rØ  r÷   Úq_idx_clampedÚkv_idx_clampedÚq_groupÚkv_grouprÓ  s
            €r]   Ú
inner_maskz0token_type_ids_mask_function.<locals>.inner_mask3  sŸ   ø€ Ø$×*Ñ*¨2Ñ.ˆ
ð Ÿ™¨
°Q©˜Ó7ˆØŸ™¨*°q©.˜Ó9ˆð " )¨]Ð":Ñ;ˆØ" 9¨nÐ#<Ñ=ˆÜ—+‘+˜e jÑ0°'¸2Ó>ˆÜ—;‘;˜v¨
Ñ2°H¸bÓAˆØ˜8Ñ#¨°1©Ñ5Ð5r_   )r�   r‘  )rÒ  rÓ  rÞ  s    ` r]   Útoken_type_ids_mask_functionrß  '  s>   ø€ ð ÐØð6œcð 6¬Sð 6¼ð 6Äcð 6Ìdõ 6ð Ðr_   Úmm_token_type_idsr±   c                 ó  — | j                  |«      } | dk(  | dk(  z  }t        j                  |dd¬«      }d|d<   || z  }t        j                  |j	                  «       d¬«      dz
  }t        j
                  ||d«      }|S )Nr?   r*   r¯   )ÚshiftsÚdimsFrb  r³   )r·   rk   ÚrollÚcumsumr�   rf  )rà  r±   Ú	is_visionÚis_prev_visionÚnew_vision_startsÚvision_group_idsrI   s          r]   Úget_block_sequence_ids_for_maskrê  D  s‹   € Ø)×,Ñ,¨VÓ4Ðà" aÑ'Ð,=ÀÑ,BÑC€IÜ—Z‘Z 	°!¸"Ô=€NØ"€N�6ÑØ! ^ OÑ3ÐÜ—|‘|Ð$5×$9Ñ$9Ó$;ÀÔCÀaÑGÐÜŸ™ YÐ0@À"ÓEÐØÐr_   zŒ
    The base Gemma 4 model comprising a vision backbone, an audio backbone, and a language model without a
    language modeling head.
    c            $       óæ  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Ze ed¬«      	 dde	j                  d	e	j                  dz  d
ee   defd„«       «       Ze ed¬«      	 dde	j                  de	j                  dz  d
ee   defd„«       «       Z	 	 d de	j                  dz  de	j                  dz  dee	j$                  e	j$                  e	j$                  f   fd„Zeee	 	 	 	 	 	 	 	 	 	 	 	 	 	 d!de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j*                  dz  de	j*                  dz  de	j                  dz  dedz  de	j                  dz  de	j                  dz  dedz  d	e	j                  dz  de	j                  dz  de	j*                  dz  d
ee   def d„«       «       «       Ze ed¬«      de	j*                  de	j*                  d
ee   deez  fd„«       «       Zˆ xZS )"ÚGemma4ModelrD   c                 óÄ  •— t         ‰| �  |«       |j                  �t        j                  |j                  «      nd | _        |j                  � t        |j                  |j                  «      nd | _        |j                  �t        j                  |j                  «      nd | _
        |j                  �&t        |j                  |j                  «      | _        y d | _        y r�   )r�   r‚   Úvision_configr+   Úfrom_configÚvision_towerrÅ  rÇ  Úembed_visionÚaudio_configÚaudio_towerÚembed_audior/  s     €r]   r‚   zGemma4Model.__init__W  sÎ   ø€ Ü‰Ñ˜Ô ØKQ×K_ÑK_ÐKkœI×1Ñ1°&×2FÑ2FÔGÐquˆÔð ×#Ñ#Ð/ô % V×%9Ñ%9¸6×;MÑ;MÔNàð 	Ôð
 JP×I\ÑI\ÐIhœ9×0Ñ0°×1DÑ1DÔEÐnrˆÔð ×"Ñ"Ð.ô % V×%8Ñ%8¸&×:LÑ:LÓMð 	Õð ð 	Õr_   c                 ó.   — | j                   j                  S r�   ©Úlanguage_modelrt  ©r‰   s    r]   Úget_per_layer_input_embeddingsz*Gemma4Model.get_per_layer_input_embeddingsf  s   € Ø×"Ñ"×9Ñ9Ð9r_   c                 ó&   — || j                   _        y r�   rö  ©r‰   r­  s     r]   Úset_per_layer_input_embeddingsz*Gemma4Model.set_per_layer_input_embeddingsi  s   € Ø5:ˆ×ÑÕ2r_   zOProjects the last hidden state from the vision model into language model space.rj  Nrk  Úimage_position_idsrT  rJ   c                 óv   —  | j                   d||dœ|¤Ž}|j                  }| j                  |¬«      |_        |S )zÝ
        image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
            The patch positions as (x, y) coordinates in the image. Padding patches are indicated by (-1, -1).
        ©rk  r^  ©rE   rS   )rð  rê  rñ  Úpooler_output)r‰   rk  rý  rT  Úvision_outputsrê  s         r]   Úget_image_featureszGemma4Model.get_image_featuresl  sV   € ð +˜×*Ñ*ð 
Ø%Ø1ñ
ð ñ
ˆð
 +×<Ñ<ÐØ'+×'8Ñ'8ÐGXÐ'8Ó'YˆÔ$ØÐr_   zQProjects the last hidden state from the vision encoder into language model space.Úpixel_values_videosÚvideo_position_idsc                 ó¾   — |j                  dd«      }|j                  dd«      } | j                  d||dœ|¤Ž}|j                  }| j                  |¬«      |_        |S )a9  
        video_position_ids (`torch.LongTensor` of shape `(num_videos, num_frames, max_patches, 2)`, *optional*):
            2D patch position coordinates from the video processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        r   r?   rÿ  r   rS   )Úflattenrð  rê  rñ  r  )r‰   r  r  rT  r  rê  s         r]   Úget_video_featureszGemma4Model.get_video_features�  s|   € ð 2×9Ñ9¸!¸QÓ?ÐØ/×7Ñ7¸¸1Ó=ÐØ*˜×*Ñ*ð 
Ø,Ø1ñ
ð ñ
ˆð
 +×<Ñ<ÐØ'+×'8Ñ'8ÐGXÐ'8Ó'YˆÔ$ØÐr_   r~  rE   c                 ó&  — |�M|| j                   j                  k(  }|| j                   j                  k(  }|| j                   j                  k(  }�n>| | j	                  «       t        j                  | j                   j                  t
        j                  |j                  ¬«      «      k(  j                  d«      }| | j	                  «       t        j                  | j                   j                  t
        j                  |j                  ¬«      «      k(  j                  d«      }| | j	                  «       t        j                  | j                   j                  t
        j                  |j                  ¬«      «      k(  j                  d«      }|||fS )a‹  
        Obtains mask for multimodal placeholders (replaced by soft tokens) and hard text tokens.

        Masks will be obtained from `mm_token_type_ids`, `input_ids`, or `inputs_embeds` as available and in that
        precedence order. If passing `input_ids` or `inputs_embeds`, the image mask will be derived using
        `config.image_token_id`. Same goes for audio and video masks

        Args:
            input_ids: A tensor containing the hard token IDs from the text tokenizer.
            inputs_embeds: A tensor containing the embeddings for all hard text tokens.

        Returns:
            image_mask, video_mask, audio_mask
        )r¶   r±   r¯   )
rD   Úimage_token_idÚvideo_token_idÚaudio_token_idÚget_input_embeddingsrk   r‡   r~  r±   r  )r‰   r~  rE   Úspecial_image_maskÚspecial_video_maskÚspecial_audio_masks         r]   Úget_placeholder_maskz Gemma4Model.get_placeholder_mask™  sP  € ð& Ð Ø!*¨d¯k©k×.HÑ.HÑ!HÐØ!*¨d¯k©k×.HÑ.HÑ!HÐØ!*¨d¯k©k×.HÑ.HÑ!HÒð Ø.�4×,Ñ,Ó.Ü—L‘L §¡×!;Ñ!;Ä5Ç:Á:ÐVc×VjÑVjÔkóñ÷ ‰c�"‹gð ð Ø.�4×,Ñ,Ó.Ü—L‘L §¡×!;Ñ!;Ä5Ç:Á:ÐVc×VjÑVjÔkóñ÷ ‰c�"‹gð ð Ø.�4×,Ñ,Ó.Ü—L‘L §¡×!;Ñ!;Ä5Ç:Á:ÐVc×VjÑVjÔkóñ÷ ‰c�"‹gð ð "Ð#5Ð7IÐIÐIr_   r   rF   r!  rH   rG   rà  r‡  r„  c                 ó&  — |du |
duz  rt        d«      ‚|�|�t        d«      ‚| j                  ||
«      \  }}}||z  |z  }d}|
€[|j                  «       }t        j                  || j
                  j                  j                  |«      } | j                  «       |«      }
|€Ë| j
                  j                  «       j                  r§| j                  j                  j                  | j
                  j                  j                  dd…f   }|j                  |
j                  «      }t        j                  |d   |j!                  ddd«      |
«      }| j                  j#                  ||«      }|��| j%                  ||d¬«      j&                  }|j                  |
j                  |
j(                  «      }|j+                  «       }|j-                  d«      j/                  |
«      j                  |
j                  «      }t1        |
|   j3                  «       |j3                  «       k(  d	|› d
|j4                  d   › �«       |
j7                  |j                  |
j                  «      |j                  |
j                  «      «      }
|��| j9                  ||d¬«      j&                  }|j                  |
j                  |
j(                  «      }|j+                  «       }|j-                  d«      j/                  |
«      j                  |
j                  «      }t1        |
|   j3                  «       |j3                  «       k(  d|› d
|j4                  d   › �«       |
j7                  |j                  |
j                  «      |j                  |
j                  «      «      }
|��+|��(| j;                  ||d¬«      }|j&                  }|j<                  }||j                  |j                  «         }|j+                  «       }|j-                  d«      j/                  |
«      j                  |
j                  «      }t1        |
|   j3                  «       |j3                  «       k(  d|› d
|j4                  d   |j4                  d   z  › �«       |
j7                  |j                  |
j                  «      |j                  |
j                  «      «      }
|€V|�|j?                  «       nd}t        j@                  |
j4                  d   |
j                  ¬«      |z   }|j-                  d«      }tC        |x} tD        «      s}| j
                  j                  «       |
|||dœ}!| j
                  j                  «       }"|"jF                  dk(  }#|#r'|	�%tI        |	|
j                  ¬«      }$tK        dd|$i|!¤Ž} ntM        di |!¤Ž}  | j                  d|| |||
|ddœ|¤Ž}%tO        |%jP                  |%jR                  |%jT                  |%jV                  |�nd|�nd|%jX                  ¬«      S )áK  
        input_features_mask (`torch.FloatTensor]` of shape `(num_images, seq_length)`):
            The attention mask for the input audio.
        image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
            2D patch position coordinates from the image processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        video_position_ids (`torch.LongTensor` of shape `(num_videos, num_frames, max_patches, 2)`, *optional*):
            2D patch position coordinates from the video processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        per_layer_inputs (`torch.Tensor`, *optional*):
            Pre-computed per-layer input text embeddings of shape `(batch_size, sequence_length, num_hidden_layers,
            hidden_size_per_layer_input)`. When provided, these are used directly instead of being computed from `input_ids`
            via `get_per_layer_inputs()` in the text model. If calling the `forward` with `inputs_embeds` instead of `input_ids`,
            you should probably precompute them and forward them along `inputs_embeds`, otherwise recomputing them needs
            to reverse the main embedding, which is expensive.
        Nr‰  rŠ  r²   r?   r¯   T)Úreturn_dictz6Image features and image tokens do not match, tokens: z, features: r   z6Video features and video tokens do not match, tokens: z6Audio features and audio tokens do not match, tokens: r°   rL   ÚvisionrI   )r„  rF   rH   rG   rE   r‡  r  )rê  rG   r‹   ro  Úimage_hidden_statesÚaudio_hidden_statesrb   rS   )-r{  r  r  rk   rf  rD   rÇ  Úpad_token_idr  Úget_text_configr9  r÷  r�  r  r·   r±   rç   rƒ  r  r  r¶   r/  r©   Ú	expand_asr$   ÚnumelrØ   Úmasked_scatterr  Úget_audio_featuresrF   rŒ  r¨   r±  rh   r  rê  r^   r   ra   rê  rG   r‹   ro  rb   )&r‰   r~  rk  r  r   rF   r!  rH   rG   rà  rE   r‡  rý  r  r„  rT  Ú
image_maskÚ
video_maskÚ
audio_maskÚmultimodal_maskÚllm_input_idsÚpad_embeddingÚllm_inputs_embedsÚimage_featuresÚn_image_tokensÚvideo_featuresÚn_video_tokensÚaudio_outputÚaudio_featuresÚaudio_mask_from_encoderÚn_audio_tokensr�  r�  rU   rÇ  Ú	use_bidirrI   rŸ  s&                                         r]   r�   zGemma4Model.forwardÆ  s  € ðJ ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ð%5Ð%AÜÐ[Ó\Ð\à-1×-FÑ-FÀyÐR_Ó-`Ñ*ˆ
�J 
Ø$ zÑ1°JÑ>ˆð ˆØÐ Ø%ŸO™OÓ-ˆMÜ!ŸK™K¨¸¿¹×9PÑ9P×9]Ñ9]Ð_lÓmˆMØ7˜D×5Ñ5Ó7¸ÓFˆMàÐ#¨¯©×(CÑ(CÓ(E×(aÒ(aØ ×/Ñ/×<Ñ<×CÑCÀDÇKÁK×D[ÑD[×DhÑDhÒjkÐDkÑlˆMØ-×0Ñ0°×1EÑ1EÓFˆOÜ %§¡¨O¸IÑ,FÈ×HZÑHZÐ[\Ð^_ÐacÓHdÐfsÓ tÐØ#×2Ñ2×GÑGÈÐWhÓiÐð Ñ#Ø!×4Ñ4°\ÐCUÐcgÐ4Óh×vÑvˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNð (Ÿ^™^Ó-ˆNØ#×-Ñ-¨bÓ1×;Ñ;¸MÓJ×MÑMÈm×NbÑNbÓcˆJÜ"Ø˜jÑ)×/Ñ/Ó1°^×5IÑ5IÓ5KÑKØHÈÐHXð YØ"×(Ñ(¨Ñ+Ð,ð.ôð *×8Ñ8Ø—‘˜m×2Ñ2Ó3°^×5FÑ5FÀ}×G[ÑG[Ó5\óˆMð Ñ*Ø!×4Ñ4Ø#Ð%7ÀTð 5ó ç‰mð ð ,×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNð (Ÿ^™^Ó-ˆNØ#×-Ñ-¨bÓ1×;Ñ;¸MÓJ×MÑMÈm×NbÑNbÓcˆJÜ"Ø˜jÑ)×/Ñ/Ó1°^×5IÑ5IÓ5KÑKØHÈÐHXð YØ"×(Ñ(¨Ñ+Ð,ð.ôð *×8Ñ8Ø—‘˜m×2Ñ2Ó3°^×5FÑ5FÀ}×G[ÑG[Ó5\óˆMð
 Ñ%Ð*=Ñ*IØ×2Ñ2°>ÐCVÐdhÐ2ÓiˆLØ)×7Ñ7ˆNØ&2×&AÑ&AÐ#ð
 ,Ð,C×,FÑ,FÀ~×G\ÑG\Ó,]Ñ^ˆNà'Ÿ^™^Ó-ˆNØ#×-Ñ-¨bÓ1×;Ñ;¸MÓJ×MÑMÈm×NbÑNbÓcˆJÜ"Ø˜jÑ)×/Ñ/Ó1°^×5IÑ5IÓ5KÑKØHÈÐHXð YØ"×(Ñ(¨Ñ+¨n×.BÑ.BÀ1Ñ.EÑEÐFðHôð *×8Ñ8Ø—‘˜m×2Ñ2Ó3°^×5FÑ5FÀ}×G[ÑG[Ó5\óˆMð
 ÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-ÄÔFàŸ+™+×5Ñ5Ó7Ø!.Ø"0Ø#2Ø ,ñˆKð Ÿ+™+×5Ñ5Ó7ˆKØ#×?Ñ?À8ÑKˆIáÐ.Ð:Ü%DÐEVÐ_l×_sÑ_sÔ%tÐ"Ü&Cñ 'Ø'9ð'à!ñ'Ñ#ô '@Ñ&NÀ+Ñ&NÐ#à%�$×%Ñ%ð 	
Ø-Ø.Ø%Ø+Ø'ØØñ	
ð ñ	
ˆô )Ø%×7Ñ7Ø#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø2>Ð2J¡ÐPTØ2@Ð2L¡ÐRVØ$×5Ñ5ô
ð 	
r_   zPProjects the last hidden state from the audio encoder into language model space.c                 ó¢   — | j                   €t        d«      ‚ | j                   ||fddi|¤Ž}| j                  |j                  ¬«      |_        |S )a0  
        input_features (`torch.FloatTensor]` of shape `(num_images, seq_length, num_features)`):
            The tensors corresponding to the input audio.
        input_features_mask (`torch.FloatTensor]` of shape `(num_images, seq_length)`):
            The attention mask for the input audio.
        z•Audio features were requested, but the model was initialized without an audio_config. Cannot process audio without an audio tower and audio embedder.r  Tr   )ró  r{  rô  rê  r  )r‰   r   r!  rT  Úaudio_outputss        r]   r  zGemma4Model.get_audio_featureso  sh   € ð ×ÑÐ#ÜðRóð ð
 )˜×(Ñ(¨Ð9LÑiÐZ^ÐiÐbhÑiˆØ&*×&6Ñ&6À]×EdÑEdÐ&6Ó&eˆÔ#àÐr_   r�   r  )NNNNNNNNNNNNNN)rd   re   rf   rA   r‚   rù  rü  r!   r    rk   rß  rÓ  r   r   r   r  r  rj   rt   r  r&   rl   r   r‘  ra   r�   rs   r  r‘   r’   s   @r]   rì  rì  P  s/  ø„ ð
˜|õ 
ò:ò;ð ÙÐ!rÔsð 7;ñà×'Ñ'ðð "×,Ñ,¨tÑ3ðð Ð+Ñ,ð	ð
 
$òó tó ðð& ÙÐ!tÔuð 7;ñà"×.Ñ.ðð "×,Ñ,¨tÑ3ðð Ð+Ñ,ð	ð
 
$òó vó ðð0 .2Ø26ñ+Jà×#Ñ# dÑ*ð+Jð ×(Ñ(¨4Ñ/ð+Jð 
ˆu×Ñ ×!1Ñ!1°5×3CÑ3CÐCÑ	Dó	+JðZ  ØØð .2Ø15Ø8<Ø37Ø.2Ø37Ø04Ø(,Ø59Ø26Ø!%Ø6:Ø6:Ø04ñd
à×#Ñ# dÑ*ðd
ð ×'Ñ'¨$Ñ.ðd
ð #×.Ñ.°Ñ5ð	d
ð
 ×)Ñ)¨DÑ0ðd
ð Ÿ™ tÑ+ðd
ð #Ÿ\™\¨DÑ0ðd
ð ×&Ñ&¨Ñ-ðd
ð  ™ðd
ð !×+Ñ+¨dÑ2ðd
ð ×(Ñ(¨4Ñ/ðd
ð ˜$‘;ðd
ð "×,Ñ,¨tÑ3ðd
ð "×,Ñ,¨tÑ3ðd
ð  Ÿ,™,¨Ñ-ðd
ð  Ð+Ñ,ð!d
ð" 
#ò#d
ó ó ó  ðd
ðL ÙÐ!sÔtðàŸ™ðð #Ÿ\™\ðð Ð+Ñ,ð	ð
 
Ð'Ñ	'òó uó ôr_   rì  z…
    The base Gemma 4 model comprising a vision backbone, an audio backbone, a language model, and a language modeling
    head.
    c            %       ó6  ‡ — e Zd ZdZd„ Zd„ Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  de
dz  dej                  dz  dej                  dz  dej                  dz  dedz  deej                  z  dej                  dz  dee   def$d„Ze	 ddej                  dej                  dz  dee   fd„«       Ze	 	 ddedej                  d	ej                  dz  de
dz  dej                  dz  dej                  dz  dedz  defd„«       Z	 	 	 	 	 	 	 	 	 	 	 	 	 d ˆ fd„	Zˆ xZS )!ÚGemma4ForConditionalGenerationr”  c                 ó6   — | j                   j                  «       S r�   )r”  rù  rø  s    r]   rù  z=Gemma4ForConditionalGeneration.get_per_layer_input_embeddings’  s   € Ø�z‰z×8Ñ8Ó:Ð:r_   c                 ó:   — | j                   j                  |«       y r�   )r”  rü  rû  s     r]   rü  z=Gemma4ForConditionalGeneration.set_per_layer_input_embeddings•  s   € Ø�
‰
×1Ñ1°%Õ8r_   Nr~  rk  r  r   rF   r!  rH   rý  r  rG   rà  rE   r•  r‡  r–  r„  rT  rJ   c           
      ó°  —  | j                   di d|“d|“d|“d|“d|“d|“d|“d|
“d	|“d
|“d|“d|“d|“d|“d|	“dd“|¤Ž}|j                  }t        |t        «      rt	        | d«      n|}| j                  |dd…|dd…f   «      }| j                  j                  «       j                  x}�||z  }t        j                  |«      }||z  }d}|�7 | j                  ||| j                  j                  «       j                  fi |¤Ž}t        |||j                  |j                  |j                   |j"                  |j$                  |j&                  ¬«      S )r  r~  rk  r  r   rF   r!  rH   rG   rà  rE   r„  r•  r‡  rý  r  r  TN)r˜  r™  rG   r‹   ro  r  r  rb   rS   )r”  rê  r±  r�   rš  r›  rD   r  rœ  rk   rñ   r�  rž  ro   rG   r‹   ro  r  r  rb   )r‰   r~  rk  r  r   rF   r!  rH   rý  r  rG   rà  rE   r•  r‡  r–  r„  rT  rŸ  r‹   r   r™  rœ  r˜  s                           r]   r�   z&Gemma4ForConditionalGeneration.forward˜  s¼  € ðH �$—*‘*ò 
Ùð
á%ð
ñ !4ð
ñ *ð	
ñ
 *ð
ñ !4ð
ñ &ð
ñ ,ð
ñ 0ð
ñ (ð
ñ .ð
ñ ð
ñ  ð
ñ  2ð
ñ  2ð
ñ  Øñ#
ˆð(  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆØ'+§{¡{×'BÑ'BÓ'D×'\Ñ'\Ð\Ð#ÐiØÐ5Ñ5ˆFÜ—Z‘Z Ó'ˆFØÐ5Ñ5ˆFàˆØÐØ%�4×%Ñ% f¨f°d·k±k×6QÑ6QÓ6S×6^Ñ6^ÑiÐbhÑiˆDä+ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;Ø '× ;Ñ ;Ø$×5Ñ5ô	
ð 		
r_   c                 ó>   —  | j                   j                  ||fi |¤ŽS )a-  
        image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
            2D patch position coordinates from the image processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        )r”  r  )r‰   rk  rý  rT  s       r]   r  z1Gemma4ForConditionalGeneration.get_image_featuresè  s$   € ð -ˆt�z‰z×,Ñ,¨\Ð;MÑXÐQWÑXÐXr_   rD   Úis_first_iterationc                 óÔ   — | j                  «       ||||dœ}| j                  «       }	t        |	dd «      dk(  }
|
r&|�$t        ||j                  ¬«      }t	        dd|i|¤ŽS t        di |¤ŽS )NrL   r  r  r°   rI   rS   )r  r¤  rê  r±   r^   r   )rD   rE   rF   rG   rH   rà  r6  rT  rU   rÇ  r-  rI   s               r]   r   z8Gemma4ForConditionalGeneration.create_masks_for_generateö  s“   € ð ×,Ñ,Ó.Ø*Ø,Ø.Ø(ñ
ˆð ×,Ñ,Ó.ˆÜ˜KÐ)FÈÓMÐQYÑYˆ	áÐ*Ð6Ü!@ÐARÐ[h×[oÑ[oÔ!pÐÜ0ñ Ø#5ðàñð ô
 )Ñ7¨;Ñ7Ð7r_   c                 óž   •— t        ‰| �  |f|||||||
|dœ|¤Ž}|s|s||d<   ||d<   ||d<   |	|d<   nd |d<   |s|j                  dd «      }|S )N)rG   rE   rF   rH   r‡  r–  rÒ  r6  rk  r  r   r!  rà  r„  )r�   Úprepare_inputs_for_generationr�  )r‰   r~  rG   rE   rH   rk  r  r   rF   r!  rÒ  r‡  r–  r•  r6  rT  Úmodel_inputsrX   rŠ   s                     €r]   r9  z<Gemma4ForConditionalGeneration.prepare_inputs_for_generation	  s›   ø€ ô& ‘wÑ<Øð
à+Ø'Ø)Ø%ØØ)Ø)Ø1ñ
ð ñ
ˆñ ¡YØ+7ˆL˜Ñ(Ø2EˆLÐ.Ñ/Ø-;ˆLÐ)Ñ*Ø2EˆLÐ.Ò/ð 15ˆLÐ,Ñ-ñ "Ø× Ñ Ð!3°TÓ:ˆAàÐr_   )NNNNNNNNNNNNNNr   Nr�   )NF)NNNNNNNNNTNNF)rd   re   rf   r¡  rù  rü  rk   rÓ  rß  rl   r   r‘  r�   r   r   ro   r�   r    r  r¾  r   rh   r   r9  r‘   r’   s   @r]   r1  r1  ‰  sÆ  ø„ ð  Ðò;ò9ð
 .2Ø15Ø8<Ø37Ø.2Ø37Ø04Ø6:Ø6:Ø(,Ø59Ø26Ø*.Ø!%Ø-.Ø04ñ#N
à×#Ñ# dÑ*ðN
ð ×'Ñ'¨$Ñ.ðN
ð #×.Ñ.°Ñ5ð	N
ð
 ×)Ñ)¨DÑ0ðN
ð Ÿ™ tÑ+ðN
ð #Ÿ\™\¨DÑ0ðN
ð ×&Ñ&¨Ñ-ðN
ð "×,Ñ,¨tÑ3ðN
ð "×,Ñ,¨tÑ3ðN
ð  ™ðN
ð !×+Ñ+¨dÑ2ðN
ð ×(Ñ(¨4Ñ/ðN
ð × Ñ  4Ñ'ðN
ð ˜$‘;ðN
ð  ˜eŸl™lÑ*ð!N
ð"  Ÿ,™,¨Ñ-ð#N
ð$ Ð+Ñ,ð%N
ð& 
&ó'N
ð` ð 7;ñYà×'Ñ'ðYð "×,Ñ,¨tÑ3ðYð Ð+Ñ,ò	Yó ðYð ð 26Ø*/ñ8Ø ð8à—|‘|ð8ð Ÿ™ tÑ+ð8ð  ™ð	8ð
 —l‘l TÑ)ð8ð !Ÿ<™<¨$Ñ.ð8ð ! 4™Kð8ð 
ò8ó ð8ðB ØØØØ ØØØ ØØØØØ ÷.ñ .r_   r1  )r£  r“  r1  rì  rK  rm  r`  )r*   )�r¤   Úcollectionsr   Úcollections.abcr   Údataclassesr   Ú	functoolsr   rk   r   Útorch.nnr   rÙ   Ú r
   rT  Úactivationsr   Úcache_utilsr   r   Úconfiguration_utilsr   Úmasking_utilsr   r   r   r   r   r   r   r   Úmodeling_flash_attention_utilsr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r    r!   r"   r#   r$   Úutils.genericr%   r&   r'   Úutils.output_capturingr(   r)   Úauto.modeling_autor+   Úgemma3.modeling_gemma3r,   r-   r.   r/   r0   r1   r2   Úgemma3n.modeling_gemma3nr3   r4   r5   r6   r7   r8   r9   r:   r;   Úllama.modeling_llamar<   Úmixtral.modeling_mixtralr=   Ú0moonshine_streaming.modeling_moonshine_streamingr>   Úconfiguration_gemma4r@   rA   rB   rC   Ú
get_loggerrd   Úloggerrl   rh   r^   ra   ro   rq   rs   rÿ  rv   r”   r˜   r¿   r  r  r$  ÚConv1dr6  r=  rK  rW  rq  r‹  r�   rž  r   rÀ  rÕ  rá  rí  rô  r  r  r&  r5  rI  rK  rm  r“  r£  r`  rÅ  rß  r±   rê  rì  r1  Ú__all__rS   r_   r]   ú<module>rX     s¢  ðó Ý  Ý $Ý !Ý %ã Ý Ý $å &Ý !ß .Ý 3÷	÷ 	ó 	õ Cß Sß Kß FÝ &÷÷ ÷ ^Ñ ]ß EÝ *÷÷ ñ ÷
÷ 
õ 
õ 8Ý 5Ý [ß gÓ gñ ÔØð 
ˆ×	Ñ	˜HÓ	%€ð4Øð4à—<‘<ð4ð —L‘L 4Ñ'ð4ð ˜T‘\ð	4ð
 —,‘, Ñ%ð4ð Ÿ™ð4ð 
ó4ônQÐ :ô Qô*QÐ#@ô Qð2 ô
QÐ$;ó 
Qó ð
Qð Ø
ô3Ð7ó 3ó ó ð3ô˜BŸI™Iô ô:	�Nô 	ô7 r§y¡yô 7ô>i)˜2Ÿ9™9ô i)ôX#¨b¯i©iô #ô8;¨¯©ô ;ô< ˜RŸY™Yô  ôH"˜bŸi™iô "ôB&˜RŸY™Yô &ôR0�r—y‘yô 0ôl*3 §	¡	ô *3ôZ@0˜Ÿ™ô @0ôFa�iô að ñ5&Ø‡|�|ð5&à	�‰ð5&ð 
�‰ð5&ð —,‘,ð	5&ð
 ð5&ð ‡\�\ó5&ôp:Ð"6ô :ðz ô:)˜Oó :)ó ð:)ô|"Ð1ô "ôJ)H˜"Ÿ)™)ô )Hô^^�Iô ^ôUÐ 5ô UôDq)˜"Ÿ)™)ô q)ôh7˜ô 7ô"@�r—y‘yô "@ôJOÐ/ô Oôd	Ð$Aô 	ô=)Ð2ô =)ñ@ Ð`ÔaôW
�oó W
ó bðW
ñt Ð]Ô^ôJ
Ð)ó J
ó _ðJ
ôZScÐ,ô ScôlAHÐ-ô AHôH6Ð8ô 6ð>Ø—L‘L 4Ñ'ðà—\‘\ DÑ(ðð ��_óð:	°u·|±|ð 	ÈUÏ\Ê\ð 	Ð^c×^jÑ^jó 	ñ ðôôp�,ó póðpñf	 ðôôtÐ%Dó tóðtòn�r_   