Ë
    ÛÓ^j"¾ ã                   ó^  — d dl Z d dlmZ d dlmZmZ d dlmZ d dlm	Z	 d dl
Z
d dlmZ d dlmc mZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZ ddlmZ ddlm Z m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1 ddl2m3Z3m4Z4 ddl5m6Z6 ddl7m8Z8 ddl9m:Z:m;Z;m<Z<m=Z=  e0«       rd dl>m?Z? e.e G d„ de!«      «       «       Z@ e.d¬«      e G d„ de «      «       «       ZA e.d¬«      e G d „ d!e,«      «       «       ZB G d"„ d#ej†                  «      ZD G d$„ d%ej†                  «      ZE G d&„ d'ej†                  «      ZF G d(„ d)ej†                  «      ZG G d*„ d+ej†                  «      ZH G d,„ d-ej†                  «      ZI G d.„ d/ej†                  «      ZJ G d0„ d1ej†                  «      ZK G d2„ d3ej†                  «      ZL G d4„ d5ej†                  «      ZM G d6„ d7ejœ                  «      ZO G d8„ d9ej†                  «      ZP G d:„ d;ej†                  «      ZQ G d<„ d=ej†                  «      ZRd>„ ZSd?e
j¨                  d@eUdAe
j¨                  fdB„ZV	 	 	 djdCej†                  dDe
j¨                  dEe
j¨                  dFe
j¨                  dGe
j¨                  dz  dHeWeUz  dIeWdz  dJeWdz  dAeXe
j¨                  e
j¨                  f   fdK„ZYdkdLe
j¨                  dMe
j¨                  dNe
j¨                  dOeUfdP„ZZ G dQ„ dRej†                  «      Z[ G dS„ dTe«      Z\e. G dU„ dVe(«      «       Z] G dW„ dXe]«      Z^ G dY„ dZej†                  «      Z_ e.d[¬«       G d\„ d]e]«      «       Z` e.d^¬«       G d_„ d`e]e«      «       Za G da„ dbej†                  «      Zb e.dc¬«       G dd„ dee]«      «       Zc e.df¬«       G dg„ dhe]e«      «       Zdg di¢Zey)lé    N)ÚUserDict)ÚCallableÚSequence)Ú	dataclass)ÚOptionalé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_accelerate_availableÚtorch_compilable_check)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú	AutoModelé   )ÚGemma3nAudioConfigÚGemma3nConfigÚGemma3nTextConfigÚGemma3nVisionConfig)Úadd_hook_to_modulec                   ó:   — e Zd ZU dZdZej                  dz  ed<   y)ÚGemma3nAudioEncoderModelOutputzy
    audio_mel_mask (`torch.BoolTensor`, *optional*):
        A torch.BoolTensor of shape `(batch_size, num_frames)`
    NÚaudio_mel_mask)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r,   ÚtorchÚ
BoolTensorÚ__annotations__© ó    úw/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/gemma3n/modeling_gemma3n.pyr+   r+   ;   s   … ñð
 /3€N�E×$Ñ$ tÑ+Ô2r5   r+   zL
    Base class for Gemma3n outputs, with hidden states and attentions.
    ©Úcustom_introc                   ób   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   y)ÚGemma3nModelOutputWithPastaÝ  
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    audio_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
    NÚimage_hidden_statesÚaudio_hidden_states)	r-   r.   r/   r0   r;   r1   ÚFloatTensorr3   r<   r4   r5   r6   r:   r:   F   s5   … ñð 59Ð˜×*Ñ*¨TÑ1Ó8à48Ð˜×*Ñ*¨TÑ1Ô8r5   r:   zS
    Base class for Gemma3n causal language model (or autoregressive) outputs.
    c                   ó"  — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   dZej                  dz  ed<   dZej                  dz  ed	<   y)
ÚGemma3nCausalLMOutputWithPastaF  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.text_config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
    audio_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
    NÚlossÚlogitsÚpast_key_valuesÚhidden_statesÚ
attentionsr;   r<   )r-   r.   r/   r0   r@   r1   r=   r3   rA   rB   r   rC   ÚtuplerD   r;   r<   r4   r5   r6   r?   r?   `   s¡   … ñð$ &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø'+€FˆE×Ñ Ñ$Ó+Ø$(€O�U˜T‘\Ó(Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ó6Ø48Ð˜×*Ñ*¨TÑ1Ó8à48Ð˜×*Ñ*¨TÑ1Ô8r5   r?   c                   ó†   ‡ — e Zd Zd	dededefˆ fd„Zdej                  fd„Z	dej                  dej                  fd„Z
ˆ xZS )
ÚGemma3nRMSNormÚdimÚepsÚ
with_scalec                 ó¶   •— t         ‰| �  «        || _        || _        | j                  r0t	        j
                  t        j                  |«      d¬«      | _        y y )NT)Úrequires_grad)	ÚsuperÚ__init__rI   rJ   ÚnnÚ	Parameterr1   ÚonesÚweight)ÚselfrH   rI   rJ   Ú	__class__s       €r6   rN   zGemma3nRMSNorm.__init__„   sB   ø€ Ü‰ÑÔØˆŒØ$ˆŒà�?Š?ÜŸ,™,¤u§z¡z°#£ÀdÔKˆD�Kð r5   rC   c                 ó’   — |j                  d«      j                  dd¬«      | j                  z   }|t        j                   |d«      z  S )Nr"   éÿÿÿÿT)Úkeepdimç      à¿)ÚpowÚmeanrI   r1   )rS   rC   Úmean_squareds      r6   Ú_normzGemma3nRMSNorm._normŒ   sA   € Ø$×(Ñ(¨Ó+×0Ñ0°¸TÐ0ÓBÀTÇXÁXÑMˆàœuŸy™y¨°tÓ<Ñ<Ð<r5   Úreturnc                 ó´   — | j                  |j                  «       «      }| j                  r|| j                  j                  «       z  }|j	                  |«      S ©N)r\   ÚfloatrJ   rR   Útype_as)rS   rC   Únormed_outputs      r6   ÚforwardzGemma3nRMSNorm.forward‘   sH   € ØŸ
™
 =×#6Ñ#6Ó#8Ó9ˆØ�?Š?Ø)¨D¯K©K×,=Ñ,=Ó,?Ñ?ˆMØ×$Ñ$ ]Ó3Ð3r5   )g�íµ ÷Æ°>T)r-   r.   r/   Úintr`   ÚboolrN   r1   ÚTensorr\   rc   Ú__classcell__©rT   s   @r6   rG   rG   ƒ   sL   ø„ ñL˜Cð L eð LÀõ Lð= 5§<¡<ó =ð
4 U§\¡\ð 4°e·l±l÷ 4r5   rG   c                   ó  ‡ — e Zd Zdefˆ fd„Zdej                  dej                  dej                  fd„Zdej                  de	d	e	d
e	de	de	de	dej                  fd„Z
dej                  dej                  dej                  fd„Zˆ xZS )Ú%Gemma3nAudioRelativePositionEmbeddingÚconfigc                 óR  •— t         ‰| �  «        || _        | j                  j                  | _        | j                  j
                  | _        | j                  | j                  z  | _        t        d| j                  j                  dz
  «      | _
        | j                  j                  | _        t        j                  | j                  | j                  | j                  z  d¬«      | _        d}d}| j                  dz  }t!        j"                  t%        |«      t%        |«      z  «      t        |dz
  d«      z  }|t'        j(                  t'        j*                  |«      | z  «      z  }| j-                  d|j%                  «       j/                  d«      j/                  d«      d¬	«       y )
Nr   r$   F©Úbiasç      ð?ç     ˆÃ@r"   Úinv_timescales©Ú
persistent)rM   rN   rk   Úconf_num_attention_headsÚ	num_headsÚhidden_sizeÚchannelsÚhead_dimÚmaxÚconf_attention_context_leftÚmax_backwardÚconf_attention_context_rightÚmax_forwardrO   ÚLinearÚpos_projÚmathÚlogr`   r1   ÚexpÚarangeÚregister_bufferÚ	unsqueeze)rS   rk   Úmin_timescaleÚmax_timescaleÚnum_timescalesÚlog_timescale_incrementrq   rT   s          €r6   rN   z.Gemma3nAudioRelativePositionEmbedding.__init__œ   sL  ø€ Ü‰ÑÔØˆŒàŸ™×=Ñ=ˆŒØŸ™×/Ñ/ˆŒØŸ™¨¯©Ñ7ˆŒÜ  4§;¡;×#JÑ#JÈQÑ#NÓOˆÔØŸ;™;×CÑCˆÔäŸ	™	 $§-¡-°·±À$Ç-Á-Ñ1OÐV[Ô\ˆŒàˆØˆØŸ™¨!Ñ+ˆÜ"&§(¡(¬5°Ó+?Ä%ÈÓBVÑ+VÓ"WÔZ]Ð^lÐopÑ^pÐrsÓZtÑ"tÐØ&¬¯©´5·<±<ÀÓ3OÐSjÐRjÑ3jÓ)kÑkˆØ×ÑØØ× Ñ Ó"×,Ñ,¨QÓ/×9Ñ9¸!Ó<Øð 	õ 	
r5   ÚpositionÚdtyper]   c                 óP  — |j                  «       j                  d«      }|| j                  j                  |j                  t
        j                  ¬«      z  }t        j                  t        j                  |«      t        j                  |«      gd¬«      }|j                  |«      S )NrV   ©Údevicer‹   ©rH   )r`   r…   rq   ÚtorŽ   r1   Úfloat32ÚcatÚsinÚcosÚtype)rS   rŠ   r‹   Úscaled_timeÚtiming_signals        r6   Ú_get_timing_signal_1d_posz?Gemma3nAudioRelativePositionEmbedding._get_timing_signal_1d_pos³   s}   € Ø—>‘>Ó#×-Ñ-¨bÓ1ˆØ ×!4Ñ!4×!7Ñ!7¸x¿¹ÔV[×VcÑVcÐ!7Ó!dÑdˆÜŸ	™	¤5§9¡9¨[Ó#9¼5¿9¹9À[Ó;QÐ"RÐXZÔ[ˆØ×!Ñ! %Ó(Ð(r5   Úterm_bd_before_shiftÚ
batch_sizeru   Únum_query_blocksÚquery_block_sizeÚkey_context_sizeÚmax_span_plus_1c                 óè   — |dz   |z
  }d|f}	t         j                  j                  ||	«      }
|
j                  |||||dz   z  f«      }|dd…dd…dd…d||z  …f   }|j                  |||||f«      }|S )aZ  Performs the relative shift.

        Args:
          term_bd_before_shift: Tensor of shape [B, N, U, W, F_span]. batch_size
            (B), num_heads (N), num_query_blocks (U), query_block_size (W),
            key_context_size (C = W+L+R), max_span_plus_1 (F_span = L+R+1).

        Returns:
          Tensor of shape [B, N, U, W, C].
        r$   r   N)rO   Ú
functionalÚpadÚreshape)rS   r™   rš   ru   r›   rœ   r�   rž   Úpad_amount_last_dimÚpadding_tupleÚterm_bd_paddedÚterm_bd_reshapedÚterm_bd_slicedÚterm_bd_shifteds                 r6   Ú_relative_shiftz5Gemma3nAudioRelativePositionEmbedding._relative_shift¹   s·   € ð4  0°!Ñ3°ÑFÐð Ð/Ð0ˆäŸ™×*Ñ*Ð+?ÀÓOˆð
 *×1Ñ1àØØ Ø Ð$4°qÑ$8Ñ9ð	ó
Ðð *ª!ªQ²Ð3XÐ5EÐHXÑ5XÐ3XÐ*XÑYˆð )×0Ñ0àØØ Ø Ø ðó
ˆð Ðr5   ÚqueriesÚkeysc           	      óR  — |j                   \  }}}}}|j                   \  }}}	}}t        j                  | j                  | j                   dz
  d|j
                  ¬«      j                  d«      }
|
j                   d   }| j                  |
|j                  ¬«      }| j                  |«      }|j                  d|| j                  | j                  «      j                  d«      }|j                  ddddd«      }|j                  ddddd«      }t        j                  ||«      }|j                  ddddd«      }|j                  ddd«      }|j                  ||||z  |«      }t        j                  ||«      }|j                  |||||«      }| j!                  ||||||	|«      }||z   S )	Nr$   rV   ©rŽ   r   ©r‹   r   r"   é   )Úshaper1   rƒ   r{   r}   rŽ   r…   r˜   r‹   r   r¢   ru   rx   ÚsqueezeÚpermuteÚmatmulr©   )rS   rª   r«   rš   r›   rœ   ru   rx   Ú_r�   Úpos_indicesrž   Úsin_emb_timing_signalÚprojected_sin_embÚsin_embÚ	queries_pÚkeys_p_tÚterm_acÚ
q_permutedÚ
s_permutedÚ
q_reshapedÚterm_bd_unshifed_matmulÚterm_bd_unshifedr¨   s                           r6   rc   z-Gemma3nAudioRelativePositionEmbedding.forwardö   sÕ  € ð OVÏmÉmÑKˆ
Ð$Ð&6¸	À8Ø'+§z¡zÑ$ˆˆ1Ð  1ô —l‘l 4×#4Ñ#4°t×7GÑ7GÐ6GÈ!Ñ6KÈRÐX_×XfÑXfÔg×qÑqØó
ˆð &×+Ñ+¨AÑ.ˆà $× >Ñ >Ø˜wŸ}™}ð !?ó !
Ðð
 !ŸM™MÐ*?Ó@Ðà#×+Ñ+¨A¨ÀÇÁÐPT×P]ÑP]Ó^×fÑfØó
ˆð —O‘O A q¨!¨Q°Ó2ˆ	Ø—<‘<  1 a¨¨AÓ.ˆÜ—,‘,˜y¨(Ó3ˆð —_‘_ Q¨¨1¨a°Ó3ˆ
ð —_‘_ Q¨¨1Ó-ˆ
ð  ×'Ñ'¨
°IÐ?OÐRbÑ?bÐdlÓmˆ
ô
 #(§,¡,¨z¸:Ó"FÐð 3×:Ñ:ØØØØØó
Ðð ×.Ñ.ØØØØØØØó
ˆð ˜Ñ(Ð(r5   )r-   r.   r/   r%   rN   r1   rf   r‹   r˜   rd   r©   rc   rg   rh   s   @r6   rj   rj   ›   s¿   ø„ ð
Ð1õ 
ð.)°%·,±,ð )ÀuÇ{Á{ð )ÐW\×WcÑWcó )ð;à#Ÿl™lð;ð ð;ð ð	;ð
 ð;ð ð;ð ð;ð ð;ð 
�‰ó;ðzL)˜uŸ|™|ð L)°5·<±<ð L)ÀEÇLÁL÷ L)r5   rj   c                   ó*  ‡ — e Zd Zdefˆ fd„Zd„ Zdej                  dededej                  fd„Z	d	ej                  dej                  fd
„Z
d	ej                  dej                  fd„Zd	ej                  dej                  dej                  fd„Zˆ xZS )ÚGemma3nAudioAttentionrk   c                 óì  •— t         ‰| �  «        || _        | j                  j                  | _        | j                  j
                  | _        | j
                  | j                  z  | _        | j                  j                  | _        | j                  j                  | _
        t        d| j                  j                  dz
  «      | _        | j                  j                  | _        | j                  | j                  z   | j                  z   | _        t#        |«      | _        t'        j(                  t+        j,                  | j                  f«      «      | _        t'        j0                  | j
                  | j                  | j                  z  d¬«      | _        t'        j0                  | j
                  | j                  | j                  z  d¬«      | _        t'        j0                  | j
                  | j                  | j                  z  d¬«      | _        | j                  dz  }dt*        j&                  j8                  j;                  t+        j<                  d«      «      z  }| j?                  d||z  jA                  «       jC                  «       d¬	«       | jE                  «       }| j?                  d
|d¬	«       | j?                  dt+        j<                  | j                  «      jG                  «       d¬	«       y )Nr   r$   Frm   rX   ro   ç        Úq_scalerr   Úlocal_causal_valid_maskÚsoftcap)$rM   rN   rk   rt   ru   rv   rx   Úconf_attention_chunk_sizeÚ
chunk_sizer|   Úmax_future_horizonry   rz   Úmax_past_horizonÚconf_attention_logit_capÚattention_logits_soft_capÚcontext_sizerj   Úrelative_position_embeddingrO   rP   r1   ÚzerosÚper_dim_scaler~   Úq_projÚk_projÚv_projr    ÚsoftplusÚtensorr„   ÚcloneÚdetachÚcreate_local_causal_valid_maskr`   )rS   rk   rÅ   Úr_softplus_0rÆ   rT   s        €r6   rN   zGemma3nAudioAttention.__init__F  s)  ø€ Ü‰ÑÔØˆŒàŸ™×=Ñ=ˆŒØŸ;™;×2Ñ2ˆÔØ×(Ñ(¨D¯N©NÑ:ˆŒàŸ+™+×?Ñ?ˆŒØ"&§+¡+×"JÑ"JˆÔÜ # A t§{¡{×'NÑ'NÐQRÑ'RÓ SˆÔØ)-¯©×)MÑ)MˆÔ&Ø ŸO™O¨d×.CÑ.CÑCÀd×F]ÑF]Ñ]ˆÔä+PÐQWÓ+XˆÔ(ÜŸ\™\¬%¯+©+°t·}±}Ð6FÓ*GÓHˆÔä—i‘i × 0Ñ 0°$·.±.À4Ç=Á=Ñ2PÐW\Ô]ˆŒÜ—i‘i × 0Ñ 0°$·.±.À4Ç=Á=Ñ2PÐW\Ô]ˆŒÜ—i‘i × 0Ñ 0°$·.±.À4Ç=Á=Ñ2PÐW\Ô]ˆŒà—-‘- Ñ%ˆØœUŸX™X×0Ñ0×9Ñ9¼%¿,¹,ÀsÓ:KÓLÑLˆØ×Ñ˜Y¨°<Ñ)?×(FÑ(FÓ(H×(OÑ(OÓ(QÐ^cÐÔdà"&×"EÑ"EÓ"GÐØ×ÑÐ6Ð8OÐ\aÐÔbà×ÑØÜ�L‰L˜×7Ñ7Ó8×>Ñ>Ó@Øð 	õ 	
r5   c                 ó  — t        j                  t        j                  | j                  | j                  ft         j
                  ¬«      d¬«      j                  }t        j                  t        j                  | j                  | j                  ft         j
                  ¬«      | j                  | j                  z   ¬«      }t        j                  | j                  | j                  ft         j
                  ¬«      }||z  |z  }|S )Nr®   r   )Údiagonal)	r1   ÚtrilrQ   rÎ   rÉ   re   ÚTrË   rÊ   )rS   Úlower_causal_maskÚupper_causal_maskrÆ   s       r6   rÙ   z4Gemma3nAudioAttention.create_local_causal_valid_maskh  sÅ   € Ü!ŸJ™JÜ�J‰J˜×)Ñ)¨4¯?©?Ð;Ä5Ç:Á:ÔNØô
÷ ‰!ð 	ô "ŸJ™JÜ�J‰J˜Ÿ™¨×):Ñ):Ð;Ä5Ç:Á:ÔNØ×*Ñ*¨T×-DÑ-DÑDô
Ðô #(§*¡*¨d¯o©o¸t×?PÑ?PÐ-QÔY^×YcÑYcÔ"dÐØ"9Ð<MÑ"MÐPaÑ"aÐØ&Ð&r5   ÚxÚpad_leftÚ	pad_rightr]   c                 ó°   — |j                   ^}}}|j                  ||g|¢­«      }|j                  ||g|¢­«      }t        j                  |||gd¬«      }|S )Nr$   r�   )r°   Ú	new_zerosr1   r’   )	rS   rá   râ   rã   Úbatchr´   Ú
tail_shapeÚleftÚrights	            r6   Ú	_pad_dim1zGemma3nAudioAttention._pad_dim1u  s^   € Ø !§¡Ðˆˆq�:Ø�{‰{˜E 8Ð9¨jÑ9Ó:ˆØ—‘˜U IÐ;°
Ñ;Ó<ˆÜ�I‰I�t˜Q Ð&¨AÔ.ˆØˆr5   rC   c                 ó(  — |j                   }|dd \  }}|| j                  z   dz
  | j                  z  }|| j                  z  |z
  x}dkD  r| j                  |d|«      }||| j                  f|dd z   }|j                  |«      j	                  «       }|S )aE  Turns a sequence to non overlapping blocks.

        Args:
            hidden_states: a tensor of [batch, time, ...].

        Returns:
            A tensor of [batch, num_blocks, block_size, ...], with necessary
            paddings,
            where output[:, i, ...] are x[:, i*block_size:(i+1)*block_size, ...].
        Nr"   r$   r   )r°   rÉ   rê   r¢   Ú
contiguous)rS   rC   r°   ÚbÚtÚ
num_blocksÚpadding_lenÚpermute_dimss           r6   Ú_convert_to_blockz'Gemma3nAudioAttention._convert_to_block|  s¡   € ð ×#Ñ#ˆØ�R�aˆy‰ˆˆ1Ø˜$Ÿ/™/Ñ)¨AÑ-°$·/±/ÑAˆ
à%¨¯©Ñ7¸!Ñ;Ð;ˆK¸qÒ@Ø ŸN™N¨=¸!¸[ÓIˆMà˜: t§¡Ð7¸%ÀÀ¸)ÑCˆØ%×-Ñ-¨lÓ;×FÑFÓHˆØÐr5   c                 ó\  — | j                   }| j                  | j                  z   dz
  }| j                  |||«      }| j                  }| j                  }|j                  d||¬«      }|j                  dkD  r'|j                  dkD  rt        j                  |dd¬«      }|j                  «       S )aã  Extracts temporal context for every block.

        Args:
            hidden_states: a tensor of [batch, time, ...].

        Returns:
            A tensor of [batch, num_blocks, context_size, ...], with necessary
            paddings,
            where context_size = block_size + left_context + right_context,
            and output[:, i, ...] are x[:, start-left_context:end+right_context,
            ...],
            start = i * block_size, end = (i + 1) * block_size.
        r$   )Ú	dimensionÚsizeÚstepr"   r   rV   )ÚsourceÚdestination)
rË   rÊ   rÉ   rê   rÎ   ÚunfoldÚndimr1   Úmovedimrì   )rS   rC   râ   rã   Ú	frame_lenÚ
frame_stepÚ
x_unfoldeds          r6   Ú_extract_block_contextz,Gemma3nAudioAttention._extract_block_context’  s§   € ð ×(Ñ(ˆð ×+Ñ+¨d¯o©oÑ=ÀÑAˆ	ØŸ™ }°hÀ	ÓJˆà×%Ñ%ˆ	Ø—_‘_ˆ
ð #×)Ñ)°A¸IÈJÐ)ÓWˆ
ð ×Ñ Ò! j§o¡o¸Ò&9ô Ÿ™ z¸"È!ÔLˆJà×$Ñ$Ó&Ð&r5   Úmaskc                 ó	  — g |j                   d d ¢| j                  ‘| j                  ‘­}| j                  |«      j	                  |«      j                  «       }| j                  |«      j	                  |«      j                  «       }| j                  |«      j	                  |«      j                  «       }t        j                  j                  j                  | j                  «      }ddd| j                  f}|j                  |«      }	|| j                  z  |	z  }|j                   d d \  }
}| j                  |«      }| j!                  |«      }| j!                  |«      }|j                   d   }| }| j!                  |«      }|j"                  dk(  rI|j                   d   |j                   d   z  | j$                  k(  r|j	                  |
|| j$                  «      }|j                   |
|| j$                  fk7  r,t'        d|j                   › d|
› d|› d| j$                  › d	�	«      ‚|j)                  d«      j)                  d
«      }| j*                  j)                  d«      j)                  d«      j)                  d«      }t        j,                  ||j/                  |j0                  «      «      }| j3                  ||«      }| j4                  j/                  |j0                  «      }||z  }t        j6                  |«      }||z  }t        j8                  ||t        j:                  |j<                  «      j>                  «      }t        j                  j                  jA                  |dt        jB                  ¬«      j/                  |j<                  ¬«      }|j                   \  }}}}}|j                   d   }|jE                  ddddd«      j	                  d||«      }|jE                  ddddd«      j	                  d||«      }t        jF                  ||«      } | j	                  |||||«      jE                  ddddd«      }!|!j	                  |
|| jH                  z  | j                  | j                  f«      }!|!d d …d |…f   }!|!S )NrV   r$   r"   r¯   r   z%Shape of extracted_valid_mask_blocks z	 is not (z, z) after potential reshape.éþÿÿÿr   ©rH   r‹   r®   )%r°   ru   rx   rÒ   r¢   rì   rÓ   rÔ   r1   rO   r    rÕ   rÑ   ÚviewrÅ   rò   rÿ   rú   rÎ   Ú
ValueErrorr…   rÆ   Úlogical_andr�   rŽ   rÏ   rÇ   ÚtanhÚwhereÚfinfor‹   ÚminÚsoftmaxr‘   r²   ÚbmmrÉ   )"rS   rC   r   Ú	qkv_shapeÚquery_statesÚ
key_statesÚvalue_statesÚper_dim_scale_spÚbroadcast_shapeÚper_dim_scale_sp_broadcastrš   Úq_timeÚquery_blocksÚ
key_blocksÚvalue_blocksr›   Úoriginal_valid_maskÚextracted_valid_mask_blocksÚcondition_from_input_validityÚcondition_from_causalityÚfinal_condition_for_whererA   Úsoftcap_valÚprobabilitiesÚb_dimÚn_dimÚu_dimÚw_dimÚc_dimÚh_dimÚprob_bunÚv_bunÚ
result_bmmÚcontext_vectorss"                                     r6   rc   zGemma3nAudioAttention.forwardÂ  sT  € àN�m×)Ñ)¨#¨2Ð.ÐN°·±ÐNÀÇÁÑNˆ	Ø—{‘{ =Ó1×9Ñ9¸)ÓD×OÑOÓQˆØ—[‘[ Ó/×7Ñ7¸	ÓB×MÑMÓOˆ
Ø—{‘{ =Ó1×9Ñ9¸)ÓD×OÑOÓQˆä Ÿ8™8×.Ñ.×7Ñ7¸×8JÑ8JÓKÐà˜a  D§M¡MÐ2ˆØ%5×%:Ñ%:¸?Ó%KÐ"Ø# d§l¡lÑ2Ð5OÑOˆà)×/Ñ/°°Ð3Ñˆ
�Fà×-Ñ-¨lÓ;ˆØ×0Ñ0°Ó<ˆ
Ø×2Ñ2°<Ó@ˆØ'×-Ñ-¨aÑ0Ðð  $˜eÐð '+×&AÑ&AÐBUÓ&VÐ#ð (×,Ñ,°Ò1Ø+×1Ñ1°!Ñ4Ð7R×7XÑ7XÐYZÑ7[Ñ[Ð_c×_pÑ_pÒpà*E×*MÑ*MØÐ,¨d×.?Ñ.?ó+Ð'ð '×,Ñ,ØØØ×Ñð1
ò 
ô
 ðØ/×5Ñ5Ð6°iÀ
¸|ð LØ$Ð% R¨×(9Ñ(9Ð':Ð:TðVóð ð )D×(MÑ(MÈaÓ(P×(ZÑ(ZÐ[]Ó(^Ð%ð $(×#?Ñ#?×#IÑ#IÈ!Ó#L×#VÑ#VÐWXÓ#Y×#cÑ#cÐdeÓ#fÐ ô
 %*×$5Ñ$5Ø)Ø$×'Ñ'Ð(E×(LÑ(LÓMó%
Ð!ð ×1Ñ1°,À
ÓKˆð —l‘l—o‘o f§m¡mÓ4ˆØ˜+Ñ%ˆÜ—‘˜FÓ#ˆØ˜+Ñ%ˆô —‘Ð6¸ÄÇÁÈFÏLÉLÓ@Y×@]Ñ@]Ó^ˆÜŸ™×+Ñ+×3Ñ3°FÀÌ%Ï-É-Ð3ÓX×[Ñ[Ðbn×btÑbtÐ[Óuˆð -:×,?Ñ,?Ñ)ˆˆu�e˜U EØ×"Ñ" 2Ñ&ˆØ ×(Ñ(¨¨A¨q°!°QÓ7×?Ñ?ÀÀEÈ5ÓQˆØ×$Ñ$ Q¨¨1¨a°Ó3×;Ñ;¸BÀÀuÓMˆÜ—Y‘Y˜x¨Ó/ˆ
Ø$×,Ñ,¨U°E¸5À%ÈÓO×WÑWÐXYÐ[\Ð^_ÐabÐdeÓfˆØ)×1Ñ1àØ  4§?¡?Ñ2Ø—‘Ø—‘ð	ó
ˆð *ª!¨W¨f¨W¨*Ñ5ˆàÐr5   )r-   r.   r/   r%   rN   rÙ   r1   rf   rd   rê   rò   rÿ   r2   rc   rg   rh   s   @r6   rÂ   rÂ   E  s¢   ø„ ð 
Ð1õ  
òD'ð˜5Ÿ<™<ð °3ð À3ð È5Ï<É<ó ð¨u¯|©|ð ÀÇÁó ð,.'°E·L±Lð .'ÀUÇ\Á\ó .'ð`d U§\¡\ð d¸×9IÑ9Ið dÈeÏlÉl÷ dr5   rÂ   c                   ór   ‡ — e Zd ZdZ	 d	dedee   defˆ fd„Zdej                  dej                  fd„Z
ˆ xZS )
ÚGemma3nAudioCumulativeGroupNormaè  Applies Group Normalization cumulatively over the time dimension.

    This layer normalizes the input by calculating the mean and variance
    cumulatively over the time dimension (dim 1). The statistics are computed
    over all feature dimensions (specified by `feature_dims` and `num_channels`)
    for elements marked as valid by the optional `mask`.

    If a `mask` is provided (True for valid, False for invalid/padded),
    invalid time steps do not contribute to the statistics calculation, and
    their corresponding output values are zeroed out.

    Scale and bias, if enabled, are applied per-channel (last dimension).
    This behavior is similar to JAX's `GroupNormalization` with `num_groups=1`
    and `cumulative=True`.
    Únum_channelsÚfeature_dimsrI   c           	      ó  •— t         ‰| �  «        || _        t        |«      | _        || _        t        j                  t        j                  |«      «      | _
        t        t        ddt        | j                  «      z   dz   «      «      | _        y )Nr"   r$   )rM   rN   r+  rE   r,  rI   rO   rP   r1   rQ   rR   ÚrangeÚlenÚreduction_axes)rS   r+  r,  rI   rT   s       €r6   rN   z(Gemma3nAudioCumulativeGroupNorm.__init__:  sr   ø€ ô 	‰ÑÔØ(ˆÔÜ! ,Ó/ˆÔØˆŒô —l‘l¤5§:¡:¨lÓ#;Ó<ˆŒô $¤E¨!¨Q´°T×5FÑ5FÓ1GÑ-GÈ!Ñ-KÓ$LÓMˆÕr5   rC   r]   c                 ó  — | j                   | j                  fz   }|j                  dd |k7  rt        d|j                  dd › d|› �«      ‚|j                  }t
        j                  }|j                  |«      }t        j                  ||¬«      }t        j                  || j                  d¬«      }t        j                  |d¬	«      }t        j                  || j                  d¬«      }	t        j                  |	d¬	«      }
t        j                  |
d
¬«      }||z  }||z
  j                  d«      }t        j                  || j                  d¬«      }t        j                  |d¬	«      }||z  }||z
  t        j                  || j                  z   «      z  }| j                   j                  |«      }dg|j#                  «       dz
  z  | j                  gz   }||j%                  |«      z  }||z  }|j                  |«      S )zÞApplies cumulative group norm, optionally using a mask.

        Args:
          hidden_states: Input tensor, shape [B, T, *feature_dims, C].

        Returns:
          Normalized tensor with the same shape as x.
        r"   NzInput tensor shape suffix z> does not match expected suffix (feature_dims + num_channels) r®   T©rH   rW   r$   r�   ro   )r
  )r,  r+  r°   r  r‹   r1   r‘   r�   Ú	ones_likeÚsumr0  ÚcumsumÚclamprY   ÚrsqrtrI   rR   rH   r  )rS   rC   Úexpected_input_suffixÚinput_dtypeÚ
calc_dtypeÚx_calcÚ	mask_calcÚsum_values_at_tÚcum_sum_valuesÚelements_in_group_at_tÚcum_count_elementsÚsafe_cum_count_elementsÚcum_meanÚsquared_diff_from_meanÚsum_sq_diff_at_tÚcum_sum_sq_diffÚcum_varianceÚnormalized_xÚscaleÚscale_view_shapeÚfinal_outputs                        r6   rc   z'Gemma3nAudioCumulativeGroupNorm.forwardL  sù  € ð !%× 1Ñ 1°T×5FÑ5FÐ4HÑ HÐØ×Ñ˜q˜rÐ"Ð&;Ò;ÜØ,¨]×-@Ñ-@ÀÀÐ-DÐ,Eð F9Ø9NÐ8OðQóð ð
 $×)Ñ)ˆä—]‘]ˆ
Ø×!Ñ! *Ó-ˆô —O‘O F°*Ô=ˆ	ô  Ÿ)™) F°×0CÑ0CÈTÔRˆäŸ™ o¸1Ô=ˆô "'§¡¨9¸$×:MÑ:MÐW[Ô!\Ðä"Ÿ\™\Ð*@ÀaÔHÐä"'§+¡+Ð.@ÀcÔ"JÐð "Ð$;Ñ;ˆð
 #)¨8Ñ"3×!8Ñ!8¸Ó!;ÐÜ Ÿ9™9Ð%;À×ATÑATÐ^bÔcÐô  Ÿ,™,Ð'7¸QÔ?ˆð 'Ð)@Ñ@ˆð  Ñ)¬U¯[©[¸ÈÏÉÑ9PÓ-QÑQˆð —‘—‘˜zÓ*ˆà˜3 -×"3Ñ"3Ó"5¸Ñ"9Ñ:¸d×>OÑ>OÐ=PÑPÐØ# e§j¡jÐ1AÓ&BÑBˆð $ iÑ/ˆà�‰˜{Ó+Ð+r5   )gü©ñÒMbP?)r-   r.   r/   r0   rd   r   r`   rN   r1   rf   rc   rg   rh   s   @r6   r*  r*  )  sT   ø„ ñð( ñ	NàðNð ˜s‘mðNð õ	Nð$G, U§\¡\ð G,°e·l±l÷ G,r5   r*  c                   ó~   ‡ — e Zd ZdZ	 d
dedededeeeeef   fˆ fd„Zdej                  dej                  fd	„Z
ˆ xZS )ÚGemma3nAudioSSCPConvBlockzÙA single convolution block for the SubSampleConvProjection.

    This block consists of a 2D convolution, followed by CumulativeGroupNorm,
    and a ReLU activation. It handles manual padding for the convolution.
    rk   ÚidxÚinput_freq_dimÚmanual_paddingc                 óJ  •— t         ‰| �  «        || _        || _        |dk(  rdn| j                  j                  |dz
     }| j                  j                  |   }| j                  j
                  |   \  }}| j                  j                  |   \  }	}
t        j                  ||||f|	|
fdd¬«      | _	        || j                  d   z   | j                  d   z   }||z
  |
z  dz   }t        ||f| j                  j                  ¬«      | _        t        j                  «       | _        y )Nr   r$   )r   r   F)Úin_channelsÚout_channelsÚkernel_sizeÚstrideÚpaddingrn   )r+  r,  rI   )rM   rN   rk   rO  Ússcp_conv_channel_sizeÚsscp_conv_kernel_sizeÚsscp_conv_stride_sizerO   ÚConv2dÚconvr*  Ússcp_conv_group_norm_epsÚnormÚReLUÚ
activation)rS   rk   rM  rN  rO  rQ  rR  Úkernel_hÚkernel_wÚstride_hÚstride_wÚf_in_paddedÚ
f_out_convrT   s                €r6   rN   z"Gemma3nAudioSSCPConvBlock.__init__�  s%  ø€ ô 	‰ÑÔØˆŒØ,ˆÔð  !š8‘a¨¯©×)KÑ)KÈCÐRSÉGÑ)TˆØ—{‘{×9Ñ9¸#Ñ>ˆØ!Ÿ[™[×>Ñ>¸sÑCÑˆ�(Ø!Ÿ[™[×>Ñ>¸sÑCÑˆ�(ä—I‘IØ#Ø%àØðð ˜hÐ'ØØô

ˆŒ	ð % t×':Ñ':¸1Ñ'=Ñ=À×@SÑ@SÐTUÑ@VÑVˆØ! HÑ,°Ñ9¸AÑ=ˆ
ä3Ø%Ø$˜Ø—‘×4Ñ4ô
ˆŒ	ô Ÿ'™'›)ˆ�r5   Úaudio_encodingsr]   c                 ó�  — t        j                  || j                  dd¬«      j                  | j                  j
                  j                  «      }| j	                  |«      }|j                  dddd«      j                  «       }| j                  |«      }|j                  dddd«      j                  «       }| j                  |«      S )NÚconstantrÄ   )ÚmodeÚvaluer   r"   r   r$   )ÚFr¡   rO  r�   rZ  rR   r‹   r²   rì   r\  r^  )rS   re  Úaudio_encodings_paddedÚaudio_encodings_convÚ
x_for_normÚx_normedÚaudio_encodings_normeds          r6   rc   z!Gemma3nAudioSSCPConvBlock.forwardÈ  s°   € ô "#§¡ ¸×8KÑ8KÐR\ÐdgÔ!h×!kÑ!kØ�I‰I×Ñ×"Ñ"ó"
Ðð
  $Ÿy™yÐ)?Ó@Ðð *×1Ñ1°!°Q¸¸1Ó=×HÑHÓJˆ
Ø—9‘9˜ZÓ(ˆà!)×!1Ñ!1°!°Q¸¸1Ó!=×!HÑ!HÓ!JÐØ�‰Ð5Ó6Ð6r5   ))r   r   r   r   )r-   r.   r/   r0   r%   rd   rE   rN   r1   rf   rc   rg   rh   s   @r6   rL  rL  –  sc   ø„ ñð 5Añ)$à"ð)$ð ð)$ð ð	)$ð
 ˜c 3¨¨SÐ0Ñ1õ)$ðV7 u§|¡|ð 7¸¿¹÷ 7r5   rL  c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )Ú#Gemma3nAudioSubSampleConvProjectionrk   c                 óp  •— t         ‰| �  «        || _        |j                  }g }g }t	        d«      D ]n  }|j
                  |   \  }}|j                  |   \  }}	d}
|dz
  }d}d}|||
|f}|j                  |«       ||z   |z   }||z
  |	z  dz   }|j                  |«       |}Œp t        d|j                  ||d   ¬«      | _	        t        d|d   ||d   ¬«      | _
        |j                  d   }|d   }||z  | _        t        j                  | j                  | j                  j                  d¬«      | _        y )Nr"   r   r$   )rM  rN  rk   rO  rV   Frm   )rM   rN   rk   Úinput_feat_sizer.  rW  rX  ÚappendrL  Úconv_0Úconv_1rV  Úinput_proj_in_featuresrO   r~   rv   Úinput_proj_linear)rS   rk   Úcurrent_f_for_block_inputÚcalculated_block_paddingÚcalculated_f_out_dimsÚir_  r`  ra  rb  Ú	pad_t_topÚpad_t_bottomÚ
pad_f_leftÚpad_f_rightÚmanual_padding_tuplerc  Úf_out_after_convÚfinal_c_outÚfinal_f_outrT   s                      €r6   rN   z,Gemma3nAudioSubSampleConvProjection.__init__Ü  s�  ø€ Ü‰ÑÔØˆŒà$*×$:Ñ$:Ð!Ø#%Ð Ø "Ðä�q“ò 	9ˆAØ!'×!=Ñ!=¸aÑ!@ÑˆH�hØ!'×!=Ñ!=¸aÑ!@ÑˆH�hð ˆIØ# a™<ˆLð ˆJØˆKð ØØØð	$Ð ð %×+Ñ+Ð,@ÔAð 4°jÑ@À;ÑNˆKØ +¨hÑ 6¸8ÑCÀaÑGÐØ!×(Ñ(Ð)9Ô:Ø(8Ñ%ð=	9ô@ 0ØØ!×1Ñ1ØØ3°AÑ6ô	
ˆŒô 0ØØ0°Ñ3ØØ3°AÑ6ô	
ˆŒð ×3Ñ3°BÑ7ˆØ+¨BÑ/ˆØ&1°KÑ&?ˆÔ#Ü!#§¡¨4×+FÑ+FÈÏÉ×H_ÑH_ÐfkÔ!lˆÕr5   re  r]   c                 ó   — |j                  d«      }| j                  |«      }| j                  |«      }|j                  \  }}}}|j	                  dddd«      j                  «       }|j                  ||||z  «      }	| j                  |	«      }
|
S )Nr$   r   r"   r   )r…   ru  rv  r°   r²   rì   r  rx  )rS   re  Úaudio_encodings_reshapedrá   rí   Úc_outÚt_outÚf_outÚ
x_permutedÚoutput_flattenedÚoutputs              r6   rc   z+Gemma3nAudioSubSampleConvProjection.forward  s�   € ð $3×#<Ñ#<¸QÓ#?Ð Ø�K‰KÐ0Ó1ˆØ�K‰K˜‹Nˆà!"§¡Ñˆˆ5�%˜à—Y‘Y˜q ! Q¨Ó*×5Ñ5Ó7ˆ
Ø%Ÿ?™?¨1¨e°U¸U±]ÓCÐØ×'Ñ'Ð(8Ó9ˆØˆr5   ©	r-   r.   r/   r%   rN   r1   rf   rc   rg   rh   s   @r6   rq  rq  Û  s.   ø„ ð7mÐ1õ 7mðr u§|¡|ð ¸¿¹÷ r5   rq  c                   ót   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  dej                  fd„Zˆ xZ	S )ÚGemma3nAudioConformerAttentionrk   c                 ó  •— t         ‰| �  «        || _        | j                  j                  | _        | j                  dt        j                  | j                  j                  «      d¬«       t        | j                  j                  «      | _
        t        |«      | _        t        j                  | j                  | j                  j                  d¬«      | _        t        | j                  j                  «      | _        y )NÚgradient_clippingFrr   rm   )rM   rN   rk   rv   Úpost_in_featuresr„   r1   rÖ   r‘  rG   Úpre_attn_normrÂ   ÚattnrO   r~   ÚpostÚ	post_norm©rS   rk   rT   s     €r6   rN   z'Gemma3nAudioConformerAttention.__init__%  s¯   ø€ Ü‰ÑÔØˆŒØ $§¡× 7Ñ 7ˆÔØ×ÑÐ0´%·,±,¸t¿{¹{×?\Ñ?\Ó2]ÐjoÐÔpÜ+¨D¯K©K×,CÑ,CÓDˆÔÜ)¨&Ó1ˆŒ	Ü—I‘I˜d×3Ñ3°T·[±[×5LÑ5LÐSXÔYˆŒ	Ü'¨¯©×(?Ñ(?Ó@ˆ�r5   re  r,   r]   c                 ó”  — |}t        j                  || j                   | j                  «      }| j                  |«      }| j	                  ||«      }|j
                  \  }}}}	|j                  ||||	z  «      }
| j                  |
«      }t        j                  || j                   | j                  «      }|| j                  |«      z   S r_   )	r1   r6  r‘  r“  r”  r°   r¢   r•  r–  )rS   re  r,   Úaudio_encodings_input_to_attnÚaudio_encodings_normÚaudio_encodings_attn_outrí   rî   ru   rx   r†  s              r6   rc   z&Gemma3nAudioConformerAttention.forward/  sÃ   € Ø(7Ð%ÜŸ+™+ o¸×8NÑ8NÐ7NÐPT×PfÑPfÓgˆØ#×1Ñ1°/ÓBÐà#'§9¡9Ð-AÀ>Ó#RÐ ð %=×$BÑ$BÑ!ˆˆ1ˆi˜Ø#;×#CÑ#CÀAÀqÈ)ÐV^ÑJ^Ó#_Ð àŸ)™)Ð$<Ó=ˆÜŸ+™+ o¸×8NÑ8NÐ7NÐPT×PfÑPfÓgˆØ,¨t¯~©~¸oÓ/NÑNÐNr5   ©
r-   r.   r/   r%   rN   r1   rf   r2   rc   rg   rh   s   @r6   r�  r�  $  sA   ø„ ðAÐ1õ AðO u§|¡|ð OÀU×EUÑEUð OÐZ_×ZfÑZf÷ Or5   r�  c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )Ú Gemma3nAudioConformerFeedForwardrk   c                 óŒ  •— t         ‰| �  «        || _        | j                  dt	        j
                  | j                  j                  «      d¬«       t        | j                  j                  «      | _	        t        j                  | j                  j                  | j                  j                  dz  d¬«      | _        t        j                  | j                  j                  dz  | j                  j                  d¬«      | _        t        | j                  j                  «      | _        | j                  j                  | _        y )Nr‘  Frr   r¯   rm   )rM   rN   rk   r„   r1   rÖ   r‘  rG   rv   Úpre_layer_normrO   r~   Úffw_layer_1Úffw_layer_2Úpost_layer_normÚconf_residual_weightÚpost_layer_scaler—  s     €r6   rN   z)Gemma3nAudioConformerFeedForward.__init__A  sÞ   ø€ Ü‰ÑÔØˆŒà×ÑÐ0´%·,±,¸t¿{¹{×?\Ñ?\Ó2]ÐjoÐÔpä,¨T¯[©[×-DÑ-DÓEˆÔÜŸ9™9 T§[¡[×%<Ñ%<¸d¿k¹k×>UÑ>UÐXYÑ>YÐ`eÔfˆÔÜŸ9™9 T§[¡[×%<Ñ%<¸qÑ%@À$Ç+Á+×BYÑBYÐ`eÔfˆÔÜ-¨d¯k©k×.EÑ.EÓFˆÔØ $§¡× @Ñ @ˆÕr5   re  r]   c                 ó   — |}t        j                  || j                   | j                  «      }| j                  |«      }| j	                  |«      }t
        j                  j                  |«      }| j                  |«      }t        j                  || j                   | j                  «      }| j                  |«      }||| j                  z  z   S r_   )r1   r6  r‘  r   r¡  rO   r    Úsilur¢  r£  r¥  )rS   re  Úresiduals      r6   rc   z(Gemma3nAudioConformerFeedForward.forwardM  sµ   € Ø"ˆÜŸ+™+ o¸×8NÑ8NÐ7NÐPT×PfÑPfÓgˆØ×-Ñ-¨oÓ>ˆØ(,×(8Ñ(8¸Ó(IˆÜŸ-™-×,Ñ,¨_Ó=ˆØ(,×(8Ñ(8¸Ó(IˆÜŸ+™+ o¸×8NÑ8NÐ7NÐPT×PfÑPfÓgˆØ×.Ñ.¨Ó?ˆØ˜?¨T×-BÑ-BÑBÑCÐCr5   r�  rh   s   @r6   rž  rž  @  s0   ø„ ð
AÐ1õ 
Að	D u§|¡|ð 	D¸¿¹÷ 	Dr5   rž  c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )Ú Gemma3nAudioConformerLightConv1drk   c           	      óÆ  •— t         ‰| �  «        || _        t        | j                  j                  | j                  j
                  ¬«      | _        t        j                  | j                  j                  | j                  j                  dz  d¬«      | _	        t        j                  | j                  j                  | j                  j                  | j                  j                  dd| j                  j                  d¬«      | _        | j                  dt        j                  | j                  j                   «      d¬	«       t        | j                  j                  | j                  j
                  ¬«      | _        t        j                  | j                  j                  | j                  j                  d¬«      | _        | j                  j                  dz
  | _        y )
N©rI   r"   Frm   r$   r   )rQ  rR  rS  rT  rU  Úgroupsrn   r‘  rr   )rM   rN   rk   rG   rv   Úrms_norm_epsr   rO   r~   Úlinear_startÚConv1dÚconf_conv_kernel_sizeÚdepthwise_conv1dr„   r1   rÖ   r‘  Ú	conv_normÚ
linear_endÚcausal_paddingr—  s     €r6   rN   z)Gemma3nAudioConformerLightConv1d.__init__Z  sD  ø€ Ü‰ÑÔØˆŒä,¨T¯[©[×-DÑ-DÈ$Ï+É+×JbÑJbÔcˆÔÜŸI™I d§k¡k×&=Ñ&=¸t¿{¹{×?VÑ?VÐYZÑ?ZÐafÔgˆÔÜ "§	¡	ØŸ™×/Ñ/ØŸ™×0Ñ0ØŸ™×9Ñ9ØØØ—;‘;×*Ñ*Øô!
ˆÔð 	×ÑÐ0´%·,±,¸t¿{¹{×?\Ñ?\Ó2]ÐjoÐÔpÜ'¨¯©×(?Ñ(?ÀTÇ[Á[×E]ÑE]Ô^ˆŒÜŸ)™) D§K¡K×$;Ñ$;¸T¿[¹[×=TÑ=TÐ[`ÔaˆŒà"Ÿk™k×?Ñ?À!ÑCˆÕr5   re  r]   c                 ó:  — |}| j                  |«      }| j                  |«      }t        j                  j                  j                  |d¬«      }|j                  ddd«      }t        j                  || j                  df«      }| j                  |«      }|j                  ddd«      }t        j                  || j                   | j                  «      }| j                  |«      }t        j                  j                  |«      }| j                  |«      }||z   }|S )NrV   r�   r   r"   r$   )r   r¯  r1   rO   r    Úglur²   rj  r¡   rµ  r²  r6  r‘  r³  r§  r´  )rS   re  Úaudio_encodings_residualÚaudio_encodings_permutedÚaudio_encodings_permuted_paddedrŒ  s         r6   rc   z(Gemma3nAudioConformerLightConv1d.forwardo  s  € Ø#2Ð à×-Ñ-¨oÓ>ˆØ×+Ñ+¨OÓ<ˆÜŸ(™(×-Ñ-×1Ñ1°/ÀrÐ1ÓJˆà#2×#:Ñ#:¸1¸aÀÓ#CÐ ä*+¯%©%Ð0HÈ4×K^ÑK^Ð`aÐJbÓ*cÐ'Ø×/Ñ/Ð0OÓPˆà)×1Ñ1°!°Q¸Ó:ˆÜŸ+™+ o¸×8NÑ8NÐ7NÐPT×PfÑPfÓgˆØŸ.™.¨Ó9ˆÜŸ-™-×,Ñ,¨_Ó=ˆØŸ/™/¨/Ó:ˆØ Ð#;Ñ;ˆØˆr5   r�  rh   s   @r6   rª  rª  Y  s-   ø„ ðDÐ1õ Dð* u§|¡|ð ¸¿¹÷ r5   rª  c                   ót   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  dej                  fd„Zˆ xZ	S )ÚGemma3nAudioConformerBlockrk   c                 ó¾  •— t         ‰| �  «        || _        t        | j                  «      | _        t        | j                  «      | _        t        | j                  «      | _        t        | j                  «      | _	        | j                  dt        j                  | j                  j                  «      d¬«       t        | j                  j                  «      | _        y )Nr‘  Frr   )rM   rN   rk   rž  Úffw_layer_startr�  Ú	attentionrª  Úlconv1dÚffw_layer_endr„   r1   rÖ   r‘  rG   rv   r\  r—  s     €r6   rN   z#Gemma3nAudioConformerBlock.__init__…  s—   ø€ Ü‰ÑÔØˆŒä?ÀÇÁÓLˆÔÜ7¸¿¹ÓDˆŒÜ7¸¿¹ÓDˆŒÜ=¸d¿k¹kÓJˆÔØ×ÑÐ0´%·,±,¸t¿{¹{×?\Ñ?\Ó2]ÐjoÐÔpÜ" 4§;¡;×#:Ñ#:Ó;ˆ�	r5   re  r,   r]   c                 ój  — | j                  |«      }| j                  ||«      }| }||j                  d«      j                  |j                  «      z  }| j                  |«      }| j                  |«      }t        j                  || j                   | j                  «      }| j                  |«      }|S )NrV   )r¾  r¿  r…   r�   r‹   rÀ  rÁ  r1   r6  r‘  r\  )rS   re  r,   Úvalidity_mask_for_lconvÚaudio_encodings_for_lconv_inputrŒ  s         r6   rc   z"Gemma3nAudioConformerBlock.forward�  s®   € Ø×.Ñ.¨Ó?ˆØŸ.™.¨¸.ÓIˆØ#1 /ÐØ*9Ð<S×<]Ñ<]Ð^`Ó<a×<dÑ<dØ×!Ñ!ó=
ñ +
Ð'ð Ÿ,™,Ð'FÓGˆà×,Ñ,¨_Ó=ˆÜŸ+™+ o¸×8NÑ8NÐ7NÐPT×PfÑPfÓgˆØ—‘˜?Ó+ˆØˆr5   rœ  rh   s   @r6   r¼  r¼  „  s;   ø„ ð	<Ð1õ 	<ð u§|¡|ð ÀU×EUÑEUð ÐZ_×ZfÑZf÷ r5   r¼  c            	       óZ   ‡ — e Zd ZdZd	dedededefˆ fd„Zdej                  fˆ fd„Z	ˆ xZ
S )
ÚGemma3nTextScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scalec                 ó„   •— t         ‰| �  |||«       || _        | j                  dt	        j
                  |«      d¬«       y )NrÊ  Frr   )rM   rN   Úscalar_embed_scaler„   r1   rÖ   )rS   rÇ  rÈ  rÉ  rÊ  rT   s        €r6   rN   z'Gemma3nTextScaledWordEmbedding.__init__¤  s;   ø€ Ü‰Ñ˜¨¸ÔDØ"-ˆÔØ×Ñ˜]¬E¯L©L¸Ó,EÐRWÐÕXr5   Ú	input_idsc                 ó‚   •— t         ‰| �  |«      | j                  j                  | j                  j
                  «      z  S r_   )rM   rc   rÊ  r�   rR   r‹   )rS   rÍ  rT   s     €r6   rc   z&Gemma3nTextScaledWordEmbedding.forward©  s2   ø€ Ü‰w‰˜yÓ)¨D×,<Ñ,<×,?Ñ,?ÀÇÁ×@QÑ@QÓ,RÑRÐRr5   )ro   )r-   r.   r/   r0   rd   r`   rN   r1   rf   rc   rg   rh   s   @r6   rÆ  rÆ  Ÿ  sG   ø„ ññY sð Y¸3ð YÈSð YÐ_dõ Yð
S §¡÷ Sñ Sr5   rÆ  c                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚGemma3nTextLaurelBlockz Learned Augmented Residual Layerrk   c                 ó¸  •— t         ‰| �  «        || _        t        j                  | j                  j
                  | j                  j                  d¬«      | _        t        j                  | j                  j                  | j                  j
                  d¬«      | _        t        | j                  j
                  | j                  j                  ¬«      | _        y )NFrm   r¬  )rM   rN   rk   rO   r~   rv   Úlaurel_rankÚlinear_leftÚlinear_rightrG   r®  Úpost_laurel_normr—  s     €r6   rN   zGemma3nTextLaurelBlock.__init__°  sŒ   ø€ Ü‰ÑÔØˆŒäŸ9™9 T§[¡[×%<Ñ%<¸d¿k¹k×>UÑ>UÐ\aÔbˆÔÜŸI™I d§k¡k×&=Ñ&=¸t¿{¹{×?VÑ?VÐ]bÔcˆÔÜ .¨t¯{©{×/FÑ/FÈDÏKÉK×LdÑLdÔ eˆÕr5   rC   r]   c                 ór   — | j                  |«      }| j                  |«      }| j                  |«      }||z   S r_   )rÓ  rÔ  rÕ  )rS   rC   Úlaurel_hidden_statesÚnormed_laurel_hidden_statess       r6   rc   zGemma3nTextLaurelBlock.forward¸  sC   € Ø-1×-=Ñ-=¸mÓ-LÐØ-1×->Ñ->Ð?SÓ-TÐØ&*×&;Ñ&;Ð<PÓ&QÐ#ØÐ:Ñ:Ð:r5   )
r-   r.   r/   r0   r'   rN   r1   rf   rc   rg   rh   s   @r6   rÐ  rÐ  ­  s0   ø„ Ù*ðfÐ0õ fð; U§\¡\ð ;°e·l±l÷ ;r5   rÐ  c                   óš   ‡ — e Zd Zd	dedefˆ fd„Zdej                  dej                  fd„Zdej                  dej                  fd„Z	ˆ xZ
S )
ÚGemma3nTextMLPrk   Ú	layer_idxc                 óø  •— t         ‰| �  «        || _        |j                  | _        |j                  |   | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _	        t        |j                     | _        |j                  |   | _        y ©NFrm   )rM   rN   rk   rv   Úintermediate_sizerO   r~   Ú	gate_projÚup_projÚ	down_projr
   Úhidden_activationÚact_fnÚactivation_sparsity_patternÚactivation_sparsity©rS   rk   rÛ  rT   s      €r6   rN   zGemma3nTextMLP.__init__À  s¾   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9¸)Ñ!DˆÔÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×5Ñ5Ñ6ˆŒØ#)×#EÑ#EÀiÑ#PˆÕ r5   rC   r]   c                 óÔ   — | j                  |«      }| j                  dkD  r| j                  |«      }| j                  |«      }| j	                  |«      }| j                  ||z  «      }|S )NrÄ   )rß  rå  Ú_gaussian_topkrã  rà  rá  )rS   rC   rß  Úactivationsrà  rá  s         r6   rc   zGemma3nTextMLP.forwardË  sc   € Ø—N‘N =Ó1ˆ	Ø×#Ñ# cÒ)Ø×+Ñ+¨IÓ6ˆIØ—k‘k )Ó,ˆØ—,‘,˜}Ó-ˆØ—N‘N ;°Ñ#8Ó9ˆ	ØÐr5   Úinputsc                 óØ  — t        j                  | j                  t         j                  |j                  ¬«      }t         j
                  j                  j                  dd«      }|j                  |«      }|j                  |j                  «      }t        j                  |dd¬«      }t        j                  |ddd¬«      }|||z  z   }t        j                  j                  ||z
  «      S )	N©r‹   rŽ   r   r$   rV   Tr2  F)rH   rW   Úunbiased)r1   rÖ   rå  r‘   rŽ   ÚdistributionsÚnormalÚNormalÚicdfr•   r‹   rZ   ÚstdrO   r    Úrelu)rS   rê  Útarget_sparsity_tensorÚnormal_distÚstd_multiplierÚinputs_meanÚ
inputs_stdÚcutoff_xs           r6   rè  zGemma3nTextMLP._gaussian_topkÔ  sº   € Ü!&§¡¨d×.FÑ.FÌeÏmÉmÐdj×dqÑdqÔ!rÐô ×)Ñ)×0Ñ0×7Ñ7¸¸1Ó=ˆØ'2×'7Ñ'7Ð8NÓ'OˆØ'×,Ñ,¨V¯\©\Ó:ˆÜ—j‘j ¨R¸Ô>ˆÜ—Y‘Y˜v¨2°tÀeÔLˆ
Ø ¨nÑ!<Ñ<ˆÜ�}‰}×!Ñ! &¨8Ñ"3Ó4Ð4r5   )r   )r-   r.   r/   r'   rd   rN   r1   rf   rc   rè  rg   rh   s   @r6   rÚ  rÚ  ¿  sP   ø„ ñ	QÐ0ð 	Q¸Sõ 	Qð U§\¡\ð °e·l±ló ð5 U§\¡\ð 5°e·l±l÷ 5r5   rÚ  c                   óX  ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zdej                  dej                  fd„Z	d	ej                  d
ej                  dej                  fd„Z
dej                  dej                  fd„Zdej                  dej                  fd„Zˆ xZS )ÚGemma3nTextAltUpa�  Alternating Updates (AltUp)

    The AltUp module wraps transformer layers. The `predict` step modifies the
    input to the transformer layer, and the `correct` step propagates the output
    of the transformer layer to the sparsely updated dimensions.

    See more in the research paper:

    https://proceedings.neurips.cc/paper_files/paper/2023/file/f2059277ac6ce66e7e5543001afa8bb5-Paper-Conference.pdf
    rk   c                 óF  •— t         ‰| �  «        || _        t        j                  t        j                  | j                  j                  «      «      | _        t        j                  | j                  j                  | j                  j                  d¬«      | _        t        j                  | j                  j                  | j                  j                  dz  d¬«      | _        t        j                  | j                  j                  | j                  j                  d¬«      | _        t        | j                  j                  | j                  j                  ¬«      | _        | j#                  dt        j$                  | j                  j                  dz  «      d¬«       y )NFrm   r"   r¬  Úrouter_input_scaleç      ð¿rr   )rM   rN   rk   rO   rP   r1   rÐ   rv   Úcorrect_output_scaler~   Úaltup_num_inputsÚcorrection_coefsÚprediction_coefsÚmodality_routerrG   r®  Úrouter_normr„   rÖ   r—  s     €r6   rN   zGemma3nTextAltUp.__init__ñ  s  ø€ Ü‰ÑÔØˆŒÜ$&§L¡L´·±¸T¿[¹[×=TÑ=TÓ1UÓ$VˆÔ!Ü "§	¡	¨$¯+©+×*FÑ*FÈÏÉ×HdÑHdÐkpÔ qˆÔÜ "§	¡	¨$¯+©+×*FÑ*FÈÏÉ×HdÑHdÐfgÑHgÐnsÔ tˆÔÜ!Ÿy™y¨¯©×)@Ñ)@À$Ç+Á+×B^ÑB^ÐejÔkˆÔÜ)¨$¯+©+×*AÑ*AÀtÇ{Á{×G_ÑG_Ô`ˆÔØ×ÑÐ1´5·<±<ÀÇÁ×@WÑ@WÐY]Ñ@]Ó3^ÐkpÐÕqr5   rá   r]   c                 óÄ   — | j                  |«      | j                  z  }| j                  |«      }t        j                  |j                  «       «      j                  |«      S r_   )r  rý  r  r1   r  r`   ra   )rS   rá   Úrouter_inputsÚrouteds       r6   Úcompute_router_modalitiesz*Gemma3nTextAltUp.compute_router_modalitiesû  sM   € Ø×(Ñ(¨Ó+¨d×.EÑ.EÑEˆØ×%Ñ% mÓ4ˆÜ�z‰z˜&Ÿ,™,›.Ó)×1Ñ1°!Ó4Ð4r5   rC   c                 óà  — | j                  || j                  j                     «      }| j                  ro| j                  j                  �Y| j
                  j                  j                  j                  | j                  j                   | j                  j                  «        | j                  |«      j                  g |j                  dd ¢| j                  j                  ‘| j                  j                  ‘­Ž j                  dddd«      }t        j                  |j                  dddd«      |«      }|j                  dddd«      }||z  }|j                  «       j!                  |«      S )aµ  Predicts the output of a layer using a trainable map.

        Args:
            hidden_states: A 4D tensor of shape `[num_altup_inputs, batch_size, num_tokens, hidden_size]` derived by
                stacking the input embeddings and preprocessing the last `num_altup_inputs - 1` matrices.

        Returns:
            A 4D tensor of shape `[num_altup_inputs, batch_size, num_tokens, hidden_size]` containing the predictions.
        NrV   r   r$   r   r"   )r  rk   Úaltup_active_idxÚtrainingÚaltup_coef_clipr  rR   ÚdataÚclamp_r¢   r°   r   r²   r1   r³   rì   ra   )rS   rC   Ú
modalitiesÚ	all_coefsÚpredictionss        r6   ÚpredictzGemma3nTextAltUp.predict   s@  € ð ×3Ñ3°MÀ$Ç+Á+×B^ÑB^Ñ4_Ó`ˆ
à�=Š=˜TŸ[™[×8Ñ8ÐDØ×!Ñ!×(Ñ(×-Ñ-×4Ñ4°d·k±k×6QÑ6QÐ5QÐSW×S^ÑS^×SnÑSnÔoðˆD×!Ñ! *Ó-ß‰WðiØ ×&Ñ& s¨Ð+ðiØ-1¯[©[×-IÑ-IðiØKOÏ;É;×KgÑKgòiç‰W�Q˜˜1˜aÓ ð 	ô —l‘l =×#8Ñ#8¸¸A¸qÀ!Ó#DÀiÓPˆØ!×)Ñ)¨!¨Q°°1Ó5ˆØ�}Ñ$ˆØ×%Ñ%Ó'×/Ñ/°Ó>Ð>r5   r  Ú	activatedc                 óÎ  — | j                  |«      }||| j                  j                     z
  }|j                  | j                  j                  ddd«      }| j
                  r•| j                  j                  �| j                  j                  j                  | j                  j                   | j                  j                  «      }t        j                  j                  j                  ||d¬«      dz   }n| j                  |«      dz   }|j                  ddd«      j                  d«      }t        j                   ||«      }||z  }|j#                  «       j%                  |«      S )a_  Corrects the predictions relative to the

        Args:
            predictions: A 4D tensor of shape `[num_altup_inputs, batch_size, num_tokens, hidden_size]` derived by
                stacking the input embeddings and preprocessing the last `num_altup_inputs - 1` matrices.
            activated: A 3D tensor of shape `[batch_size, num_tokens, hidden_size]` containing the activated inputs.

        Returns:
            A 4D tensor of shape `[num_altup_inputs, batch_size, num_tokens, hidden_size]` correcting the original
                predictions relative to the activated input embeddings.
        r$   Nrm   ro   r"   r   rV   )r  rk   r
  Úrepeatr   r  r  r  rR   r6  r1   rO   r    Úlinearr²   r…   Úmulrì   ra   )rS   r  r  r  Ú
innovationrR   r  Ú	correcteds           r6   ÚcorrectzGemma3nTextAltUp.correct  s,  € ð ×3Ñ3°IÓ>ˆ
Ø ¨T¯[©[×-IÑ-IÑ!JÑJˆ
Ø×&Ñ& t§{¡{×'CÑ'CÀQÈÈ1ÓMˆ
à�=Š=˜TŸ[™[×8Ñ8ÐDØ×*Ñ*×1Ñ1×7Ñ7¸¿¹×9TÑ9TÐ8TÐVZ×VaÑVa×VqÑVqÓrˆFÜŸ™×+Ñ+×2Ñ2°:¸vÈDÐ2ÓQÐTWÑW‰Ià×-Ñ-¨jÓ9¸CÑ?ˆIð
 ×%Ñ% a¨¨AÓ.×8Ñ8¸Ó<ˆ	ä—I‘I˜j¨)Ó4ˆ	Ø�[Ñ ˆ	Ø×#Ñ#Ó%×-Ñ-¨iÓ8Ð8r5   r  c                 óp   — |j                  | j                  «      | j                  z  j                  |«      S )a	  
        This is only defined as the `forward` so that accelerate hooks can move correctly `correct_output_scale`
        (which is a nn.Parameter, not a Module) between devices when offloading. It is otherwise only used in
        `scale_corrected_output`
        )ra   rÿ  ©rS   r  s     r6   rc   zGemma3nTextAltUp.forward;  s2   € ð ×!Ñ! $×";Ñ";Ó<¸t×?XÑ?XÑX×aÑaÐbkÓlÐlr5   c                 ó$   — | j                  |«      S )zMScales the provided 3D tensor of shape [batch_size, num_tokens, hidden_size].)rc   r  s     r6   Úscale_corrected_outputz'Gemma3nTextAltUp.scale_corrected_outputC  s   € à�|‰|˜IÓ&Ð&r5   )r-   r.   r/   r0   r'   rN   r1   rf   r  r  r  rc   r  rg   rh   s   @r6   rû  rû  å  s¬   ø„ ñ	ðrÐ0õ rð5¨5¯<©<ð 5¸E¿L¹Ló 5ð
? U§\¡\ð ?°e·l±ló ?ð89 5§<¡<ð 9¸E¿L¹Lð 9ÈUÏ\É\ó 9ð>m §¡ð m°%·,±,ó mð'°·±ð 'ÀÇÁ÷ 'r5   rû  c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..NrV   r"   r�   )r°   r1   r’   )rá   Úx1Úx2s      r6   Úrotate_halfr"  H  sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r5   rC   Ún_repr]   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r$   N)r°   Úexpandr¢   )rC   r#  ræ   Únum_key_value_headsÚslenrx   s         r6   Ú	repeat_kvr(  O  so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr5   ÚmoduleÚqueryÚkeyri  Úattention_maskÚdropoutÚscalingrÇ   c                 ó|  — |€| j                   dz  }t        || j                  «      }	t        || j                  «      }
t        j                  ||	j                  dd«      «      |z  }|�||z  }t        j                  |«      }||z  }|�||z   }t        j                  j                  |dt        j                  ¬«      j                  |j                  «      }t        j                  j                  ||| j                  ¬«      }t        j                  ||
«      }|j                  dd«      j                  «       }||fS )NrX   r"   r   rV   r  )Úpr  r$   )rx   r(  Únum_key_value_groupsr1   r³   Ú	transposer  rO   r    r  r‘   r�   r‹   r-  r  rì   )r)  r*  r+  ri  r,  r-  r.  rÇ   Úkwargsr  r  Úattn_weightsÚattn_outputs                r6   Úeager_attention_forwardr6  [  s  € ð €Ø—/‘/ 4Ñ'ˆä˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LàÐØ# gÑ-ˆÜ—z‘z ,Ó/ˆØ# gÑ-ˆØÐ!Ø# nÑ4ˆô —=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØ˜Ð$Ð$r5   rá   r”   r“   Úunsqueeze_dimc                 ón   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   S )a\  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        x (`torch.Tensor`): The tensor to embed.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )r…   r"  )rá   r”   r“   r7  s       r6   Úapply_rotary_pos_embr9  }  s8   € ð" �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�‰Gœ A›¨Ñ,Ñ-Ð-r5   c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 ddej                  dej                  dej                  dz  dedz  d	e	ee
ej                  ej                  f   f   dz  d
ee   de
ej                  ej                  dz  f   fd„Zˆ xZS )ÚGemma3nTextAttentionrk   rÛ  c                 óÄ  •— t         ‰| �  «        || _        || _        t	        |d«      r|j
                  |   nd | _        | j                  dk(  | _        | j                  r|j                  nd | _        t        |d|j                  |j                  z  «      | _        |j                  |j                  z  | _        d| _        | j                  j                   | _        d| _        | j                  j$                  | j                  j&                  z
  }||cxk\  xr dkD  nc | _        |j
                  d | }| j(                  r@t+        |«      dz
  |d d d…   j-                  |j
                  |   «      z
  | _        d	| _        nBd | _        |t+        |«      dz
  |d d d…   j-                  |j
                  |   «      z
  k(  | _        t3        j4                  |j                  |j                  | j                  z  |j6                  ¬
«      | _        t;        |j                  |j<                  ¬«      | _        | j(                  sÝt3        j4                  |j                  |j                  | j                  z  |j6                  ¬
«      | _         t3        j4                  |j                  |j                  | j                  z  |j6                  ¬
«      | _!        t;        |j                  |j<                  ¬«      | _"        t;        |j                  |j<                  d	¬«      | _#        t3        j4                  |j                  | j                  z  |j                  |j6                  ¬
«      | _$        y )NÚlayer_typesÚsliding_attentionrx   ro   Tr   r$   rV   Frm   )rH   rI   )rH   rI   rJ   )%rM   rN   rk   rÛ  Úhasattrr=  Ú
layer_typeÚ
is_slidingÚsliding_windowÚgetattrrv   Únum_attention_headsrx   r&  r1  r.  Úattention_dropoutÚ	is_causalÚnum_hidden_layersÚnum_kv_shared_layersÚis_kv_shared_layerr/  ÚindexÚkv_shared_layer_indexÚstore_full_length_kvrO   r~   Úattention_biasrÒ   rG   r®  Úq_normrÓ   rÔ   Úk_normÚv_normÚo_proj)rS   rk   rÛ  Úfirst_kv_shared_layer_idxÚprev_layersrT   s        €r6   rN   zGemma3nTextAttention.__init__”  sÂ  ø€ Ü‰ÑÔØˆŒØ"ˆŒÜ;BÀ6È=Ô;Y˜&×,Ñ,¨YÒ7Ð_cˆŒØŸ/™/Ð-@Ñ@ˆŒØ7;·²˜f×3Ò3ÈDˆÔä ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!ØˆŒØ!%§¡×!>Ñ!>ˆÔØˆŒà$(§K¡K×$AÑ$AÀDÇKÁK×DdÑDdÑ$dÐ!Ø"+Ð/HÖ"LÈ1Ô"LˆÔØ×(Ñ(Ð)CÐ*CÐDˆØ×"Ò"ä),¨[Ó)9¸AÑ)=ÀÉDÈbÈDÑ@Q×@WÑ@WÐX^×XjÑXjÐktÑXuÓ@vÑ)vˆDÔ&Ø(-ˆDÕ%à)-ˆDÔ&à(1´S¸Ó5EÈÑ5IÈKÑX\ÐZ\ÐX\ÑL]×LcÑLcØ×"Ñ" 9Ñ-óMñ 6ñ )ˆDÔ%ô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô %¨¯©¸f×>QÑ>QÔRˆŒð ×&Ò&ÜŸ)™)Ø×"Ñ" F×$>Ñ$>ÀÇÁÑ$NÐU[×UjÑUjôˆDŒKô Ÿ)™)Ø×"Ñ" F×$>Ñ$>ÀÇÁÑ$NÐU[×UjÑUjôˆDŒKô )¨V¯_©_À&×BUÑBUÔVˆDŒKÜ(¨V¯_©_À&×BUÑBUÐbgÔhˆDŒKä—i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆ�r5   NrC   Úposition_embeddingsr,  rB   Úshared_kv_statesr3  r]   c                 ó¼  — |j                   d d }g |¢d‘| j                  j                  ‘­}|\  }	}
| j                  |«      j	                  |«      }| j                  |«      }t        ||	|
d¬«      }|j                  dd«      }| j                  rI|| j                     \  }}|j                  |j                  «      }|j                  |j                  «      }n•| j                  |«      j	                  |«      }| j                  |«      }t        ||	|
d¬«      }|j                  dd«      }| j                  |«      j	                  |«      }| j                  |«      }|j                  dd«      }|�,| j                  s |j!                  ||| j"                  «      \  }}| j$                  r||f|| j"                  <   t'        j(                  | j                  j*                  t,        «      } || ||||f| j.                  r| j0                  nd| j2                  | j4                  dœ|¤Ž\  }} |j6                  g |¢d‘­Ž j9                  «       }| j;                  |«      }||fS )NrV   r"   )r7  r$   rÄ   )r-  r.  rB  )r°   rk   rx   rÒ   r  rN  r9  r2  rI  rK  r�   rŽ   rÓ   rO  rÔ   rP  ÚupdaterÛ  rL  r   Úget_interfaceÚ_attn_implementationr6  r  rE  r.  rB  r¢   rì   rQ  )rS   rC   rT  r,  rB   rU  r3  Úinput_shapeÚhidden_shaper”   r“   r  r  r  Úattention_interfacer5  r4  s                    r6   rc   zGemma3nTextAttention.forwardÄ  sN  € ð $×)Ñ)¨#¨2Ð.ˆØ?˜Ð? bÐ?¨$¯+©+×*>Ñ*>Ñ?ˆà&‰ˆˆSØ—{‘{ =Ó1×6Ñ6°|ÓDˆØ—{‘{ <Ó0ˆÜ+¨L¸#¸sÐRSÔTˆØ#×-Ñ-¨a°Ó3ˆð
 ×"Ò"Ø'7¸×8RÑ8RÑ'SÑ$ˆJ˜à#Ÿ™ |×':Ñ':Ó;ˆJØ'Ÿ?™?¨<×+>Ñ+>Ó?‰LàŸ™ ]Ó3×8Ñ8¸ÓFˆJØŸ™ ZÓ0ˆJÜ-¨j¸#¸sÐRSÔTˆJØ#×-Ñ-¨a°Ó3ˆJàŸ;™; }Ó5×:Ñ:¸<ÓHˆLØŸ;™; |Ó4ˆLØ'×1Ñ1°!°QÓ7ˆLàÐ&¨t×/FÒ/FØ'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜Ø×$Ò$Ø/9¸<Ð/GÐ˜TŸ^™^Ñ,ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð /3¯mªm�D×*Ò*ÀØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r5   ©NN)r-   r.   r/   r'   rd   rN   r1   rf   r   ÚdictrE   r   r   rc   rg   rh   s   @r6   r;  r;  “  s¼   ø„ ð.
Ð0ð .
¸Sõ .
ðj )-ØPTñ;)à—|‘|ð;)ð #Ÿ\™\ð;)ð Ÿ™ tÑ+ð	;)ð
  ™ð;)ð ˜s E¨%¯,©,¸¿¹Ð*DÑ$EÐEÑFÈÑMð;)ð Ð+Ñ,ð;)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷;)r5   r;  c                   óv  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 ddej                  dej                  dej                  deee	ej                  ej                  f   f   dz  d	ej                  dz  d
ej                  dz  dedz  dee   de	ej                  e	ej                  ej                  f   dz  f   fd„Zˆ xZS )ÚGemma3nTextDecoderLayerrk   rÛ  c                 óx  •— t         ‰| �  «        || _        |j                  | _        || _        t        ||«      | _        t        ||¬«      | _        t        | j                  |j                  ¬«      | _        t        | j                  |j                  ¬«      | _        t        | j                  |j                  ¬«      | _        t        | j                  |j                  ¬«      | _        |j                  | _        t         |j"                     | _        t'        |«      | _        t+        |«      | _        t/        j0                  | j                  | j                  d¬«      | _        t/        j0                  | j                  | j                  d¬«      | _        t        | j                  |j                  ¬«      | _        y )N)rÛ  r¬  Frm   )rM   rN   rk   rv   rÛ  r;  Ú	self_attnrÚ  ÚmlprG   r®  Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormÚhidden_size_per_layer_inputr
   râ  rã  rû  ÚaltuprÐ  ÚlaurelrO   r~   Úper_layer_input_gateÚper_layer_projectionÚpost_per_layer_input_normræ  s      €r6   rN   z Gemma3nTextDecoderLayer.__init__  sJ  ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ"ˆŒÜ-¨f°iÓ@ˆŒÜ! &°IÔ>ˆŒÜ-¨d×.>Ñ.>ÀF×DWÑDWÔXˆÔÜ(6°t×7GÑ7GÈV×M`ÑM`Ô(aˆÔ%Ü)7¸×8HÑ8HÈf×NaÑNaÔ)bˆÔ&Ü*8¸×9IÑ9IÈv×ObÑObÔ*cˆÔ'à+1×+MÑ+MˆÔ(Ü˜V×5Ñ5Ñ6ˆŒä% fÓ-ˆŒ
Ü,¨VÓ4ˆŒÜ$&§I¡I¨d×.>Ñ.>À×@`Ñ@`ÐglÔ$mˆÔ!Ü$&§I¡I¨d×.NÑ.NÐPT×P`ÑP`ÐglÔ$mˆÔ!Ü)7¸×8HÑ8HÈf×NaÑNaÔ)bˆÕ&r5   NrC   rT  Úper_layer_inputrU  r,  Úposition_idsrB   r3  r]   c           
      ót  — | j                   j                  |«      }	|	| j                  j                     }
| j	                  |
«      }| j                  |«      } | j                  d||||||dœ|¤Ž\  }}| j                  |«      }|
|z   }||z   t        j                  d«      z  }| j                  |«      }| j                  |«      }| j                  |«      }||z   }| j                   j                  |	|«      }|| j                  j                     j                  «       }| j                  j                  r| j                   j!                  |«      }| j#                  |«      }| j%                  |«      }t'        j(                  ||«      }| j+                  |«      }| j-                  |«      }|dd xxx |z  ccc |S )N)rC   r,  rU  ro  rT  rB   r"   r$   r4   )ri  r  rk   r
  rd  rj  rb  re  r€   Úsqrtrf  rc  rg  r  r×   Úaltup_correct_scaler  rk  rã  r1   Úmultiplyrl  rm  )rS   rC   rT  rn  rU  r,  ro  rB   r3  r  Úactive_predictionÚactive_prediction_normedÚlaurel_outputr”  r´   Ú
attn_gatedÚattn_laurelÚ	attn_normÚattn_ffwÚattn_ffw_normÚattn_ffw_laurel_gatedÚcorrected_predictionsÚfirst_predictions                          r6   rc   zGemma3nTextDecoderLayer.forward  s½  € ð —j‘j×(Ñ(¨Ó7ˆØ'¨¯©×(DÑ(DÑEÐà#'×#7Ñ#7Ð8IÓ#JÐ ØŸ™Ð$<Ó=ˆà �$—.‘.ð 
Ø2Ø)Ø-Ø%Ø 3Ø+ñ
ð ñ
‰ˆˆað ×,Ñ,¨TÓ2ˆà&¨Ñ-ˆ
Ø! MÑ1´T·Y±Y¸q³\ÑAˆà×2Ñ2°;Ó?ˆ	Ø—8‘8˜IÓ&ˆØ×7Ñ7¸ÓAˆØ +¨mÑ ;ÐØ $§
¡
× 2Ñ 2°;Ð@UÓ VÐà0°·±×1MÑ1MÑN×TÑTÓVÐØ�;‰;×*Ò*Ø#Ÿz™z×@Ñ@ÐAQÓRÐð  ×4Ñ4Ð5EÓFÐØŸ;™;Ð'7Ó8ÐÜ Ÿ>™>Ð*:¸OÓLÐð  ×4Ñ4Ð5EÓFÐØ×9Ñ9Ð:JÓKÐØ˜a˜bÓ!Ð%5Ñ5Ó!à$Ð$r5   )NNNNNN)r-   r.   r/   r'   rd   rN   r1   rf   r^  rE   Ú
LongTensorr   r   r   r=   rc   rg   rh   s   @r6   r`  r`    sÿ   ø„ ðcÐ0ð c¸Sõ cð0 -1Ø(,ØPTØ.2Ø04Ø(,ñ3%à—|‘|ð3%ð #Ÿ\™\ð3%ð Ÿ™ð	3%ð
 ˜s E¨%¯,©,¸¿¹Ð*DÑ$EÐEÑFÈÑMð3%ð Ÿ™ tÑ+ð3%ð ×&Ñ&¨Ñ-ð3%ð  ™ð3%ð Ð+Ñ,ð3%ð 
ˆu�|‰|˜U 5×#4Ñ#4°e×6GÑ6GÐ#GÑHÈ4ÑOÐOÑ	P÷3%r5   r`  c            	       ó  ‡ — e Zd ZU eed<   dZdZdgZddgZdZ	dZ
dZdZdZeedœZdZ ej&                  «       ˆ fd	„«       Zd
„ Zd„ Z	 	 	 ddedz  dedz  dedej4                  fˆ fd„Z	 	 	 ddedz  dedz  defd„Zˆ xZS )ÚGemma3nPreTrainedModelrk   ÚmodelTr`  rB   rU  )rC   rD   )ÚimageÚtextÚaudioc                 óF  •— t         ‰| �  |«       t        |t        «      r!t	        j
                  |j                  «       �nŸt        |t        «      rêt	        j                  |j                  «       |j                  dz  }dt        j                  j                  j                  t        j                  d«      «      z  }t	        j                   |j"                  ||z  «       t	        j$                  |j&                  |j(                  «       t	        j                   |j*                  |j-                  «       «       �n¥t        |t.        «      r,t	        j$                  |j0                  |j2                  «       �nit        |t4        «      rXt	        j                  |j6                  «       t	        j$                  |j8                  | j:                  j<                  dz  «       �nt        |t>        «      rËd\  }}|j@                  dz  }tC        jD                  tG        |«      tG        |«      z  «      tI        |dz
  d«      z  }|t        jJ                  t        jL                  |«      | z  «      z  }t	        j                   |jN                  |jG                  «       jQ                  d«      jQ                  d«      «       �n&t        |tR        «      rdt	        j$                  |jT                  | j<                  dz  «       t	        j$                  |jV                  dtC        jX                  d	«      z  «       n²t        |tZ        «      r¢|j\                  D ]“  }	|j^                  }
|j`                  |	   d
k7  rtb        |j`                  |	      }
 |
|j:                  |	¬«      \  }}t	        j                   te        ||	› d�«      |«       t	        j                   te        ||	› d�«      |«       Œ• tg        |d«      r5t	        j$                  |jh                  | j:                  jh                  «       y y )NrX   ro   rÄ   rþ  )ro   rp   r"   r$   r   ç       @Údefault©r@  Ú	_inv_freqÚ_original_inv_freqr‘  )5rM   Ú_init_weightsÚ
isinstancer*  ÚinitÚones_rR   rÂ   Úzeros_rÑ   rx   r1   rO   r    rÕ   rÖ   Úcopy_rÅ   Ú	constant_rÇ   rÍ   rÆ   rÙ   rÆ  rÊ  rÌ  rû  rÿ  rý  rk   rv   rj   rw   r€   r�   r`   ry   r‚   rƒ   rq   r…   ÚGemma3nTextModelÚper_layer_projection_scaleÚper_layer_input_scalerq  ÚGemma3nRotaryEmbeddingr=  Úcompute_default_rope_parametersÚ	rope_typer   rC  r?  r‘  )rS   r)  rÅ   rÚ   r†   r‡   rˆ   r‰   rq   r@  Úrope_init_fnÚcurr_inv_freqr´   rT   s                €r6   rŒ  z$Gemma3nPreTrainedModel._init_weightsa  s  ø€ ä‰Ñ˜fÔ%Ü�fÔ=Ô>Ü�J‰J�v—}‘}Ö%Ü˜Ô 5Ô6Ü�K‰K˜×,Ñ,Ô-Ø—o‘o tÑ+ˆGØ¤§¡×!4Ñ!4×!=Ñ!=¼e¿l¹lÈ3Ó>OÓ!PÑPˆLÜ�J‰J�v—~‘~ w°Ñ'=Ô>Ü�N‰N˜6Ÿ>™>¨6×+KÑ+KÔLÜ�J‰J�v×5Ñ5°v×7\Ñ7\Ó7^Ö_Ü˜Ô >Ô?Ü�N‰N˜6×-Ñ-¨v×/HÑ/HÖIÜ˜Ô 0Ô1Ü�K‰K˜×3Ñ3Ô4Ü�N‰N˜6×4Ñ4°d·k±k×6MÑ6MÈtÑ6SÖTÜ˜Ô EÔFØ+5Ñ(ˆM˜=Ø#Ÿ_™_°Ñ1ˆNÜ&*§h¡h¬u°]Ó/CÄeÈMÓFZÑ/ZÓ&[Ô^aØ Ñ" Aó_ñ 'Ð#ð +¬U¯Y©Y´u·|±|ÀNÓ7SÐWnÐVnÑ7nÓ-oÑoˆNÜ�J‰J�v×,Ñ,¨n×.BÑ.BÓ.D×.NÑ.NÈqÓ.Q×.[Ñ.[Ð\]Ó.^Ö_Ü˜Ô 0Ô1Ü�N‰N˜6×<Ñ<¸d×>NÑ>NÐPTÑ>TÔUÜ�N‰N˜6×7Ñ7¸¼T¿Y¹YÀs»^Ñ9KÕLÜ˜Ô 6Ô7Ø$×0Ñ0ò ^�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ò<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ô#UÑ �˜qÜ—
‘
œ7 6¨j¨\¸Ð+CÓDÀmÔTÜ—
‘
œ7 6¨j¨\Ð9KÐ+LÓMÈ}Õ]ð^ô �6Ð.Ô/Ü�N‰N˜6×3Ñ3°T·[±[×5RÑ5RÕSð 0r5   c                 ó.   — | j                   j                  S r_   ©Ú
base_modelÚembed_tokens_per_layer©rS   s    r6   Úget_per_layer_input_embeddingsz5Gemma3nPreTrainedModel.get_per_layer_input_embeddings‰  s   € Ø�‰×5Ñ5Ð5r5   c                 ó&   — || j                   _        y r_   rœ  ©rS   ri  s     r6   Úset_per_layer_input_embeddingsz5Gemma3nPreTrainedModel.set_per_layer_input_embeddingsŒ  s   € Ø16ˆ�‰Õ.r5   NÚnew_num_tokensÚpad_to_multiple_ofÚmean_resizingr]   c                 óR   •— t         ‰| �  |||¬«      }| j                  |||«       |S )N)r¤  r¥  r¦  )rM   Úresize_token_embeddingsÚ_resize_per_layer_embeddings)rS   r¤  r¥  r¦  Úinputs_embedsrT   s        €r6   r¨  z.Gemma3nPreTrainedModel.resize_token_embeddings�  s=   ø€ ô ™Ñ7Ø)Ø1Ø'ð 8ó 
ˆð
 	×)Ñ)¨.Ð:LÈmÔ\ØÐr5   c                 óœ  — | j                   | j                  j                  «       _        | j                  j                  «       j                  r| j                  «       }| j                  ||||«      }t        |d«      r|j                  }t        ||«       |j                  |j                  j                  «       | j                  |«       y y )NÚ_hf_hook)Ú
vocab_sizerk   Úget_text_configÚvocab_size_per_layer_inputrh  r   Ú_get_resized_embeddingsr?  r¬  r)   Úrequires_grad_rR   rL   r£  )rS   r¤  r¥  r¦  rž  Únew_embeddings_per_layerÚhooks          r6   r©  z3Gemma3nPreTrainedModel._resize_per_layer_embeddings�  s²   € ð DHÇ?Á?ˆ�‰×#Ñ#Ó%Ô@Ø�;‰;×&Ñ&Ó(×DÒDØ%)×%HÑ%HÓ%JÐ"Ø'+×'CÑ'CØ&¨Ð8JÈMó(Ð$ô Ð-¨zÔ:Ø-×6Ñ6�Ü"Ð#;¸TÔBØ$×3Ñ3Ð4J×4QÑ4Q×4_Ñ4_Ô`Ø×/Ñ/Ð0HÕIð Er5   )NNT)r-   r.   r/   r&   r3   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr`  r;  Ú_can_record_outputsÚinput_modalitiesr1   Úno_gradrŒ  r   r£  rd   re   rO   Ú	Embeddingr¨  r©  rg   rh   s   @r6   r�  r�  N  sþ   ø… àÓØÐØ&*Ð#Ø2Ð3ÐØ#4Ð6HÐ"IÐØÐØ€NØÐà!ÐØ"&Ðà0Ø*ñÐð 2Ðà€U‡]�]ƒ_ó%Tó ð%TòN6ò7ð
 &*Ø)-Ø"ñ	à˜d™
ðð   $™Jðð ð	ð
 
�‰õð  &*Ø)-Ø"ñ	Jà˜d™
ðJð   $™JðJð ÷	Jr5   r�  c                   óœ   ‡ — e Zd ZU dZeed<   dZdZdefˆ fd„Ze	e
dej                  dej                  dee   deez  fd	„«       «       Zˆ xZS )
ÚGemma3nAudioEncoderzx
    An audio encoder based on the [Universal Speech Model](https://huggingface.co/papers/2303.01037) architecture.
    rk   Ú	audio_melr…  c                 ó  •— t         ‰| �  |«       || _        t        |«      | _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _
        | j                  «        y c c}w r_   )rM   rN   rk   rq  Úsubsample_conv_projectionrO   Ú
ModuleListr.  Úconf_num_hidden_layersr¼  Ú	conformerÚ	post_init)rS   rk   r´   rT   s      €r6   rN   zGemma3nAudioEncoder.__init__º  se   ø€ Ü‰Ñ˜Ô ØˆŒä)LÈVÓ)TˆÔ&ÜŸ™Ü9>¸v×?\Ñ?\Ó9]Ö^°AÔ'¨Õ/Ò^ó
ˆŒð 	�‰Õùò _s   ÁA=r,   r3  r]   c                 ó>  — | j                  |«      }|j                  d   }d}t        t        | j                  j
                  «      «      D ]!  }|| j                  j
                  |   d   z  }Œ# t        j                  ||j                  ¬«      |z  }t        j                  ||j                  d   dz
  ¬«      }|j                  dkD  r>|j                  dk(  r/|j                  d«      j                  |j                  d   d«      }n`|j                  |j                  k(  rG|j                  d   dk(  r5|j                  d   dk7  r#||j                  d   k(  r|j                  d«      }t        j                  |d|«      }	| j                  D ]  }
 |
||	«      }Œ | j                  j                  dkD  r@|dd…dd| j                  j                  …f   }|	dd…dd| j                  j                  …f   }	|j!                  |	j                  d«      d«      }t#        ||	¬«      S )	a¬  Encodes a batch of MELs.

        Args:
            audio_mel: a torch.Tensor of shape [batch, num_frames, num_channels,
              mel_bins].

        Returns:
            audio_encodings: a torch.Tensor of shape
                `[batch_size, self.config.audio_soft_tokens_per_image,
                self.config.audio_config.hidden_size]`
            audio_mel_mask: a torch.BoolTensor of shape [batch, num_frames].
        r$   r   r­   )ry   rV   NrÄ   )Úlast_hidden_stater,   )rÅ  r°   r.  r/  rk   rX  r1   rƒ   rŽ   r6  rú   r…   r%  ÚgatherrÈ  Úconf_reduction_factorÚmasked_fillr+   )rS   rÃ  r,   r3  re  Út_subÚtime_stride_productÚstride_pair_idxÚindicesÚcurrent_maskÚblocks              r6   rc   zGemma3nAudioEncoder.forwardÄ  s  € ð" ×8Ñ8¸ÓCˆð  ×%Ñ% aÑ(ˆàÐÜ$¤S¨¯©×)JÑ)JÓ%KÓLò 	YˆOØ 4§;¡;×#DÑ#DÀ_Ñ#UÐVWÑ#XÑXÑð	Yô —,‘,˜u¨^×-BÑ-BÔCÐFYÑYˆÜ—+‘+˜g¨>×+?Ñ+?ÀÑ+BÀQÑ+FÔGˆð ×Ñ Ò" w§|¡|°qÒ'8Ø×'Ñ'¨Ó*×1Ñ1°.×2FÑ2FÀqÑ2IÈ2ÓN‰Gà×Ñ 7§<¡<Ò/Ø×$Ñ$ QÑ'¨1Ò,Ø—‘˜aÑ  AÒ%Ø˜Ÿ™ qÑ)Ò)ð ×'Ñ'¨Ó*ˆGä—|‘| N°A°wÓ?ˆà—^‘^ò 	CˆEÙ# O°\ÓB‰Oð	Cð �;‰;×,Ñ,¨qÒ0Ø-ªaÑ1U°D·K±K×4UÑ4UÐ1UÐ.UÑVˆOà'ªÑ+O¨d¯k©k×.OÑ.OÐ+OÐ(OÑPˆLà)×5Ñ5°l×6LÑ6LÈRÓ6PÐRUÓVˆÜ-Ø-Ø'ô
ð 	
r5   )r-   r.   r/   r0   r%   r3   Úmain_input_namer¾  rN   r    r!   r1   rf   r2   r   r   rE   r+   rc   rg   rh   s   @r6   rÂ  rÂ  °  sz   ø… ñð Óà!€OØÐðÐ1õ ð  Øð8
ØŸ™ð8
Ø7<×7GÑ7Gð8
ØSYÐZlÑSmð8
à	Ð/Ñ	/ò8
ó ó  ô8
r5   rÂ  c                   óÐ   ‡ — e Zd ZU ej                  ed<   defˆ fd„Ze	 	 	 	 ddedz  de	d   de
dz  dedz  d	ed
ef   f
d„«       Z ej                  «       edd„«       «       Zˆ xZS )r–  Úinv_freqrk   c                 ót  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        t        t        |j                  «      «      | _        i | _	        | j                  D ]Ç  }| j
                  j                  |   }|€Œ|d   | j                  |<   | j                  }| j                  |   dk7  rt        | j                  |      } || j
                  |¬«      \  }}| j                  |› d�|d¬«       | j                  |› d�|j                  «       d¬«       t        | |› d�|«       ŒÉ y )	Nr˜  rˆ  r‰  rŠ  Frr   r‹  Ú_attention_scaling)rM   rN   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrk   ÚlistÚsetr=  r˜  Úrope_parametersr—  r   r„   r×   Úsetattr)rS   rk   r@  Úrope_paramsr™  rš  Úcurr_attention_scalingrT   s          €r6   rN   zGemma3nRotaryEmbedding.__init__  s6  ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØˆŒÜ¤ F×$6Ñ$6Ó 7Ó8ˆÔØˆŒØ×*Ñ*ò 	UˆJØŸ+™+×5Ñ5°jÑAˆKØÐ"Øà)4°[Ñ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÑ)¨YÒ6Ü2°4·>±>À*Ñ3MÑN�Ù4@ÀÇÁÐYcÔ4dÑ1ˆMÐ1Ø× Ñ  J <¨yÐ!9¸=ÐUZÐ Ô[Ø× Ñ  J <Ð/AÐ!BÀM×DWÑDWÓDYÐfkÐ ÔlÜ�D˜Z˜LÐ(:Ð;Ð=SÕTñ	Ur5   NrŽ   ztorch.deviceÚseq_lenr@  r]   ztorch.Tensorc                 ó   — | j                   |   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a|  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetarx   Nro   r   r"   r®   r�   )	rß  rC  rv   rD  r1   rƒ   Úint64r�   r`   )rk   rŽ   rã  r@  ÚbaserH   Úattention_factorr×  s           r6   r—  z6Gemma3nRotaryEmbedding.compute_default_rope_parameters  s•   € ð2 ×%Ñ% jÑ1°,Ñ?ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r5   c                 óN  — t        | |› d�«      }t        | |› d�«      }|d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬	«      5  |j                  «       |j                  «       z  j                  dd
«      }	t        j                  |	|	fd¬«      }
|
j                  «       |z  }|
j                  «       |z  }d d d «       j	                  |j                  ¬«      j	                  |j                  ¬«      fS # 1 sw Y   ŒAxY w)NrŠ  rÙ  r   rV   r$   ÚmpsÚcpuF)Údevice_typeÚenabledr"   r�   r®   )rC  r`   r%  r°   r�   rŽ   r�  r•   Ústrr   r2  r1   r’   r”   r“   r‹   )rS   rá   ro  r@  r×  Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedrì  ÚfreqsÚembr”   r“   s                r6   rc   zGemma3nRotaryEmbedding.forward=  sl  € ô ˜4 J <¨yÐ!9Ó:ˆÜ# D¨Z¨LÐ8JÐ*KÓLÐà$ Tª1¨d ]Ñ3×9Ñ9Ó;×BÑBÀ<×CUÑCUÐVWÑCXÐZ\Ð^_Ó`×cÑcÐde×dlÑdlÓmÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	0Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)Ð/Ñ/ˆCØ—'‘'“)Ð/Ñ/ˆC÷		0ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	0ð 	0ús   Ã*A1FÆF$©NNNNr_   )r-   r.   r/   r1   rf   r3   r'   rN   Ústaticmethodr   rd   rî  rE   r`   r—  r¿  r   rc   rg   rh   s   @r6   r–  r–    s«   ø… Ø�l‰lÓðUÐ0õ Uð* à+/Ø+/Ø"Ø!%ñ	!*Ø! DÑ(ð!*à˜Ñ(ð!*ð �t‘ð!*ð ˜$‘Jð	!*ð
 
ˆ~˜uÐ$Ñ	%ò!*ó ð!*ðF €U‡]�]ƒ_Øò<ó ó ô<r5   r–  zBThe base Gemma 3n language model without a language modeling head.c                   óÂ  ‡ — e Zd ZU eed<   dZdefˆ fd„Ze ed¬«      e		 	 	 	 	 	 	 dde
j                  dz  de
j                  dz  d	e
j                  dz  d
e
j                  dz  dedz  de
j                  dz  dedz  dee   defd„«       «       «       Zde
j                  de
j                  fd„Z	 dde
j                  de
j                  dz  de
j                  fd„Zˆ xZS )r“  rk   )r„  c           
      óZ  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        |j                  |j                  | j                  | j                  j                  dz  ¬«      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                   ¬«      | _        t%        |«      | _        d| _        |j                  | _        |j*                  | _        t        |j,                  |j                  |j*                  z  | j                  |j*                  dz  ¬«      | _        t        j0                  | j                  |j                  |j*                  z  d¬«      | _        t        |j*                  |j                   ¬«      | _        t        j                  t        d| j                  j6                  «      D �cg c].  }t        j0                  | j                  | j                  d¬«      ‘Œ0 c}«      | _        t        j                  t        d| j                  j6                  «      D �cg c].  }t        j0                  | j                  | j                  d¬«      ‘Œ0 c}«      | _        | j=                  dt?        j@                  | j                  dz  «      d¬	«       | j=                  d
t?        jB                  t?        j@                  d«      «      d¬	«       g | _"        tG        | j                  «      D ]K  \  }}|jH                  jJ                  sŒ| jD                  jM                  dD �cg c]
  }d|› d|› �‘Œ c}«       ŒM | jO                  «        y c c}w c c}w c c}w c c}w )Nç      à?)rÊ  r¬  Frm   r$   r”  rX   rr   r•  r‡  )rÓ   rÔ   rO  rP  zlayers.z.self_attn.)(rM   rN   Úpad_token_idrÉ  r­  rÆ  rv   rk   Úembed_tokensrO   rÆ  r.  rG  r`  ÚlayersrG   r®  r\  r–  Ú
rotary_embÚgradient_checkpointingrh  r¯  rž  r~   Úper_layer_model_projectionÚper_layer_projection_normr   Úaltup_projectionsÚaltup_unembed_projectionsr„   r1   rÖ   r7  Ú"_keys_to_ignore_on_load_unexpectedÚ	enumeraterb  rI  ÚextendrÉ  )rS   rk   rÛ  r´   r|  ÚlayerÚnamerT   s          €r6   rN   zGemma3nTextModel.__init__U  só  ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒô ;Ø×Ñ˜v×1Ñ1°4×3CÑ3CÐQU×Q\ÑQ\×QhÑQhÐjmÑQmô
ˆÔô —m‘mÜINÈv×OgÑOgÓIhÖi¸IÔ$ V¨YÕ7Òió
ˆŒô # 6×#5Ñ#5¸6×;NÑ;NÔOˆŒ	Ü0°Ó8ˆŒØ&+ˆÔ#à!×-Ñ-ˆÔØ+1×+MÑ+MˆÔ(ä&DØ×-Ñ-Ø×$Ñ$ v×'IÑ'IÑIØ×ÑØ×:Ñ:¸CÑ?ô	'
ˆÔ#ô +-¯)©)Ø×ÑØ×$Ñ$ v×'IÑ'IÑIØô+
ˆÔ'ô *8¸×8ZÑ8ZÐ`f×`sÑ`sÔ)tˆÔ&ä!#§¡ÜPUÐVWÐY]×YdÑYd×YuÑYuÓPvÖwÈ1ŒR�Y‰Y�t×'Ñ'¨×)9Ñ)9ÀÖFÒwó"
ˆÔô *,¯©ÜPUÐVWÐY]×YdÑYd×YuÑYuÓPvÖwÈ1ŒR�Y‰Y�t×'Ñ'¨×)9Ñ)9ÀÖFÒwó*
ˆÔ&ð 	×ÑÐ9¼5¿<¹<È×HXÑHXÐZ^ÑH^Ó;_ÐlqÐÔrØ×ÑÐ4´e·k±kÄ%Ç,Á,ÈsÓBSÓ6TÐafÐÔgð 35ˆÔ/Ü! $§+¡+Ó.ò 	‰HˆAˆuØ�‰×1Ó1Ø×7Ñ7×>Ñ>Ø@hÖi¸�w˜q˜c ¨T¨FÒ3Òiõð	ð 	�‰Õùò[ jùò4 xùò xùò js   Â"NÈ3NÉ13N#Í0N(
F)Útie_last_hidden_statesNrÍ  Úper_layer_inputsr,  ro  rB   rª  Ú	use_cacher3  r]   c           	      ó|  — |du |duz  rt        d«      ‚|�"| j                  |«      }| j                  |«      }| j                  ||«      }|r|€t	        | j
                  ¬«      }|€V|�|j                  «       nd}	t        j                  |j                  d   |j                  ¬«      |	z   }|j                  d«      }t        |x}
t        «      s)| j
                  ||||dœ}t        di |¤Žt        di |¤Ždœ}
|}t        j                   |d	z  d
d¬«      dz  }t        j"                  d«      }|g}t%        d| j
                  j&                  «      D ]·  } | j(                  |dz
     |«      }|j+                  |j,                  |j                  ¬«      }t        j                   |d	z  d
d¬«      }t        j.                  t        j0                  ||j+                  |j                  «      «      «      }||z  |z  }|j3                  |«       Œ¹ t        j4                  |d¬«      }i }t7        | j
                  j8                  «      D ]  }| j;                  |||«      ||<   Œ t=        «       }t?        | j@                  d| j
                  jB                   «      D ]Z  \  }}|
| j
                  j8                  |      }|dd…dd…|dd…f   } |||| j
                  j8                  |      |f||||dœ|¤Ž}Œ\ t        j                   |d   d	z  d
d¬«      dz  }|d   g}t%        d| j
                  j&                  «      D ]º  } | jD                  |dz
     ||   «      }|j+                  |j,                  |j                  ¬«      }t        j                   |d	z  d
d¬«      }t        j.                  t        j0                  ||j+                  |j                  «      «      «      }||z  |z  }|j3                  |«       Œ¼ t        j4                  |«      }t        j                   |d¬«      }| jG                  |«      }tI        ||¬«      S )zµ
        per_layer_inputs (torch.Tensor, *optional*, defaults to None):
            Pre-computed per-layer embeddings. If None, they are derived from input_ids if provided.
        Nú:You must specify exactly one of input_ids or inputs_embeds©rk   r   r$   r­   )rk   rª  r,  rB   ro  )Úfull_attentionr>  r"   rV   Tr2  rø  gñhãˆµøä>rì  r�   )rU  r,  ro  rB   )rË  rB   r4   )%r  rú  Úget_per_layer_inputsÚproject_per_layer_inputsr   rk   Úget_seq_lengthr1   rƒ   r°   rŽ   r…   r�  r^  r   r   rZ   rÖ   r.  r   r   r�   r‹   rq  Úmaximumrt  ÚstackrÞ  r=  rü  r   r  rû  rG  r  r\  r   )rS   rÍ  r  r,  ro  rB   rª  r	  r3  Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚhidden_states_0Útarget_magnitudeÚepsilon_tensorÚtemp_hidden_statesr|  Ú
altup_projÚcurrent_hidden_stateÚnew_magnituderC   rT  r@  rU  Údecoder_layerÚcausal_maskrn  Úaltup_unemb_projs                               r6   rc   zGemma3nTextModel.forwardŽ  sL  € ð$ ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMØ#×8Ñ8¸ÓCÐà×8Ñ8¸ÐHXÓYÐá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-ÄÔFð Ÿ+™+Ø!.Ø"0Ø#2Ø ,ñˆKô #5Ñ"C°{Ñ"CÜ%FÑ%UÈÑ%Uñ#Ðð (ˆô !Ÿ:™: o°qÑ&8¸bÈ$ÔOÐSVÑVÐÜŸ™ dÓ+ˆà-Ð.ÐÜ�q˜$Ÿ+™+×6Ñ6Ó7ò 	<ˆAà6˜×/Ñ/°°A±Ñ6°ÓGˆJØ#-§=¡=°×7LÑ7LÐUe×UlÑUl =Ó#mÐ Ü!ŸJ™JÐ';¸QÑ'>ÀBÐPTÔUˆMÜ!ŸJ™J¤u§}¡}°]ÀN×DUÑDUÐVf×VmÑVmÓDnÓ'oÓpˆMØ#7Ð:JÑ#JÈ]Ñ#ZÐ Ø×%Ñ%Ð&:Õ;ð	<ô Ÿ™Ð$6¸AÔ>ˆØ ÐÜ˜dŸk™k×5Ñ5Ó6ò 	gˆJØ.2¯o©o¸mÈ\Ð[eÓ.fÐ 
Ò+ð	gô $›:Ðä )¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÓ Wò 	ÑˆAˆ}Ø-¨d¯k©k×.EÑ.EÀaÑ.HÑIˆKØ.ªq²!°Qº¨zÑ:ˆOá)ØØ# D§K¡K×$;Ñ$;¸AÑ$>Ñ?Øð	ð "2Ø*Ø)Ø /ñ	ð ñ	‰Mð		ô  !Ÿ:™: m°AÑ&6¸!Ñ&;ÀÈTÔRÐVYÑYÐØ+¨AÑ.Ð/ÐÜ�q˜$Ÿ+™+×6Ñ6Ó7ò 	<ˆAà-R¨T×-KÑ-KÈAÐPQÉEÑ-RÐS`ÐabÑScÓ-dÐØ#3×#6Ñ#6¸_×=RÑ=RÐ[k×[rÑ[rÐ#6Ó#sÐ Ü!ŸJ™JÐ';¸QÑ'>ÀBÐPTÔUˆMÜ!ŸJ™J¤u§}¡}°]ÀN×DUÑDUÐVf×VmÑVmÓDnÓ'oÓpˆMØ#7Ð:JÑ#JÈ]Ñ#ZÐ Ø×%Ñ%Ð&:Õ;ð	<ô Ÿ™Ð$6Ó7ˆÜŸ
™
 =°aÔ8ˆØŸ	™	 -Ó0ˆä&Ø+Ø+ô
ð 	
r5   c                 óœ   —  | j                  |«      j                  g |j                  ¢| j                  j                  ‘| j
                  ‘­Ž S r_   )rž  r¢   r°   rk   rG  rh  )rS   rÍ  s     r6   r  z%Gemma3nTextModel.get_per_layer_inputs   sP   € Ø=ˆt×*Ñ*¨9Ó5×=Ñ=ð 
Ø�_‰_ð
à�K‰K×)Ñ)ð
ð ×,Ñ,ò
ð 	
r5   c                 ó  — | j                  |«      }|| j                  j                  |j                  |j                  ¬«      z  } |j
                  g |j                  d d ¢| j                  j                  ‘| j                  ‘­Ž }| j                  |«      }|€|S |j                  |j                  k7  r |dd | j                  j                  …d d …f   }||z   | j                  j                  |j                  |j                  ¬«      z  S )Nrì  rV   .)rþ  r”  r�   r‹   rŽ   r¢   r°   rk   rG  rh  rÿ  r•  )rS   rª  r  rl  s       r6   r  z)Gemma3nTextModel.project_per_layer_inputs  s.  € ð
 .2×-LÑ-LÈ]Ó-[ÐØ × ?Ñ ?× BÑ BØ×%Ñ%Ð.B×.IÑ.Ið !Có !
ñ 	
Ðð  <Ð3×;Ñ;ð  
Ø× Ñ   "Ð%ð 
à�K‰K×)Ñ)ð 
ð ×,Ñ,ò 
Ðð
  $×=Ñ=Ð>RÓSÐàÐ#Ø'Ð'à×%Ñ%Ð)9×)?Ñ)?Ò?à/°Ð5T°t·{±{×7TÑ7TÐ5TÒVWÐ0WÑXÐà$Ð'7Ñ7¸4×;UÑ;U×;XÑ;XØ×%Ñ%Ð.B×.IÑ.Ið <Yó <
ñ 
ð 	
r5   )NNNNNNNr_   )r-   r.   r/   r'   r3   r¾  rN   r    r!   r   r1   r  rf   r   r=   re   r   r   r   rc   r  r  rg   rh   s   @r6   r“  r“  P  sV  ø… àÓØ Ðð7Ð0õ 7ðr  Ù¨EÔ2Øð .2Ø04Ø.2Ø04Ø(,Ø26Ø!%ñm
à×#Ñ# dÑ*ðm
ð  Ÿ,™,¨Ñ-ðm
ð Ÿ™ tÑ+ð	m
ð
 ×&Ñ&¨Ñ-ðm
ð  ™ðm
ð ×(Ñ(¨4Ñ/ðm
ð ˜$‘;ðm
ð Ð+Ñ,ðm
ð 
!òm
ó ó 3ó  ðm
ð^
¨e×.>Ñ.>ð 
À5Ç<Á<ó 
ð 15ñ
à—|‘|ð
ð  Ÿ,™,¨Ñ-ð
ð 
�‰÷	
r5   r“  z?The base Gemma 3n language model with a language modeling head.c                   óT  ‡ — e Zd ZU ddiZddiZddgdgfiZeed<   defˆ fd„Ze	e
	 	 	 	 	 	 	 	 dd
ej                  d	z  dej                  d	z  dej                  d	z  ded	z  dej                  d	z  dej                  d	z  ded	z  deej                  z  dee   defd„«       «       Zˆ xZS )ÚGemma3nForCausalLMúlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrC   rA   rk   c                 óè   •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  |j                  d¬«      | _        | j                  «        y rÝ  )
rM   rN   r“  r‚  r­  rO   r~   rv   r%  rÉ  r—  s     €r6   rN   zGemma3nForCausalLM.__init__*  sU   ø€ Ü‰Ñ˜Ô Ü% fÓ-ˆŒ
Ø ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Õr5   NrÍ  r,  ro  rB   rª  Úlabelsr	  Úlogits_to_keepr3  r]   c	           
      ó  —  | j                   d||||||dœ|	¤Ž}
|
j                  }t        |t        «      rt	        | d«      n|}| j                  |dd…|dd…f   «      }| j                  j                  �G|| j                  j                  z  }t        j                  |«      }|| j                  j                  z  }d}|� | j                  ||| j                  fi |	¤Ž}t        |||
j                  |
j                  |
j                  ¬«      S )a„  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, Gemma3nForCausalLM

        >>> model = Gemma3nForCausalLM.from_pretrained("google/gemma-2-9b")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-9b")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```)rÍ  r,  ro  rB   rª  r	  N)r@   rA   rB   rC   rD   r4   )r‚  rË  r�  rd   Úslicer%  rk   Úfinal_logit_softcappingr1   r  Úloss_functionr­  r   rB   rC   rD   )rS   rÍ  r,  ro  rB   rª  r(  r	  r)  r3  ÚoutputsrC   Úslice_indicesrA   r@   s                  r6   rc   zGemma3nForCausalLM.forward3  s  € ð@ ,6¨4¯:©:ð ,
ØØ)Ø%Ø+Ø'Øñ,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆØ�;‰;×.Ñ.Ð:Ø˜dŸk™k×AÑAÑAˆFÜ—Z‘Z Ó'ˆFØ˜dŸk™k×AÑAÑAˆFàˆØÐØ%�4×%Ñ% f¨f°d·o±oÑPÈÑPˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r5   )NNNNNNNr   )r-   r.   r/   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr'   r3   rN   r   r   r1   r  rf   r   r=   re   rd   r   r   r   rc   rg   rh   s   @r6   r#  r#  #  s%  ø… à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÓðÐ0õ ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ;
à×#Ñ# dÑ*ð;
ð Ÿ™ tÑ+ð;
ð ×&Ñ&¨Ñ-ð	;
ð
  ™ð;
ð ×(Ñ(¨4Ñ/ð;
ð × Ñ  4Ñ'ð;
ð ˜$‘;ð;
ð ˜eŸl™lÑ*ð;
ð Ð+Ñ,ð;
ð 
 ò;
ó ó ô;
r5   r#  c                   ó”   ‡ — e Zd ZdZdeez  defˆ fd„Z	 	 d
dej                  dz  dej                  dz  dej                  fd	„Zˆ xZS )ÚGemma3nMultimodalEmbedderzQEmbeds token ids or soft tokens for multimodal content into language model space.Úmultimodal_configÚtext_configc                 ór  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _        t        j                  | j                  | j                  «      | _        t        | j                  | j
                  ¬«      | _        t        | j                  | j
                  ¬«      | _        t        j                  | j                  | j                  d¬«      | _        t        | j                  | j
                  d¬«      | _        y )Nr¬  Frm   )rI   rJ   )rM   rN   rv   Úmultimodal_hidden_sizer®  rI   Úvocab_offsetr­  Útext_hidden_sizerO   rÀ  Ú	embeddingrG   Úhard_embedding_normÚsoft_embedding_normr~   Úembedding_projectionÚembedding_post_projection_norm)rS   r5  r6  rT   s      €r6   rN   z"Gemma3nMultimodalEmbedder.__init__v  sé   ø€ ô
 	‰ÑÔà&7×&CÑ&CˆÔ#Ø$×1Ñ1ˆŒØ-×:Ñ:ˆÔØ+×6Ñ6ˆŒØ +× 7Ñ 7ˆÔäŸ™ d§o¡o°t×7RÑ7RÓSˆŒÜ#1°$×2MÑ2MÐSW×S[ÑS[Ô#\ˆÔ Ü#1°$×2MÑ2MÐSW×S[ÑS[Ô#\ˆÔ Ü$&§I¡I¨d×.IÑ.IÈ4×K`ÑK`ÐglÔ$mˆÔ!Ü.<¸T×=RÑ=RÐX\×X`ÑX`ÐmrÔ.sˆÕ+r5   NrÍ  rª  r]   c                 óô   — |du |duz  rt        d«      ‚|�| j                  |«      }n/| j                  || j                  z
  «      }| j	                  |«      }| j                  |«      }| j                  |«      S )aå  Embeds token ids or soft tokens for multimodal content into language model space.

        Args:
            input_ids: A torch.LongTensor containing the token ids to embed. Values should be in the range
                `[vocab_offset, vocab_offset + vocab_size)`.
            inputs_embeds: A torch.Tensor containing the soft tokens to embed.

        Returns:
            A torch.Tensor of embeddings with  shape `[batch_size, seq_len, self.config.text_config.hidden_size]`.
        Nr  )r  r=  r;  r9  r<  r>  r?  )rS   rÍ  rª  Úemb_normÚhard_embÚemb_norm_projs         r6   rc   z!Gemma3nMultimodalEmbedder.forward‰  s‚   € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ$Ø×/Ñ/°Ó>‰Hà—~‘~ i°$×2CÑ2CÑ&CÓDˆHØ×/Ñ/°Ó9ˆHà×1Ñ1°(Ó;ˆØ×2Ñ2°=ÓAÐAr5   r]  )r-   r.   r/   r0   r%   r(   r'   rN   r1   r  rf   rc   rg   rh   s   @r6   r4  r4  s  sk   ø„ Ù[ðtà-Ð0CÑCðtð 'õtð* .2Ø-1ñBà×#Ñ# dÑ*ðBð —|‘| dÑ*ðBð 
�‰÷	Br5   r4  z�
    The base Gemma 3n model comprising a vision backbone, an audio backbone, and a language model without a
    language modeling head.
    c                   óÜ  ‡ — e Zd ZdZdefˆ fd„Ze ed¬«      dej                  de
e   deez  fd	„«       «       Z	 	 	 	 ddej                  d
z  dej                  d
z  dej                  d
z  dej                  d
z  fd„Ze	 	 	 	 	 	 	 	 	 	 	 ddej                  d
z  dej                  d
z  dej                  d
z  dej"                  d
z  dej"                  d
z  dej                  d
z  ded
z  dej                  d
z  dej                  d
z  dej                  d
z  ded
z  de
e   defd„«       Zd„ Zd„ Ze ed¬«      dej"                  dej"                  de
e   deez  fd„«       «       Zˆ xZS ) ÚGemma3nModelFrk   c                 ó$  •— t         ‰| �  |«       t        j                  |j                  ¬«      | _        |j                  j                  | _        t        j                  |j                  ¬«      }|| _        |j                  j                  | _	        t        j                  |j                  «      | _        t        |j                  |j                  «      | _        t        |j                  |j                  «      | _        | j                  «        y )Nr  )rM   rN   r#   Úfrom_configÚvision_configÚvision_towerr6  r­  Úlanguage_modelr¯  Úaudio_configÚaudio_towerr4  Úembed_visionÚembed_audiorÉ  )rS   rk   rJ  rT   s      €r6   rN   zGemma3nModel.__init__¯  sÇ   ø€ Ü‰Ñ˜Ô Ü%×1Ñ1¸×9MÑ9MÔNˆÔØ ×,Ñ,×7Ñ7ˆŒä"×.Ñ.°f×6HÑ6HÔIˆØ,ˆÔØ*0×*<Ñ*<×*WÑ*WˆÔ'Ü$×0Ñ0°×1DÑ1DÓEˆÔÜ5°f×6JÑ6JÈF×L^ÑL^Ó_ˆÔÜ4°V×5HÑ5HÈ&×J\ÑJ\Ó]ˆÔØ�‰Õr5   zOProjects the last hidden state from the vision model into language model space.r7   Úpixel_valuesr3  r]   c                 óŠ  —  | j                   d	|dddœ|¤Ž}|j                  }|j                  |j                  d   | j                  j
                  j                  | j                  j                  «      j                  ddd«      }|| j                  j
                  j                  dz  z  }| j                  |¬«      |_
        |S )
NFT)rO  Ú
do_poolingÚreturn_dictr   r"   r$   rø  ©rª  r4   )rI  rË  r¢   r°   rk   rH  rv   Úvision_soft_tokens_per_imager²   rM  Úpooler_output)rS   rO  r3  Úvision_outputsrË  s        r6   Úget_image_featureszGemma3nModel.get_image_features¼  sÃ   € ð +˜×*Ñ*Ðs¸ÐQVÐdhÑsÐlrÑsˆØ*×<Ñ<Ðð .×5Ñ5Ø×#Ñ# AÑ&Ø�K‰K×%Ñ%×1Ñ1Ø�K‰K×4Ñ4ó
÷ ‰'�!�Q˜Ó
ð	 	ð 	˜TŸ[™[×6Ñ6×BÑBÀCÑGÑGÐØ'+×'8Ñ'8ÐGXÐ'8Ó'YˆÔ$àÐr5   NrÍ  rª  Úimage_featuresÚaudio_featuresc           	      óP  — |€×| | j                  «       t        j                  | j                  j                  t        j
                  |j                  ¬«      «      k(  }|j                  d«      }| | j                  «       t        j                  | j                  j                  t        j
                  |j                  ¬«      «      k(  j                  d«      }n2|| j                  j                  k(  }|| j                  j                  k(  }|j                  «       }|j                  d«      j                  |j                  «      }|�Pt        ||j                  d   z  |j                  «       k(  d|› d|j                  d   |j                  d   z  › �«       |j                  «       }|j                  d«      j                  |j                  «      }|�Pt        ||j                  d   z  |j                  «       k(  d|› d|j                  d   |j                  d   z  › �«       ||fS )zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        rì  rV   z6Image features and image tokens do not match, tokens: z, features: r   r$   z6Audio features and audio tokens do not match, tokens: )Úget_input_embeddingsr1   rÖ   rk   Úimage_token_idÚlongrŽ   ÚallÚaudio_token_idr4  r…   r�   r   r°   Únumel)	rS   rÍ  rª  rX  rY  Úspecial_image_maskÚspecial_audio_maskÚn_image_tokensÚn_audio_tokenss	            r6   Úget_placeholder_maskz!Gemma3nModel.get_placeholder_maskÒ  sL  € ð ÐØ!.Ð2M°$×2KÑ2KÓ2MÜ—‘˜TŸ[™[×7Ñ7¼u¿z¹zÐR_×RfÑRfÔgó3ñ "Ðð "4×!7Ñ!7¸Ó!;ÐàØ.�4×,Ñ,Ó.Ü—L‘L §¡×!;Ñ!;Ä5Ç:Á:ÐVc×VjÑVjÔkóñ÷ ‰c�"‹gñ ð "+¨d¯k©k×.HÑ.HÑ!HÐØ!*¨d¯k©k×.HÑ.HÑ!HÐà+×/Ñ/Ó1ˆØ/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐØÐ%Ü"Ø ×!4Ñ!4°RÑ!8Ñ8¸N×<PÑ<PÓ<RÑRØHÈÐHXÐXdÐes×eyÑeyÐz{Ñe|ð  @N÷  @Tñ  @Tð  UVñ  @Wñ  fWð  eXð  Yôð
 ,×/Ñ/Ó1ˆØ/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐØÐ%Ü"Ø ×!4Ñ!4°RÑ!8Ñ8¸N×<PÑ<PÓ<RÑRØHÈÐHXÐXdÐes×eyÑeyÐz{Ñe|ð  @N÷  @Tñ  @Tð  UVñ  @Wñ  fWð  eXð  Yôð
 "Ð#5Ð5Ð5r5   Úinput_featuresr,  Úinput_features_maskro  rB   Útoken_type_idsr(  r	  Ú	lm_kwargsc                 ó4	  — |du |	duz  rt        d«      ‚|��_ | j                  «       |«      }	t        j                  |dk\  || j                  k  «      }t        j
                  ||t        j                  |«      «      }| j                  j                  |«      }t        j                  || j                  j                  k\  || j                  j                  k  «      }| j                  j                  | j                  j                  z   dz
  }t        j
                  |||«      j                  |	j                  «      }| j                  |¬«      }|j                  |	j                  |	j                  «      }|j!                  d«      }t        j
                  |||	«      }	|| j                  j                  k\  }| j                  j                  | j                  j                  z   dz
  }t        j
                  |||«      j                  |	j                  «      }| j                  |¬«      }|j                  |	j                  |	j                  «      }|j!                  d«      }t        j
                  |||	«      }	nd}|�l| j#                  |d¬«      j$                  }|j                  |	j                  |	j                  «      }| j'                  ||	|¬	«      \  }}|	j)                  ||«      }	|��K|��H| j+                  || d¬«      }|j$                  }|j,                  }t        j.                  | j                  dz
  ggt        j0                  |j                  ¬
«      }| j                  |¬«      } t        j
                  |j!                  d«      | |«      }|j2                  \  }!}"}#| j4                  j6                  |"z
  }$| j9                  |!|$|#«      }%t        j:                  ||%fd¬«      }|j                  |	j                  |	j                  «      }| j'                  ||	|¬«      \  }}&|	j)                  |&|«      }	 | j                  dd|||||	|ddœ|¤Ž}'t=        |'j>                  |r|'j@                  nd|'jB                  |'jD                  |�nd|�¬«      S d¬«      S )a}  
        input_features_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Attention mask for `input_features` where non-zero values mark valid audio frames.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Gemma3nForConditionalGeneration

        >>> model = Gemma3nForConditionalGeneration.from_pretrained("google/gemma3n2-3b-mix-224")
        >>> processor = AutoProcessor.from_pretrained("google/gemma3n2-3b-mix-224")

        >>> prompt = "Where is the cat standing?"
        >>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, text=prompt,  return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(**inputs,)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Where is the cat standing?\nsnow"
        ```
        Nr  r   r$   )rÍ  rV   T)rR  )rª  rX  rì  r�   )rª  rY  )rÍ  r  r,  ro  rB   rª  r	  rR  )rË  rB   rC   rD   r;   r<   r4   )#r  r[  r1   r  r¯  r  Ú
zeros_likerJ  r  rM  r9  rN  r­  r�   rŽ   r‹   r…   rW  rU  re  Úmasked_scatterÚget_audio_featuresr,   rÖ   r]  r°   rk   Úaudio_soft_tokens_per_imager%  r’   r:   rË  rB   rC   rD   )(rS   rÍ  rO  rf  r,  rg  ro  rB   rh  rª  r(  r	  ri  Úper_layer_inputs_maskÚper_layer_inputs_tokensr  Úvision_maskÚdummy_vision_token_idÚvision_input_idsÚvision_embedsÚexpanded_vision_maskÚ
audio_maskÚdummy_audio_token_idÚaudio_input_idsÚaudio_embedsÚexpanded_audio_maskrX  ra  r´   Úaudio_outputsrY  Úaudio_padding_toksÚaudio_padding_embsÚaudio_batch_sizeÚaudio_seq_lenÚaudio_embed_dimÚextra_padding_tokensÚextra_padding_featuresrb  r.  s(                                           r6   rc   zGemma3nModel.forwardþ  s–  € ð` ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÑ Ø7˜D×5Ñ5Ó7¸	ÓBˆMô %*×$5Ñ$5°iÀ1±nÀiÐRV×RqÑRqÑFqÓ$rÐ!Ü&+§k¡kÐ2GÈÔTY×TdÑTdÐenÓToÓ&pÐ#Ø#×2Ñ2×GÑGÐH_Ó`Ðô  ×+Ñ+Ø˜T×.Ñ.×;Ñ;Ñ;¸YÈ×IYÑIY×IfÑIfÑ=fóˆKð %)×$5Ñ$5×$BÑ$BÀT×EVÑEV×EaÑEaÑ$aÐdeÑ$eÐ!Ü$Ÿ{™{¨;¸	ÐCXÓY×\Ñ\Ð]j×]qÑ]qÓrÐØ ×-Ñ-Ð8HÐ-ÓIˆMØ)×,Ñ,¨]×-AÑ-AÀ=×CVÑCVÓWˆMØ#.×#8Ñ#8¸Ó#<Ð Ü!ŸK™KÐ(<¸mÈ]Ó[ˆMð # d×&6Ñ&6×&CÑ&CÑCˆJØ#'×#3Ñ#3×#@Ñ#@À4×CSÑCS×C^ÑC^Ñ#^ÐabÑ#bÐ Ü#Ÿk™k¨*°iÐAUÓV×YÑYÐZg×ZnÑZnÓoˆOØ×+Ñ+°oÐ+ÓFˆLØ'Ÿ?™?¨=×+?Ñ+?À×ATÑATÓUˆLØ",×"6Ñ"6°rÓ":ÐÜ!ŸK™KÐ(;¸\È=ÓY‰Mà#Ðð Ð#Ø!×4Ñ4°\ÈtÐ4ÓT×bÑbˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNØ$(×$=Ñ$=Ø¨À~ð %>ó %Ñ!Ð ð *×8Ñ8Ð9KÈ^Ó\ˆMð Ñ%Ð*=Ñ*IØ ×3Ñ3°NÐEXÐDXÐfjÐ3ÓkˆMØ*×8Ñ8ˆNØ&×5Ñ5ˆJô "'§¡°·±À!Ñ0CÐ/DÐ.EÌUÏZÉZÐ`n×`uÑ`uÔ!vÐØ!%×!1Ñ!1Ð<NÐ!1Ó!OÐÜ"Ÿ[™[¨×)=Ñ)=¸bÓ)AÐCUÐWeÓfˆNà?M×?SÑ?SÑ<Ð˜m¨_Ø#'§;¡;×#JÑ#JÈ]Ñ#ZÐ Ø%7×%>Ñ%>Ð?OÐQeÐgvÓ%wÐ"ä"ŸY™Y¨Ð8NÐ'OÐUVÔWˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNØ$(×$=Ñ$=Ø¨À~ð %>ó %Ñ!ˆAÐ!ð *×8Ñ8Ð9KÈ^Ó\ˆMà%�$×%Ñ%ð 

ØØ-Ø)Ø%Ø+Ø'ØØñ

ð ñ

ˆô *Ø%×7Ñ7Ù7@˜G×3Ò3ÀdØ!×/Ñ/Ø×)Ñ)Ø2>Ð2J¡ÐPTØ2@Ð2L ô
ð 	
ð SWô
ð 	
r5   c                 ó.   — | j                   j                  S r_   ©rJ  rž  rŸ  s    r6   r   z+Gemma3nModel.get_per_layer_input_embeddings‡  s   € Ø×"Ñ"×9Ñ9Ð9r5   c                 ó&   — || j                   _        y r_   r„  r¢  s     r6   r£  z+Gemma3nModel.set_per_layer_input_embeddingsŠ  s   € Ø5:ˆ×ÑÕ2r5   zPProjects the last hidden state from the audio encoder into language model space.c                 óx   —  | j                   ||fddi|¤Ž}| j                  |j                  ¬«      }||_        |S )a0  
        input_features (`torch.FloatTensor]` of shape `(num_images, seq_length, num_features)`):
            The tensors corresponding to the input audio.
        input_features_mask (`torch.FloatTensor]` of shape `(num_images, seq_length)`):
            The attention mask for the input audio.
        rR  TrS  )rL  rN  rË  rU  )rS   rf  rg  r3  r{  ry  s         r6   rm  zGemma3nModel.get_audio_features�  sV   € ð 9I¸×8HÑ8HØÐ/ñ9
Ø=Að9
ØEKñ9
ˆð ×'Ñ'°m×6UÑ6UÐ'ÓVˆØ&2ˆÔ#àÐr5   rô  )NNNNNNNNNNN)r-   r.   r/   Úaccepts_loss_kwargsr&   rN   r   r   r1   r=   r   r   rE   r   rW  r  re  rf   r   re   r:   rc   r   r£  r+   rm  rg   rh   s   @r6   rE  rE  ¥  si  ø„ ð  Ðð˜}õ ð ÙÐ!rÔsðà×'Ñ'ðð Ð+Ñ,ðð 
Ð+Ñ	+ò	ó tó ðð, .2Ø26Ø37Ø37ñ*6à×#Ñ# dÑ*ð*6ð ×(Ñ(¨4Ñ/ð*6ð ×)Ñ)¨DÑ0ð	*6ð
 ×)Ñ)¨DÑ0ó*6ðX ð .2Ø15Ø37Ø.2Ø37Ø04Ø(,Ø26Ø26Ø*.Ø!%ñF
à×#Ñ# dÑ*ðF
ð ×'Ñ'¨$Ñ.ðF
ð ×)Ñ)¨DÑ0ð	F
ð
 Ÿ™ tÑ+ðF
ð #Ÿ\™\¨DÑ0ðF
ð ×&Ñ&¨Ñ-ðF
ð  ™ðF
ð ×(Ñ(¨4Ñ/ðF
ð ×(Ñ(¨4Ñ/ðF
ð × Ñ  4Ñ'ðF
ð ˜$‘;ðF
ð Ð.Ñ/ðF
ð 
$òF
ó ðF
òP:ò;ð ÙÐ!sÔtðàŸ™ðð #Ÿ\™\ðð Ð+Ñ,ð	ð
 
Ð/Ñ	/òó uó ôr5   rE  z†
    The base Gemma 3n model comprising a vision backbone, an audio backbone, a language model, and a language modeling
    head.
    c                   ó  ‡ — e Zd ZddiZdZdefˆ fd„Zedej                  de
e   fd„«       Zee	 	 	 	 	 	 	 	 	 	 	 	 dd
ej                  d	z  dej                  d	z  dej                  d	z  dej                  d	z  dej                  d	z  dej                  d	z  ded	z  dej                  d	z  dej                  d	z  dej                  d	z  ded	z  deej                  z  de
e   defd„«       «       Z	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„	Zd„ Zd„ Zˆ xZS )ÚGemma3nForConditionalGenerationr$  z(model.language_model.embed_tokens.weightFrk   c                 óî   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  j                  |j                  j                  d¬«      | _	        | j                  «        y rÝ  )rM   rN   rE  r‚  rO   r~   r6  rv   r­  r%  rÉ  r—  s     €r6   rN   z(Gemma3nForConditionalGeneration.__init__®  sS   ø€ Ü‰Ñ˜Ô Ü! &Ó)ˆŒ
Ü—y‘y ×!3Ñ!3×!?Ñ!?À×ASÑAS×A^ÑA^ÐejÔkˆŒØ�‰Õr5   rO  r3  c                 ó<   —  | j                   j                  |fi |¤ŽS r_   )r‚  rW  )rS   rO  r3  s      r6   rW  z2Gemma3nForConditionalGeneration.get_image_features´  s   € à,ˆt�z‰z×,Ñ,¨\ÑD¸VÑDÐDr5   NrÍ  rf  r,  rg  ro  rB   rh  rª  r(  r	  r)  ri  r]   c                 óT  —  | j                   d|||||||||	|
|ddœ|¤Ž}|j                  }t        |t        «      rt	        | d«      n|}| j                  |dd…|dd…f   «      }| j                  j                  «       j                  x}�||z  }t        j                  |«      }||z  }d}|
�7 | j                  ||
| j                  j                  «       j                  fi |¤Ž}t        |||j                  |j                  |j                   |j"                  |j$                  ¬«      S )aŒ  
        input_features_mask (torch.Tensor, *optional*, defaults to None):
            The attention mask for the input audio.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels in
            `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

        >>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma-3-4b-it")
        >>> processor = AutoProcessor.from_pretrained("google/gemma-3-4b-it")

        >>> messages = [
        ...     {
        ...         "role": "system",
        ...         "content": [
        ...             {"type": "text", "text": "You are a helpful assistant."}
        ...         ]
        ...     },
        ...     {
        ...         "role": "user", "content": [
        ...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
        ...             {"type": "text", "text": "Where is the cat standing?"},
        ...         ]
        ...     },
        ... ]

        >>> inputs = processor.apply_chat_template(
        ...     messages,
        ...     tokenizer=True,
        ...     return_dict=True,
        ...     return_tensors="pt",
        ...     add_generation_prompt=True
        ... )
        >>> # Generate
        >>> generate_ids = model.generate(**inputs)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
        ```
        T)rÍ  rO  rf  r,  rg  ro  rB   rh  rª  r(  r	  rR  N)r@   rA   rB   rC   rD   r;   r<   r4   )r‚  rË  r�  rd   r+  r%  rk   r®  r,  r1   r  r-  r­  r?   rB   rC   rD   r;   r<   )rS   rÍ  rO  rf  r,  rg  ro  rB   rh  rª  r(  r	  r)  ri  r.  rC   r/  rA   r,  r@   s                       r6   rc   z'Gemma3nForConditionalGeneration.forward¸  sG  € ðD �$—*‘*ð 
ØØ%Ø)Ø)Ø 3Ø%Ø+Ø)Ø'ØØØñ
ð ñ
ˆð   ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆØ'+§{¡{×'BÑ'BÓ'D×'\Ñ'\Ð\Ð#ÐiØÐ5Ñ5ˆFÜ—Z‘Z Ó'ˆFØÐ5Ñ5ˆFàˆØÐØ%�4×%Ñ% f¨f°d·k±k×6QÑ6QÓ6S×6^Ñ6^ÑlÐbkÑlˆDä,ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;Ø '× ;Ñ ;ô
ð 	
r5   c                 ó`   •— t        ‰| �  |f|||||
||	|dœ|¤Ž}|s|
s||d<   ||d<   ||d<   |S )N)rB   rª  r,  ro  r	  r)  rh  Úis_first_iterationrO  rf  rg  )rM   Úprepare_inputs_for_generation)rS   rÍ  rB   rª  ro  rO  rf  r,  rg  rh  r	  r)  r(  rŽ  r3  Úmodel_inputsrT   s                   €r6   r�  z=Gemma3nForConditionalGeneration.prepare_inputs_for_generation!	  sk   ø€ ô$ ‘wÑ<Øð
à+Ø'Ø)Ø%ØØ)Ø)Ø1ñ
ð ñ
ˆñ  ¡YØ+7ˆL˜Ñ(Ø-;ˆLÐ)Ñ*Ø2EˆLÐ.Ñ/àÐr5   c                 ó6   — | j                   j                  «       S r_   )r‚  r   rŸ  s    r6   r   z>Gemma3nForConditionalGeneration.get_per_layer_input_embeddingsJ	  s   € Ø�z‰z×8Ñ8Ó:Ð:r5   c                 ó:   — | j                   j                  |«       y r_   )r‚  r£  r¢  s     r6   r£  z>Gemma3nForConditionalGeneration.set_per_layer_input_embeddingsM	  s   € Ø�
‰
×1Ñ1°%Õ8r5   )NNNNNNNNNNNr   )NNNNNNNNTNNF)r-   r.   r/   r0  r‡  r&   rN   r   r1   r=   r   r   rW  r   r  rf   r   re   rd   r?   rc   r�  r   r£  rg   rh   s   @r6   r‰  r‰  ¤  sÞ  ø„ ð +Ð,VÐWÐØÐð˜}õ ð ðE¨u×/@Ñ/@ð EÈFÐSeÑLfò Eó ðEð Øð .2Ø15Ø37Ø.2Ø37Ø04Ø(,Ø26Ø26Ø*.Ø!%Ø-.ñe
à×#Ñ# dÑ*ðe
ð ×'Ñ'¨$Ñ.ðe
ð ×)Ñ)¨DÑ0ð	e
ð
 Ÿ™ tÑ+ðe
ð #Ÿ\™\¨DÑ0ðe
ð ×&Ñ&¨Ñ-ðe
ð  ™ðe
ð ×(Ñ(¨4Ñ/ðe
ð ×(Ñ(¨4Ñ/ðe
ð × Ñ  4Ñ'ðe
ð ˜$‘;ðe
ð ˜eŸl™lÑ*ðe
ð Ð.Ñ/ðe
ð 
'òe
ó ó ðe
ðT ØØØØØØ ØØØØØ õ'òR;ö9r5   r‰  )rÂ  r#  r‰  rE  r�  r“  )rÄ   NN)r$   )fr€   Úcollectionsr   Úcollections.abcr   r   Údataclassesr   Útypingr   r1   Útorch.nnrO   Útorch.nn.functionalr    rj  Ú r	   rŽ  ré  r
   Úcache_utilsr   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   r   Úutils.genericr   r    Úutils.output_capturingr!   Úautor#   Úconfiguration_gemma3nr%   r&   r'   r(   Úaccelerate.hooksr)   r+   r:   r?   ÚModulerG   rj   rÂ   r*  rL  rq  r�  rž  rª  r¼  rÀ  rÆ  rÐ  rÚ  rû  r"  rf   rd   r(  r`   rE   r6  r9  r;  r`  r�  rÂ  r–  r“  r#  r4  rE  r‰  Ú__all__r4   r5   r6   ú<module>rª     s  ðó* Ý  ß .Ý !Ý ã Ý ß Ð å &Ý !ß .Ý )ß RÝ 9ß kÑ kß Kß FÝ &÷÷ ÷ HÝ 5Ý ß lÓ lñ ÔÝ3ð Ø
ô3Ð%?ó 3ó ó ð3ñ ðôð
 ô9Ð!8ó 9ó óð9ñ( ðôð
 ô9 Kó 9ó óð9ô:4�R—Y‘Yô 4ô0g)¨B¯I©Iô g)ôTa˜BŸI™Iô aôHj, b§i¡iô j,ôZB7 §	¡	ô B7ôJF¨"¯)©)ô FôRO R§Y¡Yô Oô8D r§y¡yô Dô2( r§y¡yô (ôV §¡ô ô6S R§\¡\ô Sô;˜RŸY™Yô ;ô$#5�R—Y‘Yô #5ôL`'�r—y‘yô `'òF(ð	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð$ Ø Ø ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �S‰[ð%ð �T‰\ð%ð �T‰\ð%ð ˆ5�<‰<˜Ÿ™Ð%Ñ&ó%ñD.˜EŸL™Lð .¨u¯|©|ð .À%Ç,Á,ð .Ð_bó .ô,l)˜2Ÿ9™9ô l)ô^I%Ð8ô I%ðX ô^J˜_ó ^Jó ð^JôBN
Ð0ô N
ôbL<˜RŸY™Yô L<ñ^ ÐaÔbôO
Ð-ó O
ó cðO
ñd Ð^Ô_ôL
Ð/°ó L
ó `ðL
ô^/B §	¡	ô /Bñd ðôôvÐ)ó vóðvñr ðôôd9Ð&<¸oó d9óðd9òN�r5   