
    ^j"                    f
   d dl Z d dlmZ d dlmZ d dlmZ d dlmZ d dl	m
Z
 d dlZd dlmZ d dlmZ d	d
lmZ d	dlmZ d	dlmZmZ d	dlmZ d	dlmZ d	dlmZ d	dlmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& d	dl'm(Z( d	dl)m*Z* d	dl+m,Z,m-Z- d	dl.m/Z/m0Z0 d	dl1m2Z2m3Z3 d	dl4m5Z5 d	dl6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z< d	dl=m>Z>m?Z? d	dl@mAZAmBZB ddlCmDZD ddlEmFZFmGZGmHZHmIZI  e;       rd dlJmKZK  e9d       e G d! d"e,                    ZL e9d#       e G d$ d%e7                    ZMe G d& d'e,             ZNe9e G d( d)e-                    ZO G d* d+ej                        ZQ G d, d-ej                        ZR G d. d/ej                        ZS G d0 d1ej                        ZT G d2 d3ej                        ZU G d4 d5ej                        ZV G d6 d7ej                        ZW G d8 d9ej                        ZY G d: d;ej                        ZZ G d< d=ej                        Z[ G d> d?ej                        Z\ G d@ dAej                        Z] G dB dCej                        Z^ G dD dEej                        Z_dF Z`ddGej                  dHej                  dIej                  dJebfdKZcdLej                  dMebdNej                  fdOZd	 	 	 ddPej                  dQej                  dRej                  dSej                  dTej                  dz  dUeeebz  dVeedz  dWeedz  dNefej                  ej                  f   fdXZg	 ddGej                  dHej                  dIej                  dYej                  dJebdNej                  fdZZh G d[ d\ej                        Zi G d] d^e*      Zj G d_ d`ej                        Zk G da dbej                        Zl G dc ddej                        Zm G de dfej                        Zne G dg dhej                               Zo G di djej                        Zp G dk dle*      Zq G dm dnej                        Zse9 G do dpe3             Zt e9dq        G dr dset             Zu e9dt        G du dvete             Zvdwefebebf   dNefdxZw G dy dzet      Zx G d{ d|et      Zy G d} d~ej                        Zzdedej                  dTej                  dz  dedz  dYej                  dz  dej                  dNe{fdZ|dej                  dej                  dNej                  fdZ~ e9d        G d det             Z e9d        G d dete             Zg dZy)    N)UserDict)Callable)	dataclass)cached_property)Optional)nn)
functional   )initialization)ACT2FN)CacheDynamicCache)PreTrainedConfig)GenerationMixin)use_experts_implementation)_preprocess_mask_argumentsblockwise_overlaycreate_bidirectional_maskcreate_causal_maskcreate_masks_for_generate!create_sliding_window_causal_maskmaybe_pad_block_sequence_idssliding_window_overlay)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputWithPastBaseModelOutputWithPooling)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)ModelOutputTransformersKwargsauto_docstringcan_return_tupleis_accelerate_availabletorch_compilable_check)maybe_autocastmerge_with_config_defaults)OutputRecordercapture_outputs   )	AutoModel   )Gemma4AudioConfigGemma4ConfigGemma4TextConfigGemma4VisionConfig)add_hook_to_modulezK
    Base class for Gemma4 outputs, with hidden states and attentions.
    custom_introc                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
eeej                  ej                  f   f   dz  ed<   y)Gemma4ModelOutputWithPasta  
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    audio_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
    shared_kv_states (`dict`, *optional*):
        Dictionary mapping layer type strings to tuples of (key_states, value_states) tensors.
        Used to pass shared KV states between layers during KV sharing.
    Nimage_hidden_statesaudio_hidden_statesshared_kv_states)__name__
__module____qualname____doc__r9   torchFloatTensor__annotations__r:   r;   dictstrtupleTensor     u/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/gemma4/modeling_gemma4.pyr8   r8   H   sa    " 59**T1848**T18LPd3ellELL&@ AABTIPrH   r8   zR
    Base class for Gemma4 causal language model (or autoregressive) outputs.
    c                   r   e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   dZej                  dz  ed<   dZej                  dz  ed	<   dZeeeej&                  ej&                  f   f   dz  ed
<   y)Gemma4CausalLMOutputWithPasta  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.text_config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
    audio_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        audio_hidden_states of the model produced by the audio encoder and after projecting the last hidden state.
    shared_kv_states (`dict`, *optional*):
        Dictionary mapping layer type strings to tuples of (key_states, value_states) tensors.
        Used to pass shared KV states between layers during KV sharing.
    Nlosslogitspast_key_valueshidden_states
attentionsr9   r:   r;   )r<   r=   r>   r?   rL   r@   rA   rB   rM   rN   r   rO   rE   rP   r9   r:   r;   rC   rD   rF   rG   rH   rI   rK   rK   g   s    * &*D%

d
")'+FE$+$(OUT\(59M5**+d2926Je''(4/648**T1848**T18LPd3ellELL&@ AABTIPrH   rK   c                   b    e Zd ZU dZdZeeeej                  ej                  f   f   dz  e
d<   y)Gemma4TextModelOutputWithPasta9  
    BaseModelOutputWithPast extended with shared_kv_states for KV sharing.

    Args:
        shared_kv_states (`dict`, *optional*):
            Dictionary mapping layer type strings to tuples of (key_states, value_states) tensors.
            Used to pass shared KV states between layers during KV sharing.
    Nr;   )r<   r=   r>   r?   r;   rC   rD   rE   r@   rF   rB   rG   rH   rI   rR   rR      s7     MQd3ellELL&@ AABTIPrH   rR   c                   :    e Zd ZU dZdZej                  dz  ed<   y)Gemma4AudioModelOutputz
    attention_mask (`torch.BoolTensor`, *optional*):
        A torch.BoolTensor of shape `(batch_size, num_frames)`. True for valid positions, False for padding.
    Nattention_mask)r<   r=   r>   r?   rU   r@   
BoolTensorrB   rG   rH   rI   rT   rT      s    
 /3NE$$t+2rH   rT   c                   n     e Zd Zdeez  dededdf fdZdej                  dej                  fdZ	 xZ
S )	Gemma4ClippableLinearconfigin_featuresout_featuresreturnNc                    t         |           |j                  | _        t        j                  ||d      | _        | j                  r| j                  dt        j                  t        d                    | j                  dt        j                  t        d                   | j                  dt        j                  t        d                    | j                  dt        j                  t        d                   y y )NFbias	input_mininf	input_max
output_min
output_max)
super__init__use_clipped_linearsr   Linearlinearregister_bufferr@   tensorfloat)selfrY   rZ   r[   	__class__s       rI   rf   zGemma4ClippableLinear.__init__   s     	#)#=#= ii\F##  ellE%L=.IJ  ell5<.HI  u||U5\M/JK  u||E%L/IJ	 $rH   rO   c                    | j                   r+t        j                  || j                  | j                        }| j                  |      }| j                   r+t        j                  || j                  | j                        }|S N)rg   r@   clampr`   rb   ri   rc   rd   )rm   rO   s     rI   forwardzGemma4ClippableLinear.forward   s\    ##!KKt~~t~~VMM2##!KKtXMrH   )r<   r=   r>   r3   r0   intrf   r@   rF   rr   __classcell__rn   s   @rI   rX   rX      sT    K"%66K K 	K
 
K 	U\\ 	ell 	rH   rX   c                        e Zd Zd	dededef fdZdej                  fdZ	dej                  dej                  fdZ
 xZS )
Gemma4RMSNormdimeps
with_scalec                     t         |           || _        || _        | j                  r0t	        j
                  t        j                  |      d      | _        y y )NT)requires_grad)	re   rf   ry   rz   r   	Parameterr@   onesweight)rm   rx   ry   rz   rn   s       rI   rf   zGemma4RMSNorm.__init__   sB    $??,,uzz#dKDK rH   rO   c                     |j                  d      j                  dd      | j                  z   }|t        j                   |d      z  S )Nr-   T)keepdim      )powmeanry   r@   )rm   rO   mean_squareds      rI   _normzGemma4RMSNorm._norm   sA    $((+00T0BTXXMuyyt<<<rH   r\   c                     | j                  |j                               }| j                  r|| j                  j                         z  }|j	                  |      S rp   )r   rl   rz   r   type_as)rm   rO   normed_outputs      rI   rr   zGemma4RMSNorm.forward   sH    

=#6#6#89??)DKK,=,=,??M$$]33rH   )gư>T)r<   r=   r>   rs   rl   boolrf   r@   rF   r   rr   rt   ru   s   @rI   rw   rw      sL    LC Le L L=5<< =
4U\\ 4ell 4rH   rw   c                        e Zd ZU dZej
                  ed<   def fdZ ej                         dej
                  dej
                  fd       Z
 xZS ) Gemma4AudioRelPositionalEncodingzSinusoidal relative positional encoding for the audio encoder.

    Produces position embeddings of shape [1, context_size // 2 + 1, hidden_size] with
    concatenated [sin..., cos...] layout matching the original Gemma4 convention.
    inv_timescalesrY   c                    t         |           |j                  | _        |j                  |j                  z   dz
  |j
                  z   | _        d}d}| j                  dz  }t        j                  ||z        t        |dz
  d      z  }|t        j                  t        j                  |      | z        z  }| j                  d|j                  d      j                  d      d       y )	Nr/         ?     @r-   r   r   F
persistent)re   rf   hidden_sizeattention_chunk_sizeattention_context_leftattention_context_rightcontext_sizemathlogmaxr@   exparangerj   	unsqueeze)rm   rY   min_timescalemax_timescalenum_timescaleslog_timescale_incrementr   rn   s          rI   rf   z)Gemma4AudioRelPositionalEncoding.__init__   s    !--''&*G*GG!KfNlNll 	 ))Q."&((==+H"ICP^abPbdeLf"f&5<<3OSjRj3j)kk-~/G/G/J/T/TUV/WdijrH   rO   r\   c                 t   t        j                  | j                  dz  dd|j                        }|d   }|| j                  j                  |j                        z  }t        j                  t        j                  |      t        j                  |      gd      }|j                  |j                        S )Nr-   r   device.Nrx   dtype)
r@   r   r   r   r   tocatsincosr   )rm   rO   position_idsscaled_time	pos_embeds        rI   rr   z(Gemma4AudioRelPositionalEncoding.forward   s    ||D$5$5$:B=K_K_`#I."T%8%8%;%;=CWCW%;%XXIIuyy5uyy7MNTVW	||-"5"5|66rH   )r<   r=   r>   r?   r@   rF   rB   r0   rf   no_gradrr   rt   ru   s   @rI   r   r      sU     LL k0 k U]]_7U\\ 7ell 7 7rH   r   c                   P    e Zd ZdZdedef fdZdej                  dej                  fdZ	dej                  dej                  fdZ
d	ej                  dej                  fd
Z	 ddej                  dej                  dej                  dz  deej                  df   fdZ xZS )Gemma4AudioAttentionz3Chunked local attention with relative position biasrY   	layer_idxc                     t         |           || _        || _        |j                  | _        |j                  |j                  z  | _        |j                  | _	        | j                  dz  t        j                  d      z  | _        t        j                  dt        j                  z         t        j                  d      z  | _        |j                  | _        |j"                  dz
  | _        |j&                  | _        | j                   | j$                  z   | j(                  z   | _        t-        ||j                  | j                  | j                  z        | _        t-        ||j                  | j                  | j                  z        | _        t-        ||j                  | j                  | j                  z        | _        t-        ||j                  |j                        | _        t7        j8                  |j                  | j                  | j                  z  d      | _        t7        j<                  t?        j@                  | j                              | _!        | jE                  dt?        jF                  | j
                        d       y )Nr   r-   r/   Fr^   softcapr   )$re   rf   rY   r   attention_logit_capattention_logits_soft_capr   num_attention_headshead_dim	num_headsr   r   q_scaleek_scaler   
chunk_sizer   max_past_horizonr   max_future_horizonr   rX   q_projk_projv_projpostr   rh   relative_k_projr}   r@   zerosper_dim_scalerj   rk   rm   rY   r   rn   s      rI   rf   zGemma4AudioAttention.__init__   s   ")/)C)C&**f.H.HH33t+txx{:xxDFF
+dhhqk9 55 & = = A"("@"@ OOd.C.CCdF]F]]+FF4F4FY]YfYfHfg+FF4F4FY]YfYfHfg+FF4F4FY]YfYfHfg)&&2D2DfFXFXY	!yy););T^^dmm=[bgh\\%++dmm*DEYT5S5S(TafgrH   rO   r\   c           	         |j                   \  }}}}|| j                  z   dz
  | j                  z  }|| j                  z  |z
  }t        j                  |ddddd|f      }|j	                  ||| j                  ||      j                         S )zSplits a `(batch_size, seq_len, num_heads, head_dim)` tensor into non-overlapping blocks of `chunk_size` along the sequence dim.r/   r   )shaper   Fpadreshape
contiguous)rm   rO   
batch_sizeseq_lenr   r   
num_blocksr   s           rI   _convert_to_blockz&Gemma4AudioAttention._convert_to_block  s    3@3F3F0
GY/!3G
4??*W4maAq!S-AB$$ZT__iYabmmoorH   c           
      @   |j                   \  }}}}t        j                  |dddd| j                  | j                  | j
                  z   dz
  f      }|j                  d| j                  | j
                        }t        j                  |dd      }|j                         S )z`Extracts overlapping context windows of `context_size` for every block, strided by `chunk_size`.r   r/   r   r-   )r   r   r   r   r   r   unfoldr   r@   movedimr   )rm   rO   r   r   r   r   s         rI   _extract_block_contextz+Gemma4AudioAttention._extract_block_context  s    3@3F3F0
GYAq!Q(=(=t?V?VY]YhYh?hkl?lm
 &,,Q0A0A4??SmR;''))rH   xc                     |j                   \  }}}}}| j                  }t        j                  |d|dz   |z
  f      }|j	                  |||||dz   z        }|dd||z  f   }|j	                  |||||      S )zjRelative position shift for blocked attention. See appendix B of https://huggingface.co/papers/1901.02860.r   r/   .N)r   r   r   r   view)rm   r   r   r   r   
block_sizeposition_lengthr   s           rI   
_rel_shiftzGemma4AudioAttention._rel_shift(  s    IJF
Iz:((EE!a)O;<=FF:y*jLSTDT6UVc.Z,.../vvj)Z\RRrH   Nposition_embeddingsrU   c                    |j                   \  }}}||| j                  | j                  f}| j                  |      j	                         j                  |      }| j                  |      j	                         j                  |      }	| j                  |      j	                         j                  |      }
|| j                  z  t        j                  | j                        z  }|	| j                  z  }	| j                  |      }| j                  |	      }	| j                  |
      }
|j                   d   }| j                  |      }|j                  d| j                  | j                        }|j!                  |j"                        }|j%                  ddddd      }||	j%                  ddddd      z  }|j'                  || j                  d| j                        }||j%                  ddd      z  }|j'                  || j                  || j(                  d      }| j+                  |      }||z   }|| j,                  z  }t/        j0                  |      }|| j,                  z  }|4|j3                  |j5                         | j6                  j8                        }t        j:                  |dt.        j<                        j!                  |
j"                        }||
j%                  ddddd      z  }|j%                  ddddd      j'                  ||| j(                  z  d      }|d d d |f   j?                         }| jA                  |j!                  |j"                              }||fS )	Nr/   r   r   r   r
   r-      rx   r   )!r   r   r   r   rl   r   r   r   r   r   softplusr   r   r   r   r   r   r   permuter   r   r   r   r@   tanhmasked_filllogical_notrY   attention_invalid_logits_valuesoftmaxfloat32r   r   )rm   rO   r   rU   r   
seq_length_hidden_shapequery_states
key_statesvalue_statesr   relative_key_statesqueries	matrix_acqueries_flat	matrix_bdattn_weightsattn_outputs                      rI   rr   zGemma4AudioAttention.forward1  s    %2$7$7!
J"JN{{=1779>>|L[[/557<<\J
{{=1779>>|L#dll2QZZ@R@R5SS$,,.
--l;00<
22<@!''*
"223FG166r4>>4==Y144<;M;M4N&&q!Q15j00Aq!Q??	z4>>2t}}U #6#>#>q!Q#GG	%%j$..*doo_ab	OOI.	 9,#dll2zz,/#dll2%'33**,dkk.X.XL yy2U]]KNN|OaOab"\%9%9!Q1a%HH!))!Q1a8@@ZZ^ZiZiMikmn!![j[.1<<>ii}/B/B CDL((rH   rp   )r<   r=   r>   r?   r0   rs   rf   r@   rF   r   r   r   rV   rE   rr   rt   ru   s   @rI   r   r      s    =h0 hS h4pu|| p p*ELL *U\\ *SELL SU\\ S 37	1)||1) #\\1) ((4/	1)
 
u||T!	"1)rH   r   c                   ^     e Zd Z fdZddej
                  dej
                  dz  fdZ xZS )'Gemma4AudioSubSampleConvProjectionLayerc                     t         |           t        j                  ||dddd      | _        t        j
                  ||dd      | _        t        j                         | _        y )N)r
   r
   )r-   r-   r/   F)in_channelsout_channelskernel_sizestridepaddingr_   T)ry   elementwise_affiner_   )	re   rf   r   Conv2dconv	LayerNormnormReLUact)rm   r   r   norm_epsrn   s       rI   rf   z0Gemma4AudioSubSampleConvProjectionLayer.__init__f  sW    II#%
	 LL8PT[`a	779rH   NrO   maskc           
         |,|j                  |j                        }||d d d d d d f   z  }| j                  |j                  | j                  j                  j                              }| j                  | j                  |j                  dddd            j                  dddd      j                               }||d d d d df   }||fS )Nr   r   r-   r
   r/   )	r   r   r  r   r   r  r  r   r   )rm   rO   r  s      rI   rr   z/Gemma4AudioSubSampleConvProjectionLayer.forwards  s    77-"6"677D)DD!T1A,BBM		-"2"24993C3C3I3I"JK=+@+@Aq!+L!M!U!UVWYZ\]_`!a!l!l!no3Q3<Dd""rH   rp   )r<   r=   r>   rf   r@   rF   rr   rt   ru   s   @rI   r   r   e  s(    #U\\ #9L #rH   r   c            	            e Zd Zdef fdZ	 ddej                  dej                  dz  deej                  ej                  f   fdZ xZ	S )	"Gemma4AudioSubSampleConvProjectionrY   c                 v   t         |           t        d|j                  d   |j                        | _        t        |j                  d   |j                  d   |j                        | _        |j                  d   dz  |j                  d   z  }t        j                  ||j                  d      | _
        y )Nr/   r   )r   r   r  r   Fr^   )re   rf   r   subsampling_conv_channelsrms_norm_epslayer0layer1r   rh   r   input_proj_linear)rm   rY   proj_input_dimrn   s      rI   rf   z+Gemma4AudioSubSampleConvProjection.__init__  s    =99!<((

 >88;99!<((

 !::1=BfFfFfghFii!#>6;M;MTY!ZrH   Ninput_featuresinput_features_maskr\   c                 &   |j                  d      }| j                  ||      \  }}| j                  ||      \  }}|j                  \  }}}}|j	                  dddd      j                         j                  ||d      }| j                  |      |fS )Nr/   r   r-   r
   r   )r   r  r  r   r   r   r   r  )rm   r  r  rO   r  r   r   r   s           rI   rr   z*Gemma4AudioSubSampleConvProjection.forward  s    
 '003"kk-9LMt"kk->t$1$7$7!
Aw%--aAq9DDFNNz[bdfg%%m4d::rH   rp   )
r<   r=   r>   r0   rf   r@   rF   rE   rr   rt   ru   s   @rI   r
  r
    sW    [0 [$ 48;; #\\D0; 
u||U\\)	*	;rH   r
  c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )Gemma4AudioFeedForwardrY   c                    t         |           || _        t        ||j                  |j                  dz        | _        t        ||j                  dz  |j                        | _        t        |j                        | _        t        |j                        | _	        t        |j                     | _        |j                  | _        |j                  | _        y )Nr   )re   rf   rY   rX   r   ffw_layer_1ffw_layer_2rw   pre_layer_normpost_layer_normr   
hidden_actact_fngradient_clippingresidual_weightpost_layer_scalerm   rY   rn   s     rI   rf   zGemma4AudioFeedForward.__init__  s    09K9KVM_M_bcMcd09K9Ka9OQWQcQcd+F,>,>?,V-?-?@V../!'!9!9 & 6 6rH   rO   r\   c                    t        | j                  t        j                  |j                        j
                        }|}t        j                  || |      }| j                  |      }| j                  |      }| j                  |      }| j                  |      }t        j                  || |      }| j                  |      }|| j                  z  }||z  }|S rp   )minr  r@   finfor   r   rq   r  r  r  r  r  r   )rm   rO   r  residuals       rI   rr   zGemma4AudioFeedForward.forward  s     6 6MDWDW8X8\8\] M4E3EGXY++M:((7M2((7M4E3EGXY,,];...!rH   	r<   r=   r>   r0   rf   r@   rF   rr   rt   ru   s   @rI   r  r    s+    70 7U\\ ell rH   r  c                   `     e Zd Zed        Zdej                  dej                  f fdZ xZS )Gemma4AudioCausalConv1dc                 p    | j                   d   dz
  | j                  d   z  dz   }|| j                  d   z
  S )Nr   r/   )r   dilationr   )rm   effective_kernel_sizes     rI   left_padz Gemma4AudioCausalConv1d.left_pad  s>    !%!1!1!!4q!8DMM!<L Lq P$t{{1~55rH   r   r\   c                 z    t         j                  j                  || j                  df      }t        |   |      S Nr   )r   r	   r   r,  re   rr   )rm   r   rn   s     rI   rr   zGemma4AudioCausalConv1d.forward  s3     MMa$--!34wq!!rH   )	r<   r=   r>   r   r,  r@   rF   rr   rt   ru   s   @rI   r(  r(    s;     6 6"<<" 
	" "rH   r(  c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )Gemma4AudioLightConv1drY   c                 6   t         |           || _        t        ||j                  |j                  dz        | _        t        ||j                  |j                        | _        t        |j                  |j                  |j                  |j                  d      | _	        t        |j                  |j                  d      | _        t        |j                  |j                  d      | _        t        |j                     | _        |j"                  | _        y )Nr-   F)r   r   r   groupsr_   Try   rz   )re   rf   rY   rX   r   linear_start
linear_endr(  conv_kernel_sizedepthwise_conv1drw   r  r  	conv_normr   r  r  r  r!  s     rI   rf   zGemma4AudioLightConv1d.__init__  s    1&&:L:LfN`N`cdNde/8J8JFL^L^_ 7**++//%%!
 ,F,>,>FDWDWdhi&v'9'9v?R?R_cdV../!'!9!9rH   rO   r\   c                    |}| j                  |      }| j                  |      }t        j                  j	                  |d      }| j                  |j                  dd            j                  dd      }t        | j                  t        j                  |j                        j                        }t        j                  || |      }| j                  |      }| j                  |      }| j!                  |      }||z  }|S )Nr   r   r/   r-   )r  r4  r   r	   glur7  	transposer#  r  r@   r$  r   r   rq   r8  r  r5  )rm   rO   r%  r  s       rI   rr   zGemma4AudioLightConv1d.forward  s     ++M:))-8))-R)@--m.E.Ea.KLVVWXZ[\   6 6MDWDW8X8\8\]M4E3EGXY}5M26!rH   r&  ru   s   @rI   r0  r0    s+    :0 :(U\\ ell rH   r0  c            
            e Zd Zdedef fdZdej                  dej                  dz  dej                  de	e
   d	ej                  f
d
Z xZS )Gemma4AudioLayerrY   r   c                 p   t         |           || _        t        |      | _        t        |      | _        t        ||      | _        t        |      | _	        t        |j                        | _        t        |j                        | _        t        |j                        | _        |j                  | _        y rp   )re   rf   rY   r  feed_forward1feed_forward2r   	self_attnr0  lconv1drw   r   norm_pre_attnnorm_post_attnnorm_outr  r   s      rI   rf   zGemma4AudioLayer.__init__  s    3F;3F;-fi@-f5*6+=+=>+F,>,>?%f&8&89!'!9!9rH   rO   rU   Nr   kwargsr\   c                 @   t        | j                  t        j                  | j                  j
                  j                        j                        }| j                  |      }|}t        j                  || |      }| j	                  |      }| j                  |||      \  }}t        j                  || |      }| j                  |      }||z  }| j                  |      }| j                  |      }t        j                  || |      }| j                  |      }|S )N)rO   r   rU   )r#  r  r@   r$  rC  r   r   r   r?  rq   rA  rD  rB  r@  rE  )rm   rO   rU   r   rF  r  r%  r   s           rI   rr   zGemma4AudioLayer.forward  s      6 6DDVDVD]D]DcDc8d8h8hi**=9 M4E3EGXY**=9>>' 3) * 
q M4E3EGXY++M:!]3**=9M4E3EGXYm4rH   )r<   r=   r>   r0   rs   rf   r@   rF   rV   r"   r$   rr   rt   ru   s   @rI   r=  r=    si    :0 :S : ||  ((4/  #\\	 
 +,  
 rH   r=  c                        e Zd Zdef fdZdej                  dej                  dej                  fdZdej                  dej                  dej                  dej                  fdZ xZ	S )	Gemma4VisionPatchEmbedderrY   c                    t         |           || _        |j                  | _        |j                  | _        |j
                  | _        t        j                  d| j                  dz  z  | j                  d      | _        t        j                  t        j                  d| j
                  | j                              | _        y )Nr
   r-   Fr^   )re   rf   rY   r   
patch_sizeposition_embedding_sizer   rh   
input_projr}   r@   r~   position_embedding_tabler!  s     rI   rf   z"Gemma4VisionPatchEmbedder.__init__D  s    !-- ++'-'E'E$))A(:$:D<L<LSXY(*UZZ4C_C_aeaqaq5r(s%rH   pixel_position_idspadding_positionsr\   c                    |j                  d      }t        j                  |d   | j                  d         }t        j                  |d   | j                  d         }||z   }t	        j
                  |j                  d      d|      }|S )ak  Compute 2-D patch position embeddings via embedding lookup.

        ``pixel_position_ids`` has shape ``(batch, num_patches, 2)`` where the
        last dimension holds (x, y) indices into ``position_embedding_table``
        (shape ``(2, position_embedding_size, hidden_size)``).  The result is the
        sum of the x- and y-embeddings for each patch.
        r   r#  .r   .r/   r/   r           )rq   r   	embeddingrN  r@   wherer   )rm   rO  rP  clamped_positionsx_emby_embr   s          rI   _position_embeddingsz.Gemma4VisionPatchEmbedder._position_embeddingsN  s     /444; -f5t7T7TUV7WX-f5t7T7TUV7WX#em#kk*;*E*Eb*I3Pcd""rH   pixel_valuesc                     d|dz
  z  }| j                   j                  j                  x}j                  r|j	                  |      }| j                  |      }| j                  ||      }||z   S )Nr-         ?)rM  r   r   is_floating_pointr   r[  )rm   r\  rO  rP  target_dtyperO   r   s          rI   rr   z!Gemma4VisionPatchEmbedder.forwardd  sn     L3./ OO22888LKK'??<8L5"778JL]^222rH   )
r<   r=   r>   r3   rf   r@   rF   r[  rr   rt   ru   s   @rI   rI  rI  C  su    t1 t#u|| #X]XdXd #iniuiu #,	3!LL	3>Cll	3_d_k_k	3		3rH   rI  c                   .    e Zd ZdZdef fdZdej                  dej                  dede	ej                  ej                  f   fdZ
	 ddej                  dej                  d
ej                  ded	z  de	ej                  ej                  f   f
dZ xZS )Gemma4VisionPoolera[  Spatial pooling and ``sqrt(hidden_size)`` scaling for vision encodings.

    The scaling expands the activation magnitude, which can exceed the float16 range, so it is
    computed in float32 and the pooled features are returned in float32. The caller
    (``Gemma4VisionModel.forward``) standardizes them and casts back to the working dtype.
    rY   c                 l    t         |           |j                  | _        | j                  dz  | _        y )Nr^  )re   rf   r   root_hidden_sizer!  s     rI   rf   zGemma4VisionPooler.__init__x  s/    !-- $ 0 0# 5rH   rO   rO  lengthr\   c                    |j                   d   }t        ||z  dz        }|dz  }||z  |k7  r%t        d|j                    d| d|d|d| d	      |j                  d
      }|d   j	                  dd      d
   dz   }t        j                  ||d      }	|	d   ||z  |	d   z  z   }	t        j                  |	j                         |      j                         |z  }
|
j                  dd      |j                         z  }t        j                  |
d
k(  j                  d            }|j                  |j                        |fS )z
        2D spatial pooling according to patch positions.
        Pools the input tokens by averaging patches within a `k^2` grid, where `k` is determined by the ratio between
        input and output lengths
        r/   r^  r-   zCannot pool z to z: k=z^2 times length=z	 must be .r   rR  rS  r   Trx   r   floor)rounding_moderT  r   )r   rs   
ValueErrorrq   r   r@   divr   one_hotlongrl   r;  r   allr   r   )rm   rO   rO  re  input_seq_lenk	k_squaredrX  max_xkernel_idxsweightsoutputr  s                rI   _avg_pool_by_positionsz)Gemma4VisionPooler._avg_pool_by_positions}  sh    &++A.&(S01qD	v.}2234xu!EVviW`an`oopq  /444;!&)--"d-CAFJii 11GL!&)UaZ;v;N,NN))K,,.7==?)K""1a(=+>+>+@@  'Q,!3!3!3!:;yy,,-t33rH   NrP  output_lengthc                 8   ||j                   d   kD  rt        d| d|j                   d    d      |j                  |j                  d      d      }|j                   d   |k7  r| j	                  |||      \  }}|j                         | j                  z  }||fS )Nr/   z*Cannot output more soft tokens (requested z) than there are patches (z9). Change the value of `num_soft_tokens` when processing.r   rU  )r   rk  r   r   rw  rl   rd  )rm   rO   rO  rP  rx  s        rI   rr   zGemma4VisionPooler.forward  s     =..q11<]O L"((+,,eg 
 &112C2M2Mb2QSVWq!]2/3/J/J1=0,M, &++-0E0EE///rH   rp   )r<   r=   r>   r?   r3   rf   r@   rF   rs   rE   rw  rr   rt   ru   s   @rI   rb  rb  p  s    61 6
4"\\4?D||4UX4	u||U\\)	*4@ %)0||0 "LL0 !<<	0
 Tz0 
u||U\\)	*0rH   rb  c                   *     e Zd Zdef fdZd Z xZS )Gemma4VisionMLPrY   c                    t         |           || _        |j                  | _        |j                  | _        t        || j                  | j                        | _        t        || j                  | j                        | _        t        || j                  | j                        | _        t        |j                     | _        y rp   )re   rf   rY   r   intermediate_sizerX   	gate_projup_proj	down_projr   hidden_activationr  r!  s     rI   rf   zGemma4VisionMLP.__init__  s    !--!'!9!9.vt7G7GI_I_`,VT5E5EtG]G]^.vt7M7MtO_O_`V556rH   c                     | j                  | j                  | j                  |            | j                  |      z        }|S rp   r  r  r~  r  rm   r   r  s      rI   rr   zGemma4VisionMLP.forward  6    NN4;;t~~a/@#ADLLQRO#ST	rH   )r<   r=   r>   r3   rf   rr   rt   ru   s   @rI   r{  r{    s    71 7rH   r{  c                        e Zd ZU ej                  ed<   ddef fdZe	 	 	 ddedz  dej                  dz  de
dz  dedef   fd	       Z ej                         ed
               Z xZS )Gemma4VisionRotaryEmbeddinginv_freqNrY   c                    t         |           |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |      \  }| _
        | j                  d|d       | j                  d|j                         d       y )N	rope_typedefaultr  Fr   original_inv_freq)re   rf   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenrY   rope_parametersr  compute_default_rope_parametersr   attention_scalingrj   clone)rm   rY   r   rope_init_fnr  rn   s        rI   rf   z$Gemma4VisionRotaryEmbedding.__init__  s    "("@"@$*$B$B!44[A!%!E!E>>Y&.t~~>L+7V+L($(ZeD0(..2BuUrH   r   r   r\   torch.Tensorc                 $   | j                   d   }t        | dd      xs | j                  | j                  z  }|dz  }d}d|t	        j
                  d|dt        j                        j                  |t        j                        |z  z  z  }||fS )	a  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        
rope_thetar   Nr-   r   r   r   r   r   	r  getattrr   r   r@   r   int64r   rl   )rY   r   r   baserx   spatial_dimattention_factorr  s           rI   r  z;Gemma4VisionRotaryEmbedding.compute_default_rope_parameters  s    & %%l3fj$/c63E3EIcIc3c QhQQekkBEEV[`[f[fEgjuuw
 )))rH   c                 .   | j                   d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}g g }}t        d      D ]  }|d d d d |f   }|d d d d d f   j                         }	t        |d      5  |j                         |	j                         z  j                  dd      }
t        j                  |
|
fd	      }|j                         | j                  z  }|j!                         | j                  z  }d d d        |j#                         |j#                          t        j                  |d	      j	                  |j$                  
      }t        j                  |d	      j	                  |j$                  
      }||fS # 1 sw Y   xY w)Nr   r   r/   mpscpur-   Fdevice_typeenabledr   r   )r  rl   expandr   r   r   
isinstancetyperD   ranger)   r;  r@   r   r   r  r   appendr   )rm   r   r   inv_freq_expandedr  all_cosall_sinidim_position_idsdim_position_ids_expandedfreqsembr   r   s                 rI   rr   z#Gemma4VisionRotaryEmbedding.forward  s    !MM$4-8>>@GGHZHZ[\H]_acdehhijiqiqr'1!((--'E!((--[`J`ahhmmfk rq 
	 A+Aq!G4(8D!(D(J(J(L%KG 9*0025N5T5T5VVaabcefgiiB7ggi$"8"88ggi$"8"88	9
 NN3NN3
	  iiR(++!''+:iiR(++!''+:Cx9 9s   4BHH	rp   NNN)r<   r=   r>   r@   rF   rB   r3   rf   staticmethodr   rs   rE   rl   r  r   r   rr   rt   ru   s   @rI   r  r    s    llV1 V  ,0&*" *"T) *t# * t * 
~u$	%	 *  *D U]]_  rH   r  c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..Nr   r-   r   )r   r@   r   )r   x1x2s      rI   rotate_halfr    sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''rH   r   r   r   unsqueeze_dimc                 n    |j                  |      }|j                  |      }| |z  t        |       |z  z   S )a\  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        x (`torch.Tensor`): The tensor to embed.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )r   r  r   r   r   r  s       rI   apply_rotary_pos_embr    s8    " --
&C
--
&CGA,--rH   rO   n_repr\   c                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r/   N)r   r  r   )rO   r  batchnum_key_value_headsslenr   s         rI   	repeat_kvr  /  so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TTrH   modulequerykeyvaluerU   dropoutscalingr   c                 |   || j                   dz  }t        || j                        }	t        || j                        }
t        j                  ||	j                  dd            |z  }|||z  }t        j                  |      }||z  }|||z   }t        j                  j                  |dt        j                        j                  |j                        }t        j                  j                  ||| j                        }t        j                  ||
      }|j                  dd      j                         }||fS )Nr   r-   r
   r   r   )ptrainingr/   )r   r  num_key_value_groupsr@   matmulr;  r   r   r	   r   r   r   r   r  r  r   )r  r  r  r  rU   r  r  r   rF  r   r   r   r   s                rI   eager_attention_forwardr  ;  s    //4'3 ; ;<JUF$?$?@L<<z';';Aq'ABWLL#g-zz,/#g-!#n4 ==((2U]](SVVW\WbWbcL==((6??([L,,|\:K''1-88:K$$rH   r   c           	         |j                   d   }| j                   d   }d|d|z  z  z  }|dk  rt        d| d| d| d      |g|z  }t        j                  | |d      }	t        j                  ||d      }
t        j                  ||d      }t	        |      D cg c]  }t        |	|   |
|   ||   |	       }}t        j                  |d      S c c}w )
ak  Applies multidimensional RoPE to inputs.

    Args:
        x (`torch.Tensor`): The tensor to embed.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            If position_ids.ndim + 2 == x.ndim, then this function passes through to `apply_rotary_pos_emb()`.
            Otherwise, position_ids is used to split the inputs, x, into multiple pieces, where each piece is fed to
            `apply_rotary_pos_emb()`, and then concatenated back together.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.

    Returns:
      Tensor of shape [B, L, N, H] with RoPE applied.
    r   r-   r   zEInvalid configuration: num_rotated_channels_per_dim must be > 0, got z (num_input_channels=z, ndim=)r   r  )r   rk  r@   splitr  r  r   )r   r   r   r   r  ndimnum_input_channelsnum_rotated_channels_per_dimsplit_sizesx_parts	cos_parts	sin_partsrq  y_partss                 rI   apply_multidimensional_roper  ]  s   8 b!D#$(:q4x(H#I #q(,--BCUBV WF!
 	
 0047Kkk![b1GC"5IC"5I t  	aj!!'		
G  99W"%%s   Cc                       e Zd ZdZdedef fdZ	 	 	 ddej                  dej                  dej                  dz  d	ej                  dz  d
e
e   deej                  ej                  dz  eej                     dz  f   fdZ xZS )Gemma4VisionAttention=Multi-headed attention from 'Attention Is All You Need' paperrY   r   c                    t         |           t        |d      r|j                  |   nd | _        || _        || _        t        |d|j                  |j                  z        | _
        |j                  |j                  z  | _        d| _        | j
                  j                  | _        d| _        t!        ||j                  |j                  | j                  z        | _        t!        ||j                  |j                  | j                  z        | _        t!        ||j                  |j                  | j                  z        | _        t!        ||j                  | j                  z  |j                        | _        t+        |j                  |j,                        | _        t+        |j                  |j,                        | _        t+        | j                  |j,                  d      | _        y )Nlayer_typesr   r   Frx   ry   r3  )re   rf   hasattrr  
layer_typerY   r   r  r   r   r   r  r  r  attention_dropout	is_causalrX   r   r   r   o_projrw   r  q_normk_normv_normr   s      rI   rf   zGemma4VisionAttention.__init__  s   ;B6=;Y&,,Y7_c"
F4F4F&JdJd4de$*$>$>&B\B\$\!!%!>!>+FF4F4FHbHbeiererHrs+FF4F4FHbHbeiererHrs+FF4F4FHbHbeiererHrs+FF4N4NQUQ^Q^4^`f`r`rs#V=P=PQ#V=P=PQ#DMMv7J7JW\]rH   NrO   r   rU   r   rF  r\   c                 N   |j                   d d }g |d| j                  }|\  }}	| j                  |      j                  |      }
| j	                  |
      }
t        |
||	|      }
|
j                  dd      }
| j                  |      j                  |      }| j                  |      }t        |||	|      }|j                  dd      }| j                  |      j                  |      }| j                  |      }|j                  dd      }t        j                  | j                  j                  t              } || |
|||f| j                   r| j"                  nd| j$                  d|\  }} |j&                  g |d j)                         }| j+                  |      }||fS )Nr   r/   r-   rU  )r  r  )r   r   r   r   r  r  r;  r   r  r   r  r    get_interfacerY   _attn_implementationr  r  r  r  r   r   r  )rm   rO   r   rU   r   rF  input_shaper   r   r   r   r   r   attention_interfacer   r   s                   rI   rr   zGemma4VisionAttention.forward  s    $))#2.88b8$--8&S{{=166|D{{<02<c<X#--a3[[/44\B
[[,
0S#|T
))!Q/
{{=166|D{{<0#--a3(?(M(MKK,,.E)
 %8	%
 /3mmD**LL	%
 	%
!\ *k));;;;FFHkk+.L((rH   r  )r<   r=   r>   r?   r3   rs   rf   r@   rF   
LongTensorr"   r$   rE   rr   rt   ru   s   @rI   r  r    s    G^1 ^c ^, -1.204,)||,) #\\,) t+	,)
 &&-,) +,,) 
u||U\\D0%2E2LL	M,)rH   r  c                       e Zd Zdedef fdZ	 	 	 ddej                  dej                  dej                  dz  dej                  dz  d	e	e
   d
eej                  eej                  ej                  f   dz  f   fdZ xZS )Gemma4VisionEncoderLayerrY   r   c                    t         |           || _        |j                  | _        || _        t        ||      | _        t        |      | _        t        | j                  |j                        | _        t        | j                  |j                        | _        t        | j                  |j                        | _        t        | j                  |j                        | _        y )NrY   r   ry   )re   rf   rY   r   r   r  rA  r{  mlprw   r  input_layernormpost_attention_layernormpre_feedforward_layernormpost_feedforward_layernormr   s      rI   rf   z!Gemma4VisionEncoderLayer.__init__  s    !--".f	R"6*,T-=-=6CVCVW(5d6F6FFL_L_(`%)6t7G7GVM`M`)a&*78H8HfNaNa*b'rH   NrO   r   rU   r   rF  r\   c                     |}| j                  |      } | j                  d||||d|\  }}| j                  |      }||z   }|}| j                  |      }| j	                  |      }| j                  |      }||z   }|S )N)rO   r   rU   r   rG   )r  rA  r  r  r  r  )rm   rO   r   rU   r   rF  r%  r   s           rI   rr   z Gemma4VisionEncoderLayer.forward  s     !,,];)4>> 
' 3)%	

 
q 55mD =0 66}E/77F =0rH   r  )r<   r=   r>   r3   rs   rf   r@   rF   r  r"   r$   rE   rA   rr   rt   ru   s   @rI   r  r    s    
c1 
cc 
c -1.204|| #\\ t+	
 &&- +, 
u  %(9(95;L;L(L"MPT"TT	UrH   r  c                        e Zd Zdef fdZ	 d
dej                  dej                  dej                  dz  dee	   de
f
d	Z xZS )Gemma4VisionEncoderrY   c           	         t         |           || _        |j                  | _        t        |      | _        t        j                  t        | j                        D cg c]  }t        ||       c}      | _        y c c}w )Nr  )re   rf   rY   num_hidden_layers
num_layersr  
rotary_embr   
ModuleListr  r  layers)rm   rY   r  rn   s      rI   rf   zGemma4VisionEncoder.__init__  sc     225f=mmKPQUQ`Q`Kaba%VqAb
bs   A?Ninputs_embedsrU   rO  rF  r\   c                     t        | j                  ||      }|}| j                  ||      }| j                  d| j                  j                   D ]  } ||f|||d|} t        |      S )z
        pixel_position_ids (torch.Tensor):
            Patch positions as (x, y) coordinates in the image as [batch, num_patches, 2].
        )rY   r  rU   N)rU   r   r   last_hidden_state)r   rY   r  r   r  r   )rm   r  rU   rO  rF  rO   r   decoder_layers           rI   rr   zGemma4VisionEncoder.forward  s     3;;')
 &"oom=OP "[[)H4;;+H+HI 	M)-$7/	
 M	 'GGrH   rp   )r<   r=   r>   r3   rf   r@   rF   r  r"   r$   r   rr   rt   ru   s   @rI   r  r    si    
1 
 7;	H||H H ",,t3	H
 +,H 
!HrH   r  c                   .     e Zd Zdedef fdZd Z xZS )Gemma4TextMLPrY   r   c                 J   t         |           |j                  |j                  z
  }||cxk\  xr dkD  nc }|j                  xr |}|| _        |j                  | _        |j                  |rdndz  | _        t        j                  | j                  | j                  d      | _
        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _        t        |j                     | _        y )Nr   r-   r/   Fr^   )re   rf   r  num_kv_shared_layersuse_double_wide_mlprY   r   r}  r   rh   r~  r  r  r   r  r  )rm   rY   r   first_kv_shared_layer_idxis_kv_shared_layerr
  rn   s         rI   rf   zGemma4TextMLP.__init__3  s    $*$<$<v?Z?Z$Z!&*CGaG$88O=O!--!'!9!9BUQ[\!]4#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXV556rH   c                     | j                  | j                  | j                  |            | j                  |      z        }|S rp   r  r  s      rI   rr   zGemma4TextMLP.forward@  r  rH   )r<   r=   r>   r2   rs   rf   rr   rt   ru   s   @rI   r  r  2  s    7/ 7C 7rH   r  c                        e Zd ZU ej                  ed<   ddef fdZe	 	 	 	 ddedz  de	d   de
dz  dedz  d	ed
ef   f
d       Z ej                         edd              Z xZS )Gemma4TextRotaryEmbeddingr  NrY   c                    t         
|           |j                  | _        |j                  | _        || _        t        |j                        | _        i | _        i | _	        | j                  D ]  }| j
                  j                  |   }||d   x}dk7  r
t        |   }n| j                  }|| j                  |<   || j                  |<   ||d}|dk(  r
|dk(  rd|d<    || j
                  fi |\  }}	| j                  | d|d	
       | j                  | d|j                         d	
       t        | | d|	        y )Nr  r  )r   r  full_attentionproportionalglobal_head_dimhead_dim_key	_inv_freqFr   _original_inv_freq_attention_scaling)re   rf   r  r  r  rY   setr  rope_init_fnsr  r  r   r  rj   r  setattr)rm   rY   r   r  rope_paramsr  r  rope_init_fn_kwargscurr_inv_freqcurr_attention_scalingrn   s             rI   rf   z"Gemma4TextRotaryEmbedding.__init__H  se   "("@"@$*$B$B!v112SU)+** 	UJ++55jAK"(55	)C29=#CC-9Dz*)2DNN:&-3:"N--)~2M6G#N34@4dPc4d1M1  J<y!9=UZ [  J</A!BMDWDWDYfk lDZL(:;=ST)	UrH   r   ztorch.devicer   r  r\   r  c                     | j                   |   d   }t        | dd      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                        j                  |t        j                        |z  z  z  }||fS )	a|  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        r  r   Nr   r   r-   r   r  r  )rY   r   r   r  r  rx   r  r  s           rI   r  z9Gemma4TextRotaryEmbedding.compute_default_rope_parametersh  s    2 %%j1,?fj$/c63E3EIcIc3c U\\!S!5;;?BB&X]XcXcBdgjjk
 )))rH   c                 N   t        | | d      }t        | | d      }|d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d	      5  |j                         |j                         z  j                  dd
      }	t        j                  |	|	fd      }
|
j                         |z  }|
j                         |z  }d d d        j	                  |j                        j	                  |j                        fS # 1 sw Y   AxY w)Nr  r  r   r   r/   r  r  Fr  r-   r   r   )r  rl   r  r   r   r   r  r  rD   r)   r;  r@   r   r   r   r   )rm   r   r   r  r  r  r  position_ids_expandedr  r  r  r   r   s                rI   rr   z!Gemma4TextRotaryEmbedding.forward  sl    4J<y!9:#DZL8J*KL$T1d]399;BB<CUCUVWCXZ\^_`ccdedldlm ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfkUC 	0&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')//C'')//C		0 vvAGGv$cff177f&;;;	0 	0s   *A1FF$NN)NNNNrp   )r<   r=   r>   r@   rF   rB   r2   rf   r  r   rs   rD   rE   rl   r  r   r   rr   rt   ru   s   @rI   r  r  E  s    llU/ U@ *.+/"!%	!* 4'!*(!* t!* $J	!*
 
~u$	%!* !*F U]]_<  <rH   r  c                       e Zd ZdZdedef fdZ	 ddej                  dej                  dej                  dz  d	e	e
eej                  ej                  f   f   d
edz  dee   deej                  ej                  dz  f   fdZ xZS )Gemma4TextAttentionr  rY   r   c                    t         |           t        |d      r|j                  |   nd | _        || _        || _        | j                  dk(  | _        | j                  r|j                  nd | _        | j                  s|j                  r|j                  n|j                  | _
        |j                  xr | j                   | _        | j                  r|j                  n|j                  }|j                  |z  | _        d| _        | j
                  j$                  | _        |j&                  dk7  | _        | j
                  j*                  t-        | j
                  dd      z
  }||cxk\  xr dk\  nc | _        |j                  d | }| j.                   xr6 |t1        |      dz
  |d d d   j3                  |j                  |         z
  k(  | _        t7        j8                  |j:                  |j                  | j                  z  |j<                  	      | _        tA        | j                  |jB                  
      | _"        | j.                  stA        | j                  |jB                  
      | _#        tA        | j                  |jB                  d      | _$        t7        j8                  |j:                  || j                  z  |j<                  	      | _%        | j                  s9t7        j8                  |j:                  || j                  z  |j<                  	      nd | _&        t7        j8                  |j                  | j                  z  |j:                  |j<                  	      | _'        y )Nr  sliding_attentionr   ro  r	  r   r/   r   r^   r  Fr3  )(re   rf   r  r  r  rY   r   
is_slidingsliding_windowr  r   attention_k_eq_vuse_alternative_attentionnum_global_key_value_headsr  r   r  r  r  use_bidirectional_attentionr  r  r  r  lenindexstore_full_length_kvr   rh   r   attention_biasr   rw   r  r  r  r  r   r   r  )rm   rY   r   r  r  prev_layersrn   s         rI   rf   zGemma4TextAttention.__init__  s   ;B6=;Y&,,Y7_c"//-@@7;f33D6:oo&J`J`..flfufu)/)@)@)XEX&151O1OF--U[UoUo 	 %+$>$>BU$U!!%!>!>;;uD %)KK$A$AGDKKYoqrDs$s!"+/H"MA"M(()C*CD(,(?(?$? %/IQTU`QadeQehsbDi

%""9-
.R/ E/! ii : :T]] JQWQfQf
 $6;N;NO &&'DMMv?R?RSDK'6;N;N[`aDK))""$7$--$GfNcNcDK
 55 		&,,.ADMM.QX^XmXmn K ii&&68J8JQWQfQf
rH   NrO   r   rU   r;   rN   rF  r\   c                    |j                   d d }g |d| j                  }|\  }	}
| j                  |      j                  |      }| j	                  |      }t        ||	|
d      }|j                  dd      }| j                  rI|| j                     \  }}|j                  |j                        }|j                  |j                        }n| j                  |      j                  |      }| j                   | j                  |      j                  |      n|}| j                  |      }t        ||	|
d      }|j                  dd      }| j                  |      }|j                  dd      }|,| j                  s |j                  ||| j                         \  }}| j"                  r||f|| j                  <   t%        j&                  | j(                  j*                  t,              } || ||||f| j.                  r| j0                  nd| j2                  | j4                  d|\  }} |j6                  g |d j9                         }| j;                  |      }||fS )Nr   r-   )r  r/   rU  )r  r  r(  )r   r   r   r   r  r  r;  r  r  r   r   r   r   r  r  updater   r/  r    r  rY   r  r  r  r  r  r(  r   r   r  )rm   rO   r   rU   r;   rN   rF  r  r   r   r   r   r   r   r  r   r   s                    rI   rr   zGemma4TextAttention.forward  sR    $))#2.88b8$--8&S{{=166|D{{<0+L#sRST#--a3
 ""'7'H$J#|':':;J'??<+>+>?L]388FJLPKKLc4;;}5::<HisLZ0J-j#sRSTJ#--a3J;;|4L'11!Q7L&t/F/F'6'='=j,X\XfXf'g$J$$0:L0HT__-(?(M(MKK,,.E)
 %8
%
 /3mmD**LL..
%
 
%
!\ *k));;;;FFHkk+.L((rH   rp   )r<   r=   r>   r?   r2   rs   rf   r@   rF   rC   rD   rE   r   r"   r   rr   rt   ru   s   @rI   r$  r$    s    G/
/ /
C /
n )-=)||=) #\\=) t+	=)
 sE%,,*D$EEF=) =) -.=) 
u||U\\D00	1=)rH   r$  c                        e Zd ZdZdef fdZdej                  dej                  dej                  dej                  fdZ xZ	S )	Gemma4TextExpertsz2Collection of expert weights stored as 3D tensors.rY   c                    t         |           |j                  | _        |j                  | _        |j
                  | _        t        j                  t        j                  | j                  d| j                  z  | j                              | _        t        j                  t        j                  | j                  | j                  | j                              | _        t        |j                     | _        y )Nr-   )re   rf   num_expertsr   
hidden_dimmoe_intermediate_sizeintermediate_dimr   r}   r@   emptygate_up_projr  r   r  r  r!  s     rI   rf   zGemma4TextExperts.__init__  s    !-- ,, & < <LLT5E5Eq4K`K`G`bfbqbq)rsekk$2B2BDOOUYUjUj&klV556rH   rO   top_k_indextop_k_weightsr\   c                 f   t        j                  |      }t        j                         5  t         j                  j                  j                  || j                        }|j                  ddd      }t        j                  |j                  d      d      j                         }d d d        D ]  }|d   }|| j                  k(  rt        j                  |         \  }}	||	   }
t        j                  j                  |
| j                  |         j                  dd      \  }}| j                  |      |z  }t        j                  j                  || j                   |         }|||	|d f   z  }|j#                  d|	|j%                  |j&                                |S # 1 sw Y   xY w)N)num_classesr-   r/   r   )r   r   r   )r@   
zeros_liker   r   r	   rm  r7  r   greatersumnonzerorW  ri   r<  chunkr  r  
index_add_r   r   )rm   rO   r=  r>  final_hidden_statesexpert_mask
expert_hit
expert_idx	top_k_pos	token_idxcurrent_stategateupcurrent_hidden_statess                 rI   rr   zGemma4TextExperts.forward   s    $..}=]]_ 	S((--55ktO_O_5`K%--aA6K{8'DaHPPRJ	S
 % 
	nJ#AJT---#(;;{:/F#G Iy))4M}}++M4;L;LZ;XY__`agi_jHD"$(KK$5$:!$&MM$8$89NPTP^P^_iPj$k!$9M)U^`dJd<e$e!**1i9N9Q9QReRkRk9lm
	n #"#	S 	Ss   A=F&&F0)
r<   r=   r>   r?   r2   rf   r@   rF   rr   rt   ru   s   @rI   r5  r5    sN    <7/ 7#||# \\# ||	#
 
#rH   r5  c                   z     e Zd Zdef fdZdej                  deej                  ej                  f   fdZ xZ	S )Gemma4TextRouterrY   c                 (   t         |           || _        |j                  | _        | j                  dz  | _        |j
                  | _        t        | j                  | j                  d      | _        t        j                  |j                  |j                  d      | _        t        j                  t        j                  | j                              | _        t        j                  t        j                  |j                              | _        y )Nr   Fr3  r^   )re   rf   rY   r   scalar_root_sizer  ry   rw   r  r   rh   r7  projr}   r@   r~   scaleper_expert_scaler!  s     rI   rf   zGemma4TextRouter.__init__<  s    !-- $ 0 0$ 6&&!$"2"2US	IIf00&2D2D5Q	\\%**T-=-=">?
 "UZZ8J8J-K LrH   rO   r\   c                    | j                  |      }|| j                  z  | j                  z  }| j                  |      }t        j
                  j                  |dt        j                        }t        j                  || j                  j                  d      \  }}||j                  dd      z  }|| j                  |   z  }|||fS )Nr   r   )rq  rx   Trh  )r  rW  rU  rV  r   r	   r   r@   r   topkrY   top_k_expertsrD  rX  )rm   rO   expert_scoresrouter_probabilitiesr>  r=  s         rI   rr   zGemma4TextRouter.forwardH  s    		-0%

2T5J5JJ		-0!}}44]RWR_R_4` &+ZZ kk''&
"{ 	**r4*@@ &(=(=k(JJ#]K??rH   )
r<   r=   r>   r2   rf   r@   rF   rE   rr   rt   ru   s   @rI   rS  rS  ;  s>    
M/ 
M@U\\ @eELL%,,<V6W @rH   rS  c                   0    e Zd Zdeez  def fdZ	 	 	 	 	 	 ddej                  dej                  de	e
eej                  ej                  f   f   dz  dej                  d	ej                  dz  d
ej                  dz  dedz  dej                  fdZ xZS )Gemma4TextDecoderLayerrY   r   c                    t         |           || _        |j                  | _        || _        t        ||      | _        t        ||      | _        t        | j                  |j                        | _        t        | j                  |j                        | _        t        | j                  |j                        | _        t        | j                  |j                        | _        | j                  dt!        j"                  d             |j$                  | _        | j$                  rt&        |j(                     | _        t-        j.                  | j                  | j$                  d      | _        t-        j.                  | j$                  | j                  d      | _        t        | j                  |j                        | _        |j6                  | _        | j6                  rt9        |      | _        t=        |      | _        t        | j                  |j                        | _         t        | j                  |j                        | _!        t        | j                  |j                        | _"        y y )Nr  r  layer_scalarr/   Fr^   )#re   rf   rY   r   r   r$  rA  r  r  rw   r  r  r  r  r  rj   r@   r~   hidden_size_per_layer_inputr   r  r  r   rh   per_layer_input_gateper_layer_projectionpost_per_layer_input_normenable_moe_blockrS  routerr5  expertspost_feedforward_layernorm_1post_feedforward_layernorm_2pre_feedforward_layernorm_2r   s      rI   rf   zGemma4TextDecoderLayer.__init__a  s   !--",FiP 3,T-=-=6CVCVW(5d6F6FFL_L_(`%)6t7G7GVM`M`)a&*78H8HfNaNa*b'^UZZ];+1+M+M(++ !9!9:DK(*		$2B2BDDdDdkp(qD%(*		$2R2RTXTdTdkp(qD%-:4;K;KQWQdQd-eD* & 7 7  *62DK,V4DL0=d>N>NTZTgTg0hD-0=d>N>NTZTgTg0hD-/<T=M=MSYSfSf/gD, !rH   NrO   per_layer_inputr;   r   rU   r   rN   r\   c           
      &   |}	| j                  |      } | j                  d||||||d|\  }}
| j                  |      }|	|z   }|}	| j                  |      }| j	                  |      }| j
                  r| j                  |      }|	j                  d|	j                  d         }| j                  |      \  }
}}| j                  |      }| j                  |||      }|j                  |	j                        }| j                  |      }||z   }| j                  |      }|	|z   }| j                  rP|}	| j                  |      }| j!                  |      }||z  }| j#                  |      }| j%                  |      }|	|z   }|| j&                  z  }|S )N)rO   r   rU   r;   r   rN   r   rG   )r  rA  r  r  r  rf  ri  r   r   rg  rk  rh  rj  r  rb  rc  r  rd  re  ra  )rm   rO   rl  r;   r   rU   r   rN   rF  r%  r   hidden_states_1hidden_states_flatr>  r=  hidden_states_2s                   rI   rr   zGemma4TextDecoderLayer.forward}  s    !,,];)4>> 
' 3)-%+
 
q 55mD =0 66}E/  "??NO "*!1!1"hnnR6H!I,0KK8J,K)A}k">>?QRO"ll?KWO-55hnnEO"??PO ,o=M77F =0++$H 55mDM KK6M)O;M 55mDM ::=IM$}4M***rH   )NNNNNN)r<   r=   r>   r2   r3   rs   rf   r@   rF   rC   rD   rE   r  r   rr   rt   ru   s   @rI   r_  r_  `  s    h/2DD hQT h> )-PT,0.204(,9||9 9 sE%,,*D$EEFM	9
 #\\9 t+9 &&-9 9 
9rH   r_  c            	       Z     e Zd ZdZd	dedededef fdZdej                  f fdZ	 xZ
S )
Gemma4TextScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    num_embeddingsembedding_dimpadding_idxembed_scalec                     t         |   |||       || _        | j                  dt	        j
                  |      d       y )Nrv  Fr   )re   rf   scalar_embed_scalerj   r@   rk   )rm   rs  rt  ru  rv  rn   s        rI   rf   z&Gemma4TextScaledWordEmbedding.__init__  s;    D"-]ELL,ERWXrH   	input_idsc                     t         |   |      | j                  j                  | j                  j
                        z  S rp   )re   rr   rv  r   r   r   )rm   ry  rn   s     rI   rr   z%Gemma4TextScaledWordEmbedding.forward  s2    wy)D,<,<,?,?@Q@Q,RRRrH   )r   )r<   r=   r>   r?   rs   rl   rf   r@   rF   rr   rt   ru   s   @rI   rr  rr    sG    Ys Y3 YS Y_d Y
S S SrH   rr  c            	            e Zd ZU eed<   dZdZg dZddgZdZ	dZ
dZdZdZdZdZ ej"                          fd	       Zd
 Zd Z	 	 	 ddedz  dedz  dedej0                  f fdZ	 	 	 ddedz  dedz  defdZ xZS )Gemma4PreTrainedModelrY   modelT)r_  r  rI  r=  rN   r;   N)imagetextvideoaudioc                 	   t         |   |       t        |t              r t	        j
                  |j                         y t        |t              rd}d}|j                  dz  }t        j                  ||z        t        |dz
  d      z  }|t        j                  t        j                  |      | z        z  }t	        j                  |j                   |j#                  d      j#                  d             y t        |t$              rJt	        j&                  |j(                  |j*                         t	        j,                  |j.                         y t        |t0              r|j2                  j5                         D ]  \  }}d|i}	|dk(  r|j6                  |   dk(  rd	|	d
<    ||j8                  fi |	\  }
}t	        j                  t;        || d      |
       t	        j                  t;        || d      |
        y t        |t<              r|j6                  dk7  rt>        |j6                     n|j@                  } ||j8                        \  }}t	        j                  |jB                  |       t	        j                  |jD                  |       y t        |tF              r+t	        j&                  |jH                  |jJ                         y t        |tL              r?t	        j
                  |jN                         t	        j
                  |jP                         y t        |tR              r[| j8                  jT                  }t	        jV                  |jX                  d|       t	        jV                  |jZ                  d|       y t        |t\              r t	        j
                  |j^                         y t        |t`              r|jb                  rt	        j&                  |jd                  tg        d              t	        j&                  |jh                  tg        d             t	        j&                  |jj                  tg        d              t	        j&                  |jl                  tg        d             y t        |tn              rV|j8                  jp                  r?t	        j,                  |jr                         t	        j
                  |jt                         y y y )Nr   r   r-   r/   r   r  r  r  r  r  r  r  r  rU  )r   stdra   );re   _init_weightsr  rI  initones_rN  r   r   r   r   r   r@   r   r   copy_r   r   r   	constant_r   r   zeros_r   r  r  itemsr  rY   r  r  r   r  r  r  rr  rv  rx  rS  rW  rX  r5  initializer_rangenormal_r<  r  r_  ra  rX   rg   r`   rl   rb   rc   rd   Gemma4VisionModelstandardizestd_bias	std_scale)rm   r  r   r   r   r   r   r  r  r  r  r   rope_fnbuffer_valuer  rn   s                  rI   r  z#Gemma4PreTrainedModel._init_weights  s   f%f78JJv667 @AM#M#//14N&*hh}}/L&MPSTbefTfhiPj&j#*UYYu||N7SWnVn7n-ooNJJv,,n.F.Fq.I.S.STU.VW 45NN6>>6+K+KLKK,,- 9:,2,@,@,F,F,H ^(
L'3Z&@#!11f6F6Fz6RVd6d:K'7#/#UAT#U q

76j\+CDmT

76j\9K+LM}]^  ;< ##y0 $F$4$45;; 
 &fmm4OL!JJv5JJv//> =>NN6--v/H/HI 01JJv||$JJv../ 12++//CLL,,3C@LL))= 67JJv**+ 566;U;UNN6++eEl];NN6++U5\:NN6,,uU|m<NN6,,eEl; 12v}}7P7PKK(JJv''( 8Q2rH   c                 .    | j                   j                  S rp   
base_modelembed_tokens_per_layerrm   s    rI   get_per_layer_input_embeddingsz4Gemma4PreTrainedModel.get_per_layer_input_embeddings  s    555rH   c                 &    || j                   _        y rp   r  rm   r  s     rI   set_per_layer_input_embeddingsz4Gemma4PreTrainedModel.set_per_layer_input_embeddings  s    16.rH   new_num_tokenspad_to_multiple_ofmean_resizingr\   c                 R    t         |   |||      }| j                  |||       |S )N)r  r  r  )re   resize_token_embeddings_resize_per_layer_embeddings)rm   r  r  r  r  rn   s        rI   r  z-Gemma4PreTrainedModel.resize_token_embeddings  s=     7)1' 8 

 	)).:Lm\rH   c                    | j                   | j                  j                         _        | j                  j                         j                  r| j                         }| j                  ||||      }t        |d      r|j                  }t        ||       |j                  |j                  j                         | j                  |       y y )N_hf_hook)
vocab_sizerY   get_text_configvocab_size_per_layer_inputrb  r  _get_resized_embeddingsr  r  r4   requires_grad_r   r|   r  )rm   r  r  r  r  new_embeddings_per_layerhooks          rI   r  z2Gemma4PreTrainedModel._resize_per_layer_embeddings%  s     DH??##%@;;&&(DD%)%H%H%J"'+'C'C&8JM($ -z:-66"#;TB$334J4Q4Q4_4_`//0HI ErH   )NNT)r<   r=   r>   r1   rB   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraph_supports_attention_backend_can_record_outputsinput_modalitiesr@   r   r  r  r  rs   r   r   	Embeddingr  r  rt   ru   s   @rI   r|  r|    s    &*# $56H"IN!"&:U]]_2) 2)h67
 &*)-"	d
  $J 	
 
  &*)-"	Jd
J  $JJ 	JrH   r|  zAThe base Gemma 4 language model without a language modeling head.c                       e Zd ZU eed<   dZ eed      ee	dZ
def fdZeee	 	 	 	 	 	 	 ddej                   dz  d	ej"                  dz  d
ej                   dz  dedz  dej&                  dz  dej"                  dz  dedz  dee   defd                     Zdej"                  dz  dej"                  dz  dej"                  fdZ	 ddej"                  dej"                  dz  dej"                  fdZ xZS )Gemma4TextModelrY   )r  r   )r.  )router_logitsrO   rP   c           
         t         |   |       |j                  | _        |j                  | _        t        |j                  |j                  | j                  | j                  j                  dz        | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        |j                  |j                         | _        t%        |      | _        d| _        t+        | j                  j,                        | _        |j0                  | _        | j0                  rt        |j2                  |j                  |j0                  z  | j                  |j0                  dz        | _        d| _        t        j8                  |j                  |j                  |j0                  z  d      | _        |j                  dz  | _        t        |j0                  |j                         | _        g | _         tC        | j                        D ]K  \  }}|jD                  jF                  s| j@                  jI                  dD cg c]
  }d	| d
|  c}       M | jK                          y c c}w c c}w )Nr^  )rv  r  Fg;f?r^   r   )r   r   r  r  zlayers.z.self_attn.)&re   rf   pad_token_idru  r  rr  r   rY   embed_tokensr   r  r  r  r_  r   rw   r  r  r  r  gradient_checkpointingr  r  unique_layer_typesrb  r  r  per_layer_input_scalerh   per_layer_model_projection per_layer_model_projection_scaleper_layer_projection_norm"_keys_to_ignore_on_load_unexpected	enumeraterA  r  extend	post_init)rm   rY   r   r  layernamern   s         rI   rf   zGemma4TextModel.__init__B  s$    !.. ++ :v1143C3CQUQ\Q\QhQhjmQm
 mmHMfNfNfHgh9#FI6h
 "&"4"4&:M:MN	3F;&+#"%dkk&=&=">
 ,2+M+M(++*G11((6+M+MM  ">>C	+D' *3D&.0ii""((6+M+MM/D+
 5;4F4F4LD1-:6;];]cicvcv-wD* 35/!$++. 	HAu1177>>@hiwqcTF3i	 	I i@ js   "JJ

Nry  rU   r   rN   r  per_layer_inputs	use_cacherF  r\   c           
      <   |du |duz  rt        d      ||t        d      || j                  |      }| j                  r&|| j                  ||      }| j	                  ||      }|r|t        | j                        }|V||j                         nd}	t        j                  |j                  d   |j                        |	z   }|j                  d      }t        |x}
t              s)| j                  ||||d}t        di |t!        di |d	}
|}i }| j"                  D ]  }| j%                  |||      ||<    |j'                  d
t)                     }t+        | j,                  d| j                  j.                         D ]\  \  }}||dddd|ddf   nd} |||f||| j                  j0                  |      |
| j                  j0                  |      ||d|}^ | j3                  |      }t5        |||j7                  dd      r|      S d      S )ak  
        per_layer_inputs (`torch.Tensor`, *optional*):
            Pre-computed per-layer input text embeddings of shape `(batch_size, sequence_length, num_hidden_layers,
            hidden_size_per_layer_input)`. When provided, these are used directly instead of being computed from `input_ids`
            via `get_per_layer_inputs()` in the text model. If calling the `forward` with `inputs_embeds` instead of `input_ids`,
            you should probably precompute them and forward them along `inputs_embeds`, otherwise recomputing them needs
            to reverse the main embedding, which is expensive.
        N:You must specify exactly one of input_ids or inputs_embeds<You cannot specify per_layer_inputs if input_ids is providedrY   r   r/   r   rY   r  rU   rN   r   r  r&  r;   )r;   r   rU   r   rN   return_shared_kv_statesF)r  rN   r;   rG   )rk  r  rb  get_per_layer_inputsproject_per_layer_inputsr   rY   get_seq_lengthr@   r   r   r   r   r  rC   r   r   r  r  popr   r  r   r  r  r  rR   get)rm   ry  rU   r   rN   r  r  r  rF  past_seen_tokenscausal_mask_mappingmask_kwargsrO   r   r  r;   r  r  rl  s                      rI   rr   zGemma4TextModel.forwardr  s}   , -t";<YZZ %5%A[\\  --i8M++'#'#<#<Y#V #<<]L\]0*$++>OCRC^==?de <<(;(;A(>}G[G[\_ooL'11!4L ?-F ++!."0#2 ,K #5"C{"C%F%U%U# & 11 	gJ.2oom\[e.f
+	g "::&8(*E !*$++6U8U8U*V W 	A}>N>Z.q!Qz:`dO)	 "2$78O8OPQ8R$S24;;3J3J13MN) /	 	M	 		-0,++17<UW\1]-
 	
 dh
 	
rH   c                    | j                   st        d| j                         |t        j                         5  |dddddddf   | j
                  j                  ddddddf   | j                  j                  dz  z  k(  j                  d      j                         dddf   }	 |j                  |j                  dd       }	 ddd        | j                  |      j                  g |j                  | j                  j                  | j                    S # t        $ r t        d      w xY w# 1 sw Y   nxY w)a  Compute the token-identity component of Per-Layer Embeddings (PLE).

        Looks up `input_ids` in `embed_tokens_per_layer` (a scaled embedding that multiplies
        by `sqrt(hidden_size_per_layer_input)`) and reshapes the packed output from
        `[batch, seq, num_hidden_layers * hidden_size_per_layer_input]` to
        `[batch, seq, num_hidden_layers, hidden_size_per_layer_input]`.

        If only `inputs_embeds` is provided (no `input_ids`), reverses the main embedding
        to recover `input_ids` for the PLE lookup.
        z}Attempting to call get_per_layer_inputs() from a model initialized with a config that does not support per-layer embeddings. Nr^  r
   r   r-   a)  It seems like you tried to call `forward` from `inputs_embeds` without providing `input_ids`, and that the `inputs_embeds` you provided do not exactly match the embedding weights. Since Gemma4 needs to reverse the embedding to compute another embedding, make sure you provide exact `inputs_embeds`)rb  RuntimeErrorrY   r@   r   r  r   r   ro  rE  r   r   r  r   r  )rm   ry  r  s      rI   r  z$Gemma4TextModel.get_per_layer_inputs  sU    //**.++8    &aD!m4,,33D$14DEH_H_adHdde SQSZWYq!t%  )}/B/B2A/F GI$ >t**95== 
__
KK))
 ,,
 	
 $ &r  s   A1D9-D!!D66D99Ec                 V   | j                   st        d| j                         | j                  |      | j                  z  } |j
                  g |j                  dd | j                  j                  | j                    }| j                  |      }||S ||z   | j                  z  S )a  Compute the context-aware component of PLE and combine with token-identity.

        Projects `inputs_embeds` through `per_layer_model_projection` (Linear), scales by
        `1/sqrt(hidden_size)`, reshapes to `[batch, seq, num_layers, ple_dim]`, and normalizes
        with `per_layer_projection_norm` (RMSNorm).

        If `per_layer_inputs` (the token-identity component from `get_per_layer_inputs()`)
        is provided, combines both: `(context_projection + token_identity) * (1/sqrt(2))`.
        If `per_layer_inputs` is None (e.g. for multimodal inputs where input_ids are not
        available), returns just the context projection.
        zAttempting to call project_per_layer_inputs() from a model initialized with a config that does not support per-layer embeddings. Nr   )
rb  r  rY   r  r  r   r   r  r  r  )rm   r  r  rd  s       rI   r  z(Gemma4TextModel.project_per_layer_inputs  s      //226++@ 
  $>>}MPTPuPuu;3;;  
  "% 
KK)) 
 ,, 

  $==>RS#''$'774;U;UUUrH   )NNNNNNNrp   )r<   r=   r>   r2   rB   r  r+   rS  r_  r$  r  rf   r*   r,   r%   r@   r  rF   r   rA   r   r"   r$   rR   rr   r  r  rt   ru   s   @rI   r  r  8  s    '(8B/)./ .`   .2.204(,2604!%Y
##d*Y
 t+Y
 &&-	Y

 Y
 ((4/Y
  ,,-Y
 $;Y
 +,Y
 
'Y
    Y
v*
ellT.A *
RWR^R^aeRe *
jojvjv *
^ 15!V||!V  ,,-!V 
	!VrH   r  z>The base Gemma 4 language model with a language modeling head.c                   x    e Zd ZU ddiZddiZddgdgfiZeed<   dZdef fd	Z	e
e	 	 	 	 	 	 	 	 	 ddej                  d
z  dej                  d
z  dej                  d
z  ded
z  dej                   d
z  dej                  d
z  ded
z  deej                  z  dej                  d
z  dee   defd              Z xZS )Gemma4ForCausalLMlm_head.weightzmodel.embed_tokens.weightlm_headcolwise_gather_outputrO   rM   rY   r}  c                     t         |   |       t        |      | _        |j                  | _        t        j                  |j                  |j                  d      | _        | j                          y NFr^   )
re   rf   r  r}  r  r   rh   r   r  r  r!  s     rI   rf   zGemma4ForCausalLM.__init__(  sU     $V,
 ++yy!3!3V5F5FUS 	rH   Nry  rU   r   rN   r  labelsr  logits_to_keepr  rF  r\   c
                 4    | j                   d||||||	|d|
}|j                  }t        |t              rt	        | d      n|}| j                  |dd|ddf         }| j                  j                  G|| j                  j                  z  }t        j                  |      }|| j                  j                  z  }d}| | j                  ||| j                  fi |
}t        |||j                  |j                  |j                  |j                         S )a  
        per_layer_inputs (`torch.Tensor`, *optional*):
            Pre-computed per-layer input text embeddings of shape `(batch_size, sequence_length, num_hidden_layers,
            hidden_size_per_layer_input)`. When provided, these are used directly instead of being computed from `input_ids`
            via `get_per_layer_inputs()` in the text model. If calling the `forward` with `inputs_embeds` instead of `input_ids`,
            you should probably precompute them and forward them along `inputs_embeds`, otherwise recomputing them needs
            to reverse the main embedding, which is expensive.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Gemma4ForCausalLM

        >>> model = Gemma4ForCausalLM.from_pretrained("google/gemma-4-E2B-it")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-4-E2B-it")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```)ry  rU   r   rN   r  r  r  N)rL   rM   rN   rO   rP   r;   rG   )r}  r  r  rs   slicer  rY   final_logit_softcappingr@   r   loss_functionr  rK   rN   rO   rP   r;   )rm   ry  rU   r   rN   r  r  r  r  r  rF  outputsrO   slice_indicesrM   rL   s                   rI   rr   zGemma4ForCausalLM.forward1  s"   P 2< 	2
)%+'-	2
 	2
  118B>SV8W~ot4]kmA}a,?@A;;..:dkkAAAFZZ'FdkkAAAF%4%%ffdooPPD+#33!//))$55
 	
rH   )	NNNNNNNr   N)r<   r=   r>   _tied_weights_keys_tp_plan_pp_planr2   rB   r  rf   r&   r%   r@   r  rF   r   rA   r   rs   r"   r$   rK   rr   rt   ru   s   @rI   r  r     sN   *,GH23H_-z:;H/   .2.204(,26*.!%-.04E
##d*E
 t+E
 &&-	E

 E
 ((4/E
   4'E
 $;E
 ell*E
  ,,-E
 +,E
 
&E
  E
rH   r  r(  c           
      P     dt         dt         dt         dt         dt        f
 fd}|S )zL
    This creates uni/bidirectional attention mask with sliding window.
    	batch_idxhead_idxq_idxkv_idxr\   c                 P    	\  }}||z
  }|dk\  ||k  z  }|dk  | |k  z  }||z  S r.  rG   )
r  r  r  r  left_window_sizeright_window_sizedist	left_mask
right_maskr(  s
            rI   
inner_maskz0sliding_window_mask_function.<locals>.inner_mask  sM    .<++v~QY4*:#:;	QhD5+<#<=
:%%rH   )rs   r   )r(  r  s   ` rI   sliding_window_mask_functionr  {  s3    
&c &S & &c &d & rH   c                   ,    e Zd ZU dZeed<   dZdZee	dZ
def fdZdej                  dej                  fd	Zee ed
      	 ddej                  dej                  dz  dee   deej                  ej*                  f   fd                     Z xZS )Gemma4AudioModelznAn audio encoder based on the [Universal Speech Model](https://huggingface.co/papers/2303.01037) architecture.rY   r  zmodel.audio_towerrO   rP   c           	         t         |   |       || _        t        |      | _        t        |      | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        j                  |j                  |j                  d      | _        | j#                          y c c}w )NTr^   )re   rf   rY   r
  subsample_conv_projectionr   rel_pos_encr   r  r  r  r=  r   rh   r   output_proj_dimsoutput_projr  r   s      rI   rf   zGemma4AudioModel.__init__  s     )KF)S&;FCmmBGH`H`BabYfi0b
 99V%7%79P9PW[\	 cs   B?mask_4dr\   c                    |j                   \  }}}}|j                  }| j                  j                  }| j                  j                  dz
  }| j                  j
                  }||z   dz
  |z  }	|	|z  }
|
|z
  }t        j                  |d|d|fd      }|j                  |d|	||
      }t        j                  |||fd      }t        j                  |	|      |z  }t        j                  ||z   |z   |      }|dddf   |dddf   z   }|dddddddf   j                  |dd|d      }|j                  d|      S )z
        Convert a standard 4D attention mask `[batch_size, 1, seq_len, seq_len]` to the 5D blocked format
        `[batch_size, 1, num_blocks, chunk_size, context_size]` expected by the chunked local attention,
        r/   r   F)r  r   Nr   )r   r   rY   r   r   r   r   r   r   r@   r   r  gather)rm   r  r   r   r   r   r   r   r   r   padded_seq_len
pad_amountmask_5dblock_startsoffsets
kv_indicess                   rI   _convert_4d_mask_to_blocked_5dz/Gemma4AudioModel._convert_4d_mask_to_blocked_5d  sN   
 %,MM!
Aw[[55
;;==A![[@@
*Q.:=
#j0#g-
%%!ZJ!?uM//*aZX%%"24F!GuU||Jv>K,,z,<<?QQZ`a!!T'*WT1W-==
dAtQ 67>>z1bR\^`a
~~b*--rH   z&Encodes audio features to soft tokens.r5   NrU   rF  c           	         | j                  ||      \  }}| j                  |      }t        | j                  ||t	        | j                  j
                  dz
  | j                  j                  f            }|| j                  |      }| j                  d | j                  j                   D ]  } ||f||d|} | j                  |      }t        ||      S )Nr/   )rY   r  rU   and_mask_function)rU   r   )r  rU   )r   r  r   rY   r  r   r   r  r   r  r  rT   )rm   r  rU   rF  rO   output_maskr   encoder_layers           rI   rr   zGemma4AudioModel.forward  s     &*%C%CNTb%c"{"..}=2;;'&:33a79\9\]	
 %!@@PN![[)H4;;+H+HI 	M)-$7 	M	 ((7%VabbrH   rp   )r<   r=   r>   r?   r0   rB   main_input_namer  r=  r   r  rf   r@   rF   r  r*   r,   r%   r"   r$   rE   rV   rr   rt   ru   s   @rI   r  r    s    x&O+)*
0 .ell .u|| .6  !IJ /3cc t+c +,	c
 
u||U---	.c K   crH   r  c                        e Zd ZdZeZeedZdef fdZ	e
e ed      dej                  dej                  d	ee   d
efd                     Z xZS )r  zThe Gemma 4 Vision Encoder.r  rY   c                    t         |   |       t        |      | _        t	        |      | _        t        |      | _        | j                  j                  rr| j                  dt        j                  | j                  j                               | j                  dt        j                  | j                  j                               | j                          y )Nr  r  )re   rf   rI  patch_embedderr  encoderrb  poolerrY   r  rj   r@   r;  r   r  r!  s     rI   rf   zGemma4VisionModel.__init__  s     7?*62(0;;""  U[[9P9P-QR  ekk$++:Q:Q.RSrH   z1Encodes image pixels to soft tokens from patches.r5   r\  rO  rF  r\   c                    | j                   j                  }|j                  d   ||z  z  }|dk(  j                  d      }| j	                  |||      } | j
                  d|| |d|}| j                  |j                  |||      \  }	}
|	|
   }	| j                   j                  r8|	| j                  j                         z
  | j                  j                         z  }	|	j                  |j                        }	t        |	      S )a  
        pixel_values (`torch.FloatTensor` or `list[torch.FloatTensor]`):
            The images to encode. Either a single `[batch, channels, height, width]` tensor
            (all images same size) or a list of `[1, channels, height, width]` tensors (different sizes).
        pixel_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`):
            The patch positions as (x, y) coordinates in the image. Padding patches are indicated by (-1, -1).
        rA  r   r   )r  rU   rO  )rO   rO  rP  rx  r  rG   )rY   pooling_kernel_sizer   ro  r  r  r  r  r  r  rl   r  r   r   r   )rm   r\  rO  rF  r  rx  rP  r  rv  rO   pooler_masks              rI   rr   zGemma4VisionModel.forward  s     #kk==$**2.3FI\3\]/25::r:B++L:LN_` 
'--1
 	
 &*[[ 221/'	 &1 &
"{ &k2 ;;""*T]]-@-@-BBdnnFZFZF\\M%(()<)<=&GGrH   )r<   r=   r>   r?   r3   rY   r  r  r  rf   r*   r,   r%   r@   rA   r  r"   r$   r   rr   rt   ru   s   @rI   r  r    s    %F1+

1 
  !TU)H'')H ",,)H +,	)H
 
!)H V   )HrH   r  c                   j     e Zd ZdZdeez  def fdZdej                  dej                  fdZ
 xZS )Gemma4MultimodalEmbedderzQEmbeds token ids or soft tokens for multimodal content into language model space.multimodal_configtext_configc                 N   t         |           t        |d|j                        | _        |j
                  | _        |j                  | _        t        j                  | j                  | j                  d      | _
        t        | j                  | j                  d      | _        y )Nr  Fr^   r3  )re   rf   r  r   multimodal_hidden_sizer  ry   text_hidden_sizer   rh   embedding_projectionrw   embedding_pre_projection_norm)rm   r  r  rn   s      rI   rf   z!Gemma4MultimodalEmbedder.__init__(  s    
 	&-.?ASUfUrUr&s#$11 + 7 7$&IId.I.I4K`K`gl$m!-:4;V;V\`\d\dqv-w*rH   r  r\   c                 F    | j                  |      }| j                  |      S )a:  Embeds token ids or soft tokens for multimodal content into language model space.
        Args:
            inputs_embeds: A torch.Tensor containing the soft tokens to embed.
        Returns:
            A torch.Tensor of embeddings with shape `[batch_size, seq_len, self.config.text_config.hidden_size]`.
        )r#  r"  )rm   r  embs_normeds      rI   rr   z Gemma4MultimodalEmbedder.forward5  s%     88G((55rH   )r<   r=   r>   r?   r0   r3   r2   rf   r@   rF   rr   rt   ru   s   @rI   r  r  %  sA    [x,/AAx &x6U\\ 6ell 6rH   r  rY   r  rN   block_sequence_idsc                     | ||||d}t        di |}t        di |ddi\  }}	}	}	}
}	}|r|}nt        |||
|      }t        di |t        |      t	        | j
                        d}||dS )a  Create full_attention and sliding_attention masks with correct composition.

    For global (full attention) layers:  causal only (no bidirectional)
    For local (sliding window) layers:  AND(sliding_window, OR(causal, blockwise))

    Unlike Gemma 3 (which applies bidirectional attention on all layers), Gemma 4
    explicitly disables bidirectional attention on global attention layers.
    r  r   r   )or_mask_functionr  r  rG   )r   r   r   r   r   r(  )rY   r  rU   rN   r   r&  r  	full_mask
early_exitr   	kv_length	kv_offsetpadded_block_sequence_idssliding_masks                 rI   create_masks_for_vision_modelr/  @  s    " &(*$K #1[1I 4N 4
440J1aAy $6!$@	9%
! & 
*+DE01F1FGL $) rH   mm_token_type_idsr   c                    | j                  |      } | dk(  | dk(  z  }t        j                  |dd      }d|d<   || z  }t        j                  |j	                         d      dz
  }t        j
                  ||d      }|S )Nr/   r-   r   )shiftsdimsFrS  r   )r   r@   rollcumsumrs   rW  )r0  r   	is_visionis_prev_visionnew_vision_startsvision_group_idsr&  s          rI   get_block_sequence_ids_for_maskr:  w  s    ),,V4"a',=,BCIZZ	!"=N"N6!^O3||$5$9$9$;CaGY0@"ErH   z
    The base Gemma 4 model comprising a vision backbone, an audio backbone, and a language model without a
    language modeling head.
    c            $           e Zd ZdZdef fdZe ed      	 d dej                  dej                  dz  d	ee   d
efd              Z	 	 d!dej                  dz  dej                  dz  d
eej                   ej                   ej                   f   fdZeee	 	 	 	 	 	 	 	 	 	 	 	 	 	 d"dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej&                  dz  dej&                  dz  dej                  dz  dedz  dej                  dz  dej                  dz  dedz  dej                  dz  dej                  dz  dej&                  dz  d	ee   d
ef d                     Zd Zd Ze ed      dej&                  dej&                  d	ee   d
eez  fd              Ze ed      	 d dej                  dej                  dz  d	ee   d
efd              Z xZS )#Gemma4ModelFrY   c                    t         |   |       |j                  t        j                  |j                        nd | _        |j                  j                  | _        t        j                  |j                        }|| _        |j                  j                  | _	        |j                  t        j                  |j                        nd | _        |j                   t        |j                  |j                        nd | _        |j                   t        |j                  |j                        nd | _        | j                          y )Nr  )re   rf   vision_configr.   from_configvision_towerr  r  language_modelr  audio_configaudio_towerr  embed_visionembed_audior  )rm   rY   rA  rn   s      rI   rf   zGemma4Model.__init__  s    KQK_K_KkI11&2F2FGqu ,,77"..f6H6HI,*0*<*<*W*W'IOI\I\Ih9001D1DEnr ##/ %V%9%96;M;MN 	 "". %V%8%8&:L:LM 	
 	rH   zOProjects the last hidden state from the vision model into language model space.r5   Nr\  image_position_idsrF  r\   c                 v     | j                   d||d|}|j                  }| j                  |      |_        |S )z
        image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
            The patch positions as (x, y) coordinates in the image. Padding patches are indicated by (-1, -1).
        r\  rO  r  rG   )r@  r  rD  pooler_output)rm   r\  rF  rF  vision_outputsr  s         rI   get_image_featureszGemma4Model.get_image_features  sV     +** 
%1
 

 +<<'+'8'8GX'8'Y$rH   ry  r  c                 &   |M|| j                   j                  k(  }|| j                   j                  k(  }|| j                   j                  k(  }n>| | j	                         t        j                  | j                   j                  t
        j                  |j                              k(  j                  d      }| | j	                         t        j                  | j                   j                  t
        j                  |j                              k(  j                  d      }| | j	                         t        j                  | j                   j                  t
        j                  |j                              k(  j                  d      }|||fS )a  
        Obtains mask for multimodal placeholders (replaced by soft tokens) and hard text tokens.

        Masks will be obtained from `mm_token_type_ids`, `input_ids`, or `inputs_embeds` as available and in that
        precedence order. If passing `input_ids` or `inputs_embeds`, the image mask will be derived using
        `config.image_token_id`. Same goes for audio and video masks

        Args:
            input_ids: A tensor containing the hard token IDs from the text tokenizer.
            inputs_embeds: A tensor containing the embeddings for all hard text tokens.

        Returns:
            image_mask, video_mask, audio_mask
        )r   r   r   )
rY   image_token_idvideo_token_idaudio_token_idget_input_embeddingsr@   rk   rn  r   ro  )rm   ry  r  special_image_maskspecial_video_maskspecial_audio_masks         rI   get_placeholder_maskz Gemma4Model.get_placeholder_mask  sP   &  !*dkk.H.H!H!*dkk.H.H!H!*dkk.H.H!H .4,,.LL!;!;5::VcVjVjk c"g  .4,,.LL!;!;5::VcVjVjk c"g  .4,,.LL!;!;5::VcVjVjk c"g  "#57IIIrH   pixel_values_videosr  rU   r  r   rN   r0  r  video_position_idsr  c                 &   |du |
duz  rt        d      ||t        d      | j                  ||
      \  }}}||z  |z  }d}|
[|j                         }t        j                  || j
                  j                  j                  |      } | j                         |      }
|| j
                  j                         j                  r| j                  j                  j                  | j
                  j                  j                  ddf   }|j                  |
j                        }t        j                  |d   |j!                  ddd      |
      }| j                  j#                  ||      }|| j%                  ||d      j&                  }|j                  |
j                  |
j(                        }|j+                         }|j-                  d      j/                  |
      j                  |
j                        }t1        |
|   j3                         |j3                         k(  d	| d
|j4                  d           |
j7                  |j                  |
j                        |j                  |
j                              }
|| j9                  ||d      j&                  }|j                  |
j                  |
j(                        }|j+                         }|j-                  d      j/                  |
      j                  |
j                        }t1        |
|   j3                         |j3                         k(  d| d
|j4                  d           |
j7                  |j                  |
j                        |j                  |
j                              }
|+|(| j;                  ||d      }|j&                  }|j<                  }||j                  |j                           }|j+                         }|j-                  d      j/                  |
      j                  |
j                        }t1        |
|   j3                         |j3                         k(  d| d
|j4                  d   |j4                  d   z          |
j7                  |j                  |
j                        |j                  |
j                              }
|V||j?                         nd}t        j@                  |
j4                  d   |
j                        |z   }|j-                  d      }tC        |x} tD              s}| j
                  j                         |
|||d}!| j
                  j                         }"|"jF                  dk(  }#|#r'|	%tI        |	|
j                        }$tK        dd|$i|!} ntM        di |!}  | j                  d|| |||
|dd|}%tO        |%jP                  |%jR                  |%jT                  |%jV                  |nd|nd|%jX                        S )K  
        input_features_mask (`torch.FloatTensor]` of shape `(num_images, seq_length)`):
            The attention mask for the input audio.
        image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
            2D patch position coordinates from the image processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        video_position_ids (`torch.LongTensor` of shape `(num_videos, num_frames, max_patches, 2)`, *optional*):
            2D patch position coordinates from the video processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        per_layer_inputs (`torch.Tensor`, *optional*):
            Pre-computed per-layer input text embeddings of shape `(batch_size, sequence_length, num_hidden_layers,
            hidden_size_per_layer_input)`. When provided, these are used directly instead of being computed from `input_ids`
            via `get_per_layer_inputs()` in the text model. If calling the `forward` with `inputs_embeds` instead of `input_ids`,
            you should probably precompute them and forward them along `inputs_embeds`, otherwise recomputing them needs
            to reverse the main embedding, which is expensive.
        Nr  r  r   r/   r   T)return_dictz6Image features and image tokens do not match, tokens: z, features: r   z6Video features and video tokens do not match, tokens: z6Audio features and audio tokens do not match, tokens: r   r  visionr&  )r  rU   r   rN   r  r  rZ  )r  rN   rO   rP   r9   r:   r;   rG   )-rk  rU  r  r@   rW  rY   r  r  rQ  r  rb  rA  r  r   r   r   r   r  rL  rJ  r   rD  r   	expand_asr(   numelr   masked_scatterget_video_featuresget_audio_featuresrU   r  r   r  rC   r,  r:  r/  r   r8   r  rN   rO   rP   r;   )&rm   ry  r\  rV  r  rU   r  r   rN   r0  r  r  rF  rW  r  rF  
image_mask
video_mask
audio_maskmultimodal_maskllm_input_idspad_embeddingllm_inputs_embedsimage_featuresn_image_tokensvideo_featuresn_video_tokensaudio_outputaudio_featuresaudio_mask_from_encodern_audio_tokensr  r  r  r  	use_bidirr&  r  s&                                         rI   rr   zGemma4Model.forward  s   J -t";<YZZ %5%A[\\-1-F-FyR_-`*
J
$z1J>  %OO-M!KK9P9P9]9]_lmM7D557FM#(C(C(E(a(a //<<CCDKKD[D[DhDhjkDklM-001E1EFO %OI,FHZHZ[\^_acHdfs t#22GGWhi #!44\CUcg4hvvN+..}/C/C]EXEXYN (^^-N#--b1;;MJMMmNbNbcJ"j)//1^5I5I5KKHHX Y"((+,. *88m223^5F5F}G[G[5\M *!44#%7T 5 m  ,..}/C/C]EXEXYN (^^-N#--b1;;MJMMmNbNbcJ"j)//1^5I5I5KKHHX Y"((+,. *88m223^5F5F}G[G[5\M
 %*=*I22>CVdh2iL)77N&2&A&A#
 ,,C,F,F~G\G\,]^N'^^-N#--b1;;MJMMmNbNbcJ"j)//1^5I5I5KKHHX Y"((+n.B.B1.EEFH *88m223^5F5F}G[G[5\M
 CRC^==?de <<(;(;A(>}G[G[\_ooL'11!4L?-F++557!."0#2 ,K ++557K#??8KI.:%DEV_l_s_s%t"&C ''9'!'# '@&N+&N#%$%% 	
-.%+'	
 	
 )%77#33!//))2>2JPT2@2LRV$55
 	
rH   c                 .    | j                   j                  S rp   rA  r  r  s    rI   r  z*Gemma4Model.get_per_layer_input_embeddings	  s    ""999rH   c                 &    || j                   _        y rp   rr  r  s     rI   r  z*Gemma4Model.set_per_layer_input_embeddings	  s    5:2rH   zPProjects the last hidden state from the audio encoder into language model space.c                     | j                   t        d       | j                   ||fddi|}| j                  |j                        |_        |S )a0  
        input_features (`torch.FloatTensor]` of shape `(num_images, seq_length, num_features)`):
            The tensors corresponding to the input audio.
        input_features_mask (`torch.FloatTensor]` of shape `(num_images, seq_length)`):
            The attention mask for the input audio.
        zAudio features were requested, but the model was initialized without an audio_config. Cannot process audio without an audio tower and audio embedder.rZ  TrI  )rC  rk  rE  r  rJ  )rm   r  r  rF  audio_outputss        rI   r`  zGemma4Model.get_audio_features	  sh     #R 
 )((9LiZ^ibhi&*&6&6]EdEd&6&e#rH   zQProjects the last hidden state from the vision encoder into language model space.c                     |j                  dd      }|j                  dd      } | j                  d||d|}|j                  }| j                  |      |_        |S )a9  
        video_position_ids (`torch.LongTensor` of shape `(num_videos, num_frames, max_patches, 2)`, *optional*):
            2D patch position coordinates from the video processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        r   r/   rH  rI  rG   )flattenr@  r  rD  rJ  )rm   rV  rW  rF  rK  r  s         rI   r_  zGemma4Model.get_video_features	  s|     299!Q?/771=*** 
,1
 

 +<<'+'8'8GX'8'Y$rH   rp   r"  )NNNNNNNNNNNNNN)r<   r=   r>   accepts_loss_kwargsr1   rf   r&   r%   r@   rA   r  r"   r$   r   rL  rE   rV   rU  r*   rF   r   r   r8   rr   r  r  rT   r`  r_  rt   ru   s   @rI   r<  r<    s7     | * !rs 7;'' ",,t3 +,	
 
$ t * .226+J##d*+J ((4/+J 
u!1!153C3CC	D	+JZ   .2158<37.23704(,5926!%6:6:04d
##d*d
 ''$.d
 #..5	d

 ))D0d
 t+d
 #\\D0d
 &&-d
 d
 !++d2d
 ((4/d
 $;d
 ",,t3d
 ",,t3d
  ,,-d
  +,!d
" 
##d
    d
L:; !st #\\ +,	
 
'	' u . !tu 7;".. ",,t3 +,	
 
$ v rH   r<  z
    The base Gemma 4 model comprising a vision backbone, an audio backbone, a language model, and a language modeling
    head.
    c            '       f    e Zd ZddiZdZdZdef fdZe	 d!de	j                  d	e	j                  dz  d
ee   fd       Zee	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d"de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                   dz  de	j                   dz  de	j                  dz  d	e	j                  dz  de	j                  dz  dedz  de	j                  dz  de	j                  dz  de	j                  dz  dedz  dee	j                   z  de	j                   dz  d
ee   def$d              Z	 	 	 	 	 	 	 	 	 	 	 	 	 d# fd	Zd Zd Ze	 	 d$dede	j                   de	j                   dz  dedz  de	j                   dz  de	j                   dz  dedz  defd        Z xZS )%Gemma4ForConditionalGenerationr  z(model.language_model.embed_tokens.weightFr}  rY   c                     t         |   |       t        |      | _        t	        j
                  |j                  j                  |j                  j                  d      | _	        | j                          y r  )re   rf   r<  r}  r   rh   r  r   r  r  r  r!  s     rI   rf   z'Gemma4ForConditionalGeneration.__init__	  sS      (
yy!3!3!?!?ASASA^A^ejkrH   Nr\  rF  rF  c                 >     | j                   j                  ||fi |S )a-  
        image_position_ids (`torch.LongTensor` of shape `(batch_size, max_patches, 2)`, *optional*):
            2D patch position coordinates from the image processor, with `(-1, -1)` indicating padding.
            Passed through to the vision encoder for positional embedding computation.
        )r}  rL  )rm   r\  rF  rF  s       rI   rL  z1Gemma4ForConditionalGeneration.get_image_features	  s$     -tzz,,\;MXQWXXrH   ry  rV  r  rU   r  r   rW  rN   r0  r  r  r  r  r  r\   c           
          | j                   di d|d|d|d|d|d|d|d|
d	|d
|d|d|d|d|d|	dd|}|j                  }t        |t              rt	        | d      n|}| j                  |dd|ddf         }| j                  j                         j                  x}||z  }t        j                  |      }||z  }d}|7 | j                  ||| j                  j                         j                  fi |}t        |||j                  |j                  |j                   |j"                  |j$                  |j&                        S )rY  ry  r\  rV  r  rU   r  r   rN   r0  r  r  r  r  rF  rW  rZ  TN)rL   rM   rN   rO   rP   r9   r:   r;   rG   )r}  r  r  rs   r  r  rY   r  r  r@   r   r  r  rK   rN   rO   rP   r9   r:   r;   )rm   ry  r\  rV  r  rU   r  r   rF  rW  rN   r0  r  r  r  r  r  rF  r  rO   r  rM   r  rL   s                           rI   rr   z&Gemma4ForConditionalGeneration.forward	  s   L $** 

%
 !4
 *	

 *
 !4
 &
 ,
 0
 (
 .
 
  
  2
  2
  #
(  118B>SV8W~ot4]kmA}a,?@A'+{{'B'B'D'\'\\#i55FZZ'F55F%4%%ffdkk6Q6Q6S6^6^ibhiD+#33!//)) ' ; ; ' ; ;$55	
 		
rH   c                     t        |   |f|||||||
|d|}|s|s||d<   ||d<   ||d<   |	|d<   nd |d<   |s|j                  dd       }|S )N)rN   r  rU   r   r  r  token_type_idsis_first_iterationr\  rV  r  r  r0  r  )re   prepare_inputs_for_generationr  )rm   ry  rN   r  r   r\  rV  r  rU   r  r  r  r  r  r  rF  model_inputsr   rn   s                     rI   r  z<Gemma4ForConditionalGeneration.prepare_inputs_for_generation6
  s    & w<
+')%))1
 
 Y+7L(2EL./-;L)*2EL./ 15L,- "  !3T:ArH   c                 6    | j                   j                         S rp   )r}  r  r  s    rI   r  z=Gemma4ForConditionalGeneration.get_per_layer_input_embeddingsf
  s    zz88::rH   c                 :    | j                   j                  |       y rp   )r}  r  r  s     rI   r  z=Gemma4ForConditionalGeneration.set_per_layer_input_embeddingsi
  s    

11%8rH   r  c                     | j                         ||||d}| j                         }	t        |	dd       dk(  }
|
r&|$t        ||j                        }t	        dd|i|S t        di |S )Nr  r,  r[  r   r&  rG   )r  r  r:  r   r/  r   )rY   r  rU   rN   r   r0  r  rF  r  r  rp  r&  s               rI   r   z8Gemma4ForConditionalGeneration.create_masks_for_generatel
  s     ,,.*,.(
 ,,.K)FMQYY	*6!@AR[h[o[o!p0 #5 
 )7;77rH   rp   )NNNNNNNNNNNNNNr   N)NNNNNNNNNTNNF)NF)r<   r=   r>   r  rx  r  r1   rf   r%   r@   rA   r  r"   r$   rL  r&   rF   r   r   rs   rK   rr   r  r  r  r  r   rC   r   rt   ru   s   @rI   rz  rz  	  s    +,VW|   7;Y''Y ",,t3Y +,	Y Y  .2158<37.237046:6:(,5926*.!%-.04#N
##d*N
 ''$.N
 #..5	N

 ))D0N
 t+N
 #\\D0N
 &&-N
 ",,t3N
 ",,t3N
 N
 !++d2N
 ((4/N
   4'N
 $;N
  ell*!N
"  ,,-#N
$ +,%N
& 
&'N
  N
f    .`;9  26*/8 8||8 t+8 	8
 llT)8 !<<$.8 !4K8 
8 8rH   rz  )r  r  rz  r<  r|  r  r  )r/   )rU  NN)r-   )r   collectionsr   collections.abcr   dataclassesr   	functoolsr   typingr   r@   r   torch.nnr	   r    r   r  activationsr   cache_utilsr   r   configuration_utilsr   
generationr   integrationsr   masking_utilsr   r   r   r   r   r   r   r   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   r   modeling_rope_utilsr   r   modeling_utilsr    r!   processing_utilsr"   utilsr#   r$   r%   r&   r'   r(   utils.genericr)   r*   utils.output_capturingr+   r,   auto.modeling_autor.   configuration_gemma4r0   r1   r2   r3   accelerate.hooksr4   r8   rK   rR   rT   ModulerX   rw   r   r   r   r
  r  Conv1dr(  r0  r=  rI  rb  r{  r  r  rF   rs   r  r  rl   rE   r  r  r  r  r  r  r  r$  r5  rS  r_  r  rr  r|  r  r  r  r  r  r  rC   r/  r   r:  r<  rz  __all__rG   rH   rI   <module>r     s  *    $ ! %    $ & ! . 3 ) 6	 	 	 C 9 S K F &  H E * g g 3 
 Q 7 Q Q2 
 Q; Q QD 
Q$; 
Q 
Q 
37 3  3BII :4BII 4*7ryy 7>i)299 i)X#bii #8; ;< RYY  H"bii "B&RYY &R0ryy 0l*3		 *3Z@0 @0Fbii  L")) L^(.ELL .u|| .%,, ._b .,	UU\\ 	U# 	U%,, 	U$   %II%<<% 
% <<	%
 LL4'% S[% T\% T\% 5<<%&%N 5&||5&	5& 
5& ,,	5&
 5& \\5&pB)BII B)J)9 )X)H")) )HXBII &W<		 W<tq)")) q)h $#		 $# $#N"@ryy "@JV7 VrSBLL S mJO mJ mJ` `adV+ dV bdVN ]^W
- W
 _W
tsCx X  Sc, SclAH- AHH6ryy 6644<<4 LL4'4 T\	4
 ,,%4 4 
4n	u|| 	U\\ 	^c^j^j 	 y' yyx	 ~8%:O ~8~8BrH   