
    ^j                     B   d dl mZ d dlmZ d dlmZ d dlZd dlmZ ddl	m
Z ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZmZmZmZmZmZmZ ddl m!Z!m"Z" ddl#m$Z$m%Z%m&Z&m'Z' ddl(m)Z)m*Z* ddl+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1m2Z2m3Z3m4Z4m5Z5 ddl6m7Z7m8Z8 ddl9m:Z: ddl;m<Z< ddl=m>Z>m?Z?  e3d      e G d de$                    Z@ e3d      e G d de1                    ZA G d  d!ej                        ZC G d" d#ej                        ZE G d$ d%ej                        ZF G d& d'ej                        ZGd( ZH ed)      dZd*       ZId+ej                  d,eKd-ej                  fd.ZL	 	 	 d[d/ej                  d0ej                  d1ej                  d2ej                  d3ej                  dz  d4eMeKz  d5eMdz  d6eMdz  d-eNej                  ej                  f   fd7ZO eeI       G d8 d9ej                               ZP G d: d;e"      ZQe3 G d< d=e-             ZRd>eKd-eeKeKeKeKgeSf   fd?ZTe3 G d@ dAeR             ZUe3 G dB dCeRe             ZV G dD dEej                        ZWd\dFej                  dGej                  dz  d-ej                  fdHZYdIedJej                  d3ej                  dz  dKedz  dLej                  dz  dMej                  d-eZfdNZ[ e3dO       G dP dQeR             Z\ e3dO       G dR dSeRe             Z] e3dT       G dU dVe!eR             Z^ G dW dXe!eR      Z_g dYZ`y)]    )Callable)	dataclass)OptionalN   )initialization)ACT2FN)CacheDynamicCache)PreTrainedConfig)GenerationMixin)use_kernel_func_from_hubuse_kernelized_func)_preprocess_mask_argumentsblockwise_overlaycreate_causal_maskcreate_masks_for_generate!create_sliding_window_causal_maskmaybe_pad_block_sequence_idssliding_window_overlay) GenericForSequenceClassificationGradientCheckpointingLayer)BaseModelOutputWithPastBaseModelOutputWithPoolingCausalLMOutputWithPast SequenceClassifierOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)ModelOutputTransformersKwargsauto_docstringcan_return_tupletorch_compilable_check)maybe_autocastmerge_with_config_defaults)capture_outputs   )	AutoModel   )Gemma3ConfigGemma3TextConfigzK
    Base class for Gemma3 outputs, with hidden states and attentions.
    custom_introc                   :    e Zd ZU dZdZej                  dz  ed<   y)Gemma3ModelOutputWithPasta  
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    Nimage_hidden_states)__name__
__module____qualname____doc__r2   torchFloatTensor__annotations__     u/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/gemma3/modeling_gemma3.pyr1   r1   <   s     59**T18r;   r1   zR
    Base class for Gemma3 causal language model (or autoregressive) outputs.
    c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   dZej                  dz  ed<   y)	Gemma3CausalLMOutputWithPasta8  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.text_config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
    Nlosslogitspast_key_valueshidden_states
attentionsr2   )r3   r4   r5   r6   r?   r7   r8   r9   r@   rA   r	   rB   tuplerC   r2   r:   r;   r<   r>   r>   L   s     &*D%

d
")'+FE$+$(OUT\(59M5**+d2926Je''(4/648**T18r;   r>   c            	       Z     e Zd ZdZd	dedededef fdZdej                  f fdZ	 xZ
S )
Gemma3TextScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    num_embeddingsembedding_dimpadding_idxembed_scalec                     t         |   |||       || _        | j                  dt	        j
                  |      d       y )NrJ   F
persistent)super__init__scalar_embed_scaleregister_bufferr7   tensor)selfrG   rH   rI   rJ   	__class__s        r<   rO   z&Gemma3TextScaledWordEmbedding.__init__o   s;    D"-]ELL,ERWXr;   	input_idsc                     t         |   |      | j                  j                  | j                  j
                        z  S N)rN   forwardrJ   toweightdtype)rS   rU   rT   s     r<   rX   z%Gemma3TextScaledWordEmbedding.forwardt   s2    wy)D,<,<,?,?@Q@Q,RRRr;   )      ?)r3   r4   r5   r6   intfloatrO   r7   TensorrX   __classcell__rT   s   @r<   rF   rF   j   sG    Ys Y3 YS Y_d Y
S S Sr;   rF   c                   *     e Zd Zdef fdZd Z xZS )	Gemma3MLPconfigc                    t         |           || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _        t        j                  | j                  | j                  d      | _	        t        |j                     | _        y NFbias)rN   rO   rd   hidden_sizeintermediate_sizennLinear	gate_projup_proj	down_projr   hidden_activationact_fnrS   rd   rT   s     r<   rO   zGemma3MLP.__init__y   s    !--!'!9!94#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXV556r;   c                     | j                  | j                  | j                  |            | j                  |      z        }|S rW   )ro   rq   rm   rn   )rS   xro   s      r<   rX   zGemma3MLP.forward   s6    NN4;;t~~a/@#ADLLQRO#ST	r;   )r3   r4   r5   r-   rO   rX   r`   ra   s   @r<   rc   rc   x   s    7/ 7r;   rc   c                   <     e Zd Zddedef fdZd Zd Zd Z xZ	S )Gemma3RMSNormdimepsc                     t         |           || _        t        j                  t        j                  |            | _        y rW   )rN   rO   rx   rk   	Parameterr7   zerosrZ   )rS   rw   rx   rT   s      r<   rO   zGemma3RMSNorm.__init__   s.    ll5;;s#34r;   c                     |t        j                  |j                  d      j                  dd      | j                  z         z  S )Nr)   T)keepdim)r7   rsqrtpowmeanrx   )rS   rt   s     r<   _normzGemma3RMSNorm._norm   s4    5;;quuQx}}R}>IJJJr;   c                     | j                  |j                               }|d| j                  j                         z   z  }|j                  |      S )Nr\   )r   r^   rZ   type_as)rS   rt   outputs      r<   rX   zGemma3RMSNorm.forward   sC    AGGI& 3!2!2!445~~a  r;   c                 ^    t        | j                  j                         d| j                   S )Nz, eps=)rD   rZ   shaperx   )rS   s    r<   
extra_reprzGemma3RMSNorm.extra_repr   s'    ))*+6$((<<r;   )gư>)
r3   r4   r5   r]   r^   rO   r   rX   r   r`   ra   s   @r<   rv   rv      s&    5C 5e 5
K!=r;   rv   c                        e Zd ZU ej                  ed<   def fdZe	 	 	 	 ddedz  de	d   de
dz  dedz  d	ed
ef   f
d       Z ej                         edd              Z xZS )Gemma3RotaryEmbeddinginv_freqrd   c                 t   t         |           |j                  | _        |j                  | _        || _        t        t        |j                              | _        i | _	        | j                  D ]  }| j
                  j                  |   }||d   | j                  |<   | j                  }| j                  |   dk7  rt        | j                  |      } || j
                  |      \  }}| j                  | d|d       | j                  | d|j                         d       t        | | d|        y )	N	rope_typedefault
layer_type	_inv_freqFrL   _original_inv_freq_attention_scaling)rN   rO   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenrd   listsetlayer_typesr   rope_parameterscompute_default_rope_parametersr   rQ   clonesetattr)rS   rd   r   rope_paramsrope_init_fncurr_inv_freqcurr_attention_scalingrT   s          r<   rO   zGemma3RotaryEmbedding.__init__   s6   "("@"@$*$B$B!F$6$6 78** 	UJ++55jAK")4[)ADNN:&%)%I%IL~~j)Y624>>*3MN4@Yc4d1M1  J<y!9=UZ [  J</A!BMDWDWDYfk lDZL(:;=ST	Ur;   Ndeviceztorch.deviceseq_lenr   returnztorch.Tensorc                     | j                   |   d   }t        | dd      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                        j                  |t        j                        |z  z  z  }||fS )	a|  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        
rope_thetahead_dimNr\   r   r)   r[   )r   r[   )	r   getattrri   num_attention_headsr7   arangeint64rY   r^   )rd   r   r   r   baserw   attention_factorr   s           r<   r   z5Gemma3RotaryEmbedding.compute_default_rope_parameters   s    2 %%j1,?fj$/c63E3EIcIc3c U\\!S!5;;?BB&X]XcXcBdgjjk
 )))r;   c                 N   t        | | d      }t        | | d      }|d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d	      5  |j                         |j                         z  j                  dd
      }	t        j                  |	|	fd      }
|
j                         |z  }|
j                         |z  }d d d        j	                  |j                        j	                  |j                        fS # 1 sw Y   AxY w)Nr   r   r   r}   r+   mpscpuF)device_typeenabledr)   rw   r   )r   r^   expandr   rY   r   
isinstancetypestrr&   	transposer7   catcossinr[   )rS   rt   position_idsr   r   attention_scalinginv_freq_expandedposition_ids_expandedr   freqsembr   r   s                r<   rX   zGemma3RotaryEmbedding.forward   sl    4J<y!9:#DZL8J*KL$T1d]399;BB<CUCUVWCXZ\^_`ccdedldlm ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfkUC 	0&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')//C'')//C		0 vvAGGv$cff177f&;;;	0 	0s   *A1FF$NNNNrW   )r3   r4   r5   r7   r_   r9   r-   rO   staticmethodr   r]   r   rD   r^   r   no_gradr   rX   r`   ra   s   @r<   r   r      s    llU/ U* *.+/"!%	!* 4'!*(!* t!* $J	!*
 
~u$	%!* !*F U]]_<  <r;   r   c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..Nr}   r)   r   )r   r7   r   )rt   x1x2s      r<   rotate_halfr      sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r;   rotary_pos_embc                     |j                  |      }|j                  |      }| |z  t        |       |z  z   }||z  t        |      |z  z   }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer   )qkr   r   unsqueeze_dimq_embedk_embeds          r<   apply_rotary_pos_embr      sY    & --
&C
--
&C3w;q>C/0G3w;q>C/0GGr;   rB   n_repr   c                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r+   N)r   r   reshape)rB   r   batchnum_key_value_headsslenr   s         r<   	repeat_kvr     so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TTr;   modulequerykeyvalueattention_maskdropoutscalingsoftcapc                 |   || j                   dz  }t        || j                        }	t        || j                        }
t        j                  ||	j                  dd            |z  }|||z  }t        j                  |      }||z  }|||z   }t        j                  j                  |dt        j                        j                  |j                        }t        j                  j                  ||| j                        }t        j                  ||
      }|j                  dd      j                         }||fS )N      r)   r   r}   )rw   r[   )ptrainingr+   )r   r   num_key_value_groupsr7   matmulr   tanhrk   
functionalsoftmaxfloat32rY   r[   r   r   
contiguous)r   r   r   r   r   r   r   r   kwargs
key_statesvalue_statesattn_weightsattn_outputs                r<   eager_attention_forwardr     s    //4'3 ; ;<JUF$?$?@L<<z';';Aq'ABWLL#g-zz,/#g-!#n4 ==((2U]](SVVW\WbWbcL==((6??([L,,|\:K''1-88:K$$r;   c                        e Zd ZdZdedef fdZ	 	 	 ddej                  dej                  dej                  dz  d	e	dz  d
e
e   deej                  ej                  dz  eej                     dz  f   fdZ xZS )Gemma3Attentionz=Multi-headed attention from 'Attention Is All You Need' paperrd   	layer_idxc                     t         |           t        |d      r|j                  |   nd | _        || _        || _        t        |d|j                  |j                  z        | _
        |j                  |j                  z  | _        |j                  dz  | _        | j
                  j                  | _        | j
                  j                    | _        t%        j&                  |j                  |j                  | j                  z  |j(                        | _        t%        j&                  |j                  |j                  | j                  z  |j(                        | _        t%        j&                  |j                  |j                  | j                  z  |j(                        | _        t%        j&                  |j                  | j                  z  |j                  |j(                        | _        | j
                  j2                  | _        | j                  dk(  r|j4                  nd | _        | j                  dk(  | _        t9        |j                  |j:                        | _        t9        |j                  |j:                        | _        y )Nr   r   r   rg   sliding_attention)rw   rx   ) rN   rO   hasattrr   r   rd   r   r   ri   r   r   r   r   query_pre_attn_scalarr   attention_dropoutuse_bidirectional_attention	is_causalrk   rl   attention_biasq_projk_projv_projo_projattn_logit_softcappingsliding_window
is_slidingrv   rms_norm_epsq_normk_normrS   rd   r   rT   s      r<   rO   zGemma3Attention.__init__>  s   ;B6=;Y&,,Y7_c"
F4F4F&JdJd4de$*$>$>&B\B\$\!33T9!%!>!>![[DDDii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii&&68J8JQWQfQf
 '+kk&H&H#7;J]7]f33cg//-@@#V=P=PQ#V=P=PQr;   NrB   position_embeddingsr   rA   r   r   c                 d   |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }	| j                  |      j                  |      j	                  dd      }
| j                  |      }| j                  |	      }	|\  }}t        ||	||      \  }}	| |j                  |	|
| j                        \  }	}
t        j                  | j                  j                  t               } || ||	|
|f| j"                  r| j$                  nd| j&                  | j(                  d|\  }} |j*                  g |d j-                         }| j/                  |      }||fS )Nr}   r+   r)           )r   r   r  )r   r   r   viewr   r   r   r  r  r   updater   r   get_interfacerd   _attn_implementationr   r   r   r   r  r   r   r  )rS   rB   r	  r   rA   r   input_shapehidden_shapequery_statesr   r   r   r   attention_interfacer   r   s                   r<   rX   zGemma3Attention.forward\  s    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST{{<0[[,
&S#7jRUWZ#[ j&'6'='=j,X\XfXf'g$J(?(M(MKK,,.E)
 %8
%
 /3mmD**LL..
%
 
%
!\ *k));;;;FFHkk+.L((r;   )NNN)r3   r4   r5   r6   r-   r]   rO   r7   r_   r	   r    r"   rD   rX   r`   ra   s   @r<   r   r   :  s    GR/ RC RB -1.2(,*)||*) #\\*) t+	*)
 *) +,*) 
u||U\\D0%2E2LL	M*)r;   r   c                       e Zd Zdedef fdZ	 	 	 	 ddej                  dej                  dej                  dz  dej                  dz  d	e	dz  d
e
e   deej                  eej                  ej                  f   dz  f   fdZ xZS )Gemma3DecoderLayerrd   r   c                    t         |           || _        |j                  | _        || _        t        ||      | _        t        |      | _        t        | j                  |j                        | _        t        | j                  |j                        | _        t        | j                  |j                        | _        t        | j                  |j                        | _        y )N)rd   r   rx   )rN   rO   rd   ri   r   r   	self_attnrc   mlprv   r  input_layernormpost_attention_layernormpre_feedforward_layernormpost_feedforward_layernormr  s      r<   rO   zGemma3DecoderLayer.__init__  s    !--"()LV$,T-=-=6CVCVW(5d6F6FFL_L_(`%)6t7G7GVM`M`)a&*78H8HfNaNa*b'r;   NrB   r	  r   r   rA   r   r   c           	         |}| j                  |      } | j                  d|||||d|\  }}| j                  |      }||z   }|}| j                  |      }| j	                  |      }| j                  |      }||z   }|S )N)rB   r	  r   r   rA   r:   )r  r  r  r  r  r  )	rS   rB   r	  r   r   rA   r   residual_s	            r<   rX   zGemma3DecoderLayer.forward  s     !,,];)4>> 
' 3)%+
 
q 55mD =0 66}E/77F =0r;   r   )r3   r4   r5   r-   r]   rO   r7   r_   
LongTensorr	   r    r"   rD   r8   rX   r`   ra   s   @r<   r  r    s    
c/ 
cC 
c -1.204(,|| #\\ t+	
 &&-  +, 
u  %(9(95;L;L(L"MPT"TT	Ur;   r  c                        e Zd ZU eed<   dZdZg dZdgZdZ	dZ
dZdZdZeedZdZ ej&                          fd       Z xZS )	Gemma3PreTrainedModelrd   modelT)r  SiglipVisionEmbeddingsSiglipEncoderLayer#SiglipMultiheadAttentionPoolingHeadrA   )rB   rC   )imagetextc                    t         |   |       t        |t              r t	        j
                  |j                         y d|j                  j                  v r t	        j
                  |j                         y t        |t              r+t	        j                  |j                  |j                         y t        |t              r|j                  D ]  }|j                   }|j"                  |   dk7  rt$        |j"                  |      } ||j&                  |      \  }}t	        j(                  t+        || d      |       t	        j(                  t+        || d      |        y y )NRMSNormr   r   r   r   )rN   _init_weightsr   Gemma3MultiModalProjectorinitzeros_mm_input_projection_weightrT   r3   rZ   rF   	constant_rJ   rP   r   r   r   r   r   rd   copy_r   )rS   r   r   r   r   r   rT   s         r<   r,  z#Gemma3PreTrainedModel._init_weights  s   f%f78KK99:&**333KK& =>NN6--v/H/HI 56$00 ^
%EE##J/9<#6v7G7G
7S#TL#/*#U q

76j\+CDmT

76j\9K+LM}]^ 7r;   )r3   r4   r5   r,   r9   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraph_supports_attention_backendr  r   _can_record_outputsinput_modalitiesr7   r   r,  r`   ra   s   @r<   r#  r#    sw    &*# $5"5N!"&+% )U]]_^ ^r;   r#  r  c           
      P     dt         dt         dt         dt         dt        f
 fd}|S )zA
    Enables a bidirectional mask within the sliding window.
    	batch_idxhead_idxq_idxkv_idxr   c                 &    t        ||z
        k  S )zA token can attend to any other token if their absolute distance is within
        the (exclusive) sliding window size (distance < sliding_window).)abs)r?  r@  rA  rB  r  s       r<   
inner_maskz1_bidirectional_window_overlay.<locals>.inner_mask  s     56>"^33r;   )r]   bool)r  rE  s   ` r<   _bidirectional_window_overlayrG    s3    
4c 4S 4 4c 4d 4
 r;   c                       e Zd ZU eed<   dZdef fdZeee		 	 	 	 	 	 dde
j                  dz  de
j                  dz  de
j                  dz  dedz  d	e
j                  dz  d
edz  dee   defd                     Z xZS )Gemma3TextModelrd   r)  c           	      (   t         |   |       |j                  | _        |j                  | _        t        |j                  |j                  | j                  | j                  j                  dz        | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        |j                  |j                         | _        t%        |      | _        d| _        | j+                          y c c}w )N      ?)rJ   r  F)rN   rO   pad_token_idrI   
vocab_sizerF   ri   rd   embed_tokensrk   
ModuleListrangenum_hidden_layersr  layersrv   r  normr   
rotary_embgradient_checkpointing	post_initr  s      r<   rO   zGemma3TextModel.__init__  s     !.. ++ :v1143C3CQUQ\Q\QhQhjmQm
 mmDI&JbJbDcdy	2d
 "&"4"4&:M:MN	/7&+# 	 es   "DNrU   r   r   rA   inputs_embeds	use_cacher   r   c           	         |d u |d uz  rt        d      || j                  |      }|r|t        | j                        }|V||j	                         nd}t        j                  |j                  d   |j                        |z   }|j                  d      }t        |x}	t              sw| j                  ||||d}
|
j                         }| j                  j                  r(d |
d<   t        | j                  j                        |d<   t!        di |
t#        di |d	}	|}i }t%        | j                  j&                        D ]  }| j)                  |||      ||<    t+        | j,                  d | j                  j.                         D ]G  \  }} ||f|	| j                  j&                  |      || j                  j&                  |      ||d
|}I | j1                  |      }t3        ||      S )N:You must specify exactly one of input_ids or inputs_embedsrd   r   r+   r   rd   rX  r   rA   r   c                  L    t        j                  dt         j                        S )NTr   )r7   rR   rF  )argss    r<   <lambda>z)Gemma3TextModel.forward.<locals>.<lambda>0  s    TY^YcYc@d r;   or_mask_functionfull_attentionr   )r   r	  r   rA   )last_hidden_staterA   r:   )
ValueErrorrO  r
   rd   get_seq_lengthr7   r   r   r   r   r   dictcopyr   rG  r  r   r   r   r   rU  	enumeraterS  rR  rT  r   )rS   rU   r   r   rA   rX  rY  r   past_seen_tokenscausal_mask_mappingmask_kwargssliding_mask_kwargsrB   r	  r   idecoder_layers                    r<   rX   zGemma3TextModel.forward  s#    -t";<YZZ  --i8M0*$++>OCRC^==?de <<(;(;A(>}G[G[\_ooL'11!4L ?-F ++!."0#2 ,K #."2"2"4{{662d./:WX\XcXcXrXr:s#$67 #5"C{"C%F%]I\%]# & dkk556 	gJ.2oom\[e.f
+	g !*$++6U8U8U*V W 	A})24;;3J3J13MN$78O8OPQ8R$S) / M	 		-0&++
 	
r;   )NNNNNN)r3   r4   r5   r-   r9   r=  rO   r'   r(   r#   r7   r!  r_   r	   r8   rF  r    r"   r   rX   r`   ra   s   @r<   rI  rI    s     / &   .2.204(,26!%C
##d*C
 t+C
 &&-	C

 C
 ((4/C
 $;C
 +,C
 
!C
    C
r;   rI  c                   T    e Zd ZU ddiZddiZddgdgfiZeed<   def fdZe	e
	 	 	 	 	 	 	 	 dd
ej                  d	z  dej                  d	z  dej                  d	z  ded	z  dej                  d	z  dej                  d	z  ded	z  deej                  z  dee   defd              Z xZS )Gemma3ForCausalLMlm_head.weightzmodel.embed_tokens.weightlm_headcolwise_gather_outputrB   r@   rd   c                     t         |   |       t        |      | _        |j                  | _        t        j                  |j                  |j                  d      | _        | j                          y rf   )
rN   rO   rI  r$  rN  rk   rl   ri   rt  rW  rr   s     r<   rO   zGemma3ForCausalLM.__init__X  sU     $V,
 ++yy!3!3V5F5FUS 	r;   NrU   r   r   rA   rX  labelsrY  logits_to_keepr   r   c	           
          | j                   d||||||d|	}
|
j                  }t        |t              rt	        | d      n|}| j                  |dd|ddf         }| j                  j                  G|| j                  j                  z  }t        j                  |      }|| j                  j                  z  }d}| | j                  ||| j                  fi |	}t        |||
j                  |
j                  |
j                        S )a  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, Gemma3ForCausalLM

        >>> model = Gemma3ForCausalLM.from_pretrained("google/gemma-2-9b")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-9b")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```)rU   r   r   rA   rX  rY  N)r?   r@   rA   rB   rC   r:   )r$  re  r   r]   slicert  rd   final_logit_softcappingr7   r   loss_functionrN  r   rA   rB   rC   )rS   rU   r   r   rA   rX  rw  rY  rx  r   outputsrB   slice_indicesr@   r?   s                  r<   rX   zGemma3ForCausalLM.forwarda  s   @ ,64:: ,
)%+',
 ,
  118B>SV8W~ot4]kmA}a,?@A;;..:dkkAAAFZZ'FdkkAAAF%4%%ffdooPPD%#33!//))
 	
r;   )NNNNNNNr   )r3   r4   r5   _tied_weights_keys_tp_plan_pp_planr-   r9   rO   r$   r#   r7   r!  r_   r	   r8   rF  r]   r    r"   r   rX   r`   ra   s   @r<   rr  rr  Q  s%   *,GH23H_-z:;H/   .2.204(,26*.!%-.;
##d*;
 t+;
 &&-	;

 ;
 ((4/;
   4';
 $;;
 ell*;
 +,;
 
 ;
  ;
r;   rr  c                   D     e Zd Zdef fdZdej                  fdZ xZS )r-  rd   c                    t         |           t        j                  t	        j
                  |j                  j                  |j                  j                              | _	        t        |j                  j                  |j                  j                        | _        t        |j                  j                  |j                  j                  z        | _        t        |j"                  dz        | _        | j                   | j$                  z  | _        t        j(                  | j&                  | j&                        | _        y )Nr  rL  )kernel_sizestride)rN   rO   rk   rz   r7   r{   vision_configri   text_configr0  rv   layer_norm_epsmm_soft_emb_normr]   
image_size
patch_sizepatches_per_imagemm_tokens_per_imagetokens_per_sider  	AvgPool2davg_poolrr   s     r<   rO   z"Gemma3MultiModalProjector.__init__  s    *,,,KK,,88&:L:L:X:XY+
' !.  ,,&2F2F2U2U!
 "%V%9%9%D%DH\H\HgHg%g!h"6#=#=s#BC11T5I5II1A1A$JZJZ[r;   vision_outputsc                    |j                   \  }}}|j                  dd      }|j                  ||| j                  | j                        }|j	                         }| j                  |      }|j                  d      }|j                  dd      }| j                  |      }t        j                  || j                        }|j                  |      S )Nr+   r)   )r   r   r   r  r   r  flattenr  r7   r   r0  r   )	rS   r  
batch_sizer   ri   reshaped_vision_outputspooled_vision_outputsnormed_vision_outputsprojected_vision_outputss	            r<   rX   z!Gemma3MultiModalProjector.forward  s    %3%9%9"
A{"0":":1a"@"9"A"AT%;%;T=S=S#
 #:"D"D"F $.E F 5 = =a @ 5 ? ?1 E $ 5 56K L#(<<0EtGfGf#g '//??r;   )	r3   r4   r5   r,   rO   r7   r_   rX   r`   ra   s   @r<   r-  r-    s#    \| \ @ell @r;   r-  token_type_idsr   c                    | dk(  j                  |      }t        j                  j                  |dd      d d d df   }|| z  }t	        j
                  |j                         d      dz
  }t	        j                  ||d      }|S )Nr+   r]  )r+   r   r   )r   r}   r   )rY   rk   r   padr7   cumsumr]   where)r  r   is_imageis_previous_imagenew_image_start	group_idsblock_sequence_idss          r<   get_block_sequence_ids_for_maskr    s     !#''v'6H))(F!)DQVL"3!33O_002:Q>IXy"=r;   rd   rX  rA   r   r  c                     | ||||d}t        di |d|i}t        di |ddi\  }}	}	}	}
}	}|r|}nt        |||
|      }t        di |t        |      t	        | j
                        d}||dS )zCreate full_attention and sliding_attention masks with correct composition.

    For global (full attention) layers:  OR(causal, blockwise)
    For local (sliding window) layers:  AND(sliding_window, OR(causal, blockwise))
    r^  r  r   r   )rb  and_mask_functionrc  r:   )r   r   r   r   r   r  )rd   rX  r   rA   r   r  rm  	full_mask
early_exitr   	kv_length	kv_offsetpadded_block_sequence_idssliding_masks                 r<   create_masks_for_vision_modelr    s     &(*$K #X[XEWXI 4N 4
440J1aAy $6!$@	9%
! & 
*+DE01F1FGL $) r;   zy
    The Base Gemma3 model which consists of a vision backbone and a language model without language modeling head.,
    c                       e Zd ZdZdef fdZe ed      dej                  de
e   deez  fd	              Zd
ej                  dej                  dej                  fdZee	 	 	 	 	 	 	 	 	 dd
ej                  dz  dej                  dz  dej"                  dz  dej                  dz  dedz  dej                  dz  dej                  dz  dej                  dz  dedz  de
e   deez  fd              Z xZS )Gemma3ModelFrd   c                 2   t         |   |       t        j                  |j                        | _        t        |      | _        |j                  j                  | _	        t        j                  |j                        }|| _
        | j                          y )Nr\  )rN   rO   r*   from_configr  vision_towerr-  multi_modal_projectorr  rN  language_modelrW  )rS   rd   r  rT   s      r<   rO   zGemma3Model.__init__  sq     %119M9MN%>v%F" ,,77"..f6H6HI,r;   zOProjects the last hidden state from the vision model into language model space.r.   pixel_valuesr   r   c                 t     | j                   d|dd|}|j                  }| j                  |      |_        |S )NT)r  return_dictr:   )r  re  r  pooler_output)rS   r  r   r  re  s        r<   get_image_featureszGemma3Model.get_image_features  sH    
 +**aRVaZ`a*<<'+'A'ABS'T$r;   rU   rX  image_featuresc                 .   |m| | j                         t        j                  | j                  j                  t        j
                  |j                              k(  }|j                  d      }n|| j                  j                  k(  }|j                         }|j                  d   |j                  d   z  }|j                  d      j                  |j                        }t        ||j                  d   z  |j                         k(  d| d|        |S )z
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        )r[   r   r}   r   r+   z6Image features and image tokens do not match, tokens: z, features: )get_input_embeddingsr7   rR   rd   image_token_idlongr   allsumr   r   rY   r%   numel)rS   rU   rX  r  special_image_maskn_image_tokensn_image_featuress          r<   get_placeholder_maskz Gemma3Model.get_placeholder_mask"  s    !.2M$2K2K2MT[[77uzzR_RfRfg3 " "4!7!7!;!*dkk.H.H!H+//1)//2^5I5I!5LL/99"=@@AUAUV]00448L8L8NND^DTT`aq`rs	
 "!r;   Nr   r   rA   r  rw  rY  	lm_kwargsc
           
      X   |du |duz  rt        d      |R| j                  j                  | j                  k\  r/|| j                  j                  k(  }|j	                         }d||<   n|}| | j                         |      }|i| j                  |d      j                  }|j                  |j                  |j                        }| j                  |||      }|j                  ||      }t        |x}t              sR| j                  j                         ||||d}|%t!        ||j                        }t#        dd	|i|}nt%        di |} | j&                  d|||||	dd
|
}t)        |j*                  |j,                  |j.                  |j0                  |      S d      S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

        >>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma32-3b-mix-224")
        >>> processor = AutoProcessor.from_pretrained("google/gemma32-3b-mix-224")

        >>> prompt = "Where is the cat standing?"
        >>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, text=prompt,  return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(**inputs,)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Where is the cat standing?\nsnow"
        ```Nr[  r   T)r  )rX  r  r^  r]  r  )r   r   rA   rX  rY  r  )re  rA   rB   rC   r2   r:   )rf  rd   r  rN  r   r  r  r  rY   r   r[   r  masked_scatterr   rh  get_text_configr  r  r   r  r1   re  rA   rB   rC   )rS   rU   r  r   r   rA   r  rX  rw  rY  r  r  llm_input_idsr  rl  rm  r  r}  s                     r<   rX   zGemma3Model.forward:  s   X -t";<YZZ  T[[%?%?4??%R!*dkk.H.H!H%OO-M01M,-%M 7D557FM #!44\t4TbbN+..}/C/C]EXEXYN!%!:!:~ "; " *889K^\M ?-F++557!."0#2 ,K )%D^\i\p\p%q"&C ''9'!'#
 '@&N+&N#%$%% 
.%+'
 
 )%77#33!//))2>2J
 	

 QU
 	
r;   )	NNNNNNNNN)r3   r4   r5   accepts_loss_kwargsr,   rO   r$   r#   r7   r8   r    r"   rD   r   r  r!  r  r_   r	   rF  r1   rX   r`   ra   s   @r<   r  r    s     |  !rs!--9?@R9S	+	+ t "))":?:K:K"]b]n]n"0  .215.204(,2626*.!%d
##d*d
 ''$.d
 t+	d

 &&-d
 d
 ((4/d
 ((4/d
   4'd
 $;d
 ./d
 
*	*d
  d
r;   r  c                   t    e Zd ZddiZdZdef fdZedej                  de
e   fd       Zee	 	 	 	 	 	 	 	 	 	 dd
ej                  d	z  dej                  d	z  dej                  d	z  dej                  d	z  ded	z  dej                  d	z  dej                  d	z  dej                  d	z  ded	z  deej                  z  de
e   deez  fd              Z	 	 	 	 	 	 	 	 	 	 d fd	Ze	 	 ddedej                  dej                  d	z  ded	z  dej                  d	z  dej                  d	z  ded	z  defd       Z xZS )Gemma3ForConditionalGenerationrs  z(model.language_model.embed_tokens.weightFrd   c                     t         |   |       t        |      | _        t	        j
                  |j                  j                  |j                  j                  d      | _	        | j                          y rf   )rN   rO   r  r$  rk   rl   r  ri   rN  rt  rW  rr   s     r<   rO   z'Gemma3ForConditionalGeneration.__init__  sS      (
yy!3!3!?!?ASASA^A^ejkr;   r  r   c                 <     | j                   j                  |fi |S rW   )r$  r  )rS   r  r   s      r<   r  z1Gemma3ForConditionalGeneration.get_image_features  s    ,tzz,,\DVDDr;   NrU   r   r   rA   r  rX  rw  rY  rx  r  r   c                     | j                   d	||||||||	|dd
|}|d   }t        |
t              rt        |
 d      n|
}| j	                  |dd|ddf         }d}|O|j                         }|dddddf   }|dddf   }||dd|j                  d    df   j                  |j                        }||j                  |j                        dk7     j                         }||j                  |j                        dk7     j                         }n |j                         }|j                         }t        j                         }|j                  d| j                  j                  j                        }|j                  d      j                  |j                        } |||      }t!        |||j"                  |j$                  |j&                  |j(                        S )
a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

        >>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma-3-4b-it")
        >>> processor = AutoProcessor.from_pretrained("google/gemma-3-4b-it")

        >>> messages = [
        ...     {
        ...         "role": "system",
        ...         "content": [
        ...             {"type": "text", "text": "You are a helpful assistant."}
        ...         ]
        ...     },
        ...     {
        ...         "role": "user", "content": [
        ...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
        ...             {"type": "text", "text": "Where is the cat standing?"},
        ...         ]
        ...     },
        ... ]

        >>> inputs = processor.apply_chat_template(
        ...     messages,
        ...     tokenize=True,
        ...     return_dict=True,
        ...     return_tensors="pt",
        ...     add_generation_prompt=True
        ... )
        >>> # Generate
        >>> generate_ids = model.generate(**inputs)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
        ```
        T)
rU   r  r  r   r   rA   rX  rY  rw  r  r   N.r}   r+   )r?   r@   rA   rB   rC   r2   r:   )r$  r   r]   rz  rt  r^   r   rY   r   r   rk   CrossEntropyLossr  rd   r  rN  r>   rA   rB   rC   r2   )rS   rU   r  r   r   rA   r  rX  rw  rY  rx  r  r}  rB   r~  r@   r?   shift_logitsshift_labelsshift_attention_maskloss_fctflat_logitsflat_labelss                          r<   rX   z&Gemma3ForConditionalGeneration.forward  s   z $** 
%))%+'
 
  
8B>SV8W~ot4]kmA}a,?@A\\^F!#ssA+.L!#qr'?L) (6a,:L:LQ:O9O9Q6Q'R'U'UV\VcVc'd$+,@,C,CFMM,RVW,WXcce+,@,C,CLDWDW,X\],]^iik+668+668**,H&++B0G0G0R0RSK&++B/22<3F3FGKK5D+#33!//)) ' ; ;
 	
r;   c                 Z    t        |   |f||||||	||d|}|s|s||d<   |S d |d<   |S )N)rA   rX  r   r   rY  rx  r  is_first_iterationr  r  )rN   prepare_inputs_for_generation)rS   rU   rA   rX  r   r  r   r  rY  rx  rw  r  r   model_inputsrT   s                 r<   r  z<Gemma3ForConditionalGeneration.prepare_inputs_for_generation'  sj      w<
+')%))1
 
" Y+7L(
  .2L)*r;   r  c                     | j                         ||||d}|$t        ||j                        }	t        dd|	i|S t	        di |S )Nr^  r]  r  r:   )r  r  r   r  r   )
rd   rX  r   rA   r   r  r  r   rm  r  s
             r<   r   z8Gemma3ForConditionalGeneration.create_masks_for_generateP  sl     ,,.*,.(
 %!@XeXlXl!m0 #5 
 )7;77r;   )
NNNNNNNNNr   )
NNNNNNTNNF)NF)r3   r4   r5   r  r  r,   rO   r#   r7   r8   r    r"   r  r$   r!  r_   r	   rF  r]   rD   r>   rX   r  r   r   rh  r   r`   ra   s   @r<   r  r    s5    +,VW  |  Eu/@/@ EFSeLf E E  .215.204(,2626*.!%-.k
##d*k
 ''$.k
 t+	k

 &&-k
 k
 ((4/k
 ((4/k
   4'k
 $;k
 ell*k
 ./k
 
-	-k
  k
`  'R  /3*/8 8||8 t+8 	8
 llT)8 t+8 !4K8 
8 8r;   r  z
Gemma3TextForSequenceClassification is a text-only sequence classification model that works with Gemma3TextConfig.
It uses the generic sequence classification implementation for efficiency and consistency.c                       e Zd ZU eed<   dZy)#Gemma3TextForSequenceClassificationrd   rJ  N)r3   r4   r5   r-   r9   r=  r:   r;   r<   r  r  m  s      r;   r  c                       e Zd Z	 	 	 	 	 	 	 	 ddej                  dz  dej
                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  dej
                  dz  d	ej                  dz  d
ee	   de
f fdZ xZS )Gemma3ForSequenceClassificationNrU   r  r   r   rA   r  rX  rw  r   r   c	                 4    t        
|   d||||||||d|	S )N)rU   r   r   rA   rX  r  r  rw  r:   )rN   rX   )rS   rU   r  r   r   rA   r  rX  rw  r   rT   s             r<   rX   z'Gemma3ForSequenceClassification.forwardx  s=     w 

)%+'%)

 

 
	
r;   )NNNNNNNN)r3   r4   r5   r7   r!  r8   r_   r	   r    r"   r   rX   r`   ra   s   @r<   r  r  w  s     .215.204(,2626*.
##d*
 ''$.
 t+	

 &&-
 
 ((4/
 ((4/
   4'
 +,
 
*
 
r;   r  )r#  rI  rr  r  r  r  r  )r+   )r  NNrW   )acollections.abcr   dataclassesr   typingr   r7   torch.nnrk    r   r.  activationsr   cache_utilsr	   r
   configuration_utilsr   
generationr   integrationsr   r   masking_utilsr   r   r   r   r   r   r   modeling_layersr   r   modeling_outputsr   r   r   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr    utilsr!   r"   r#   r$   r%   utils.genericr&   r'   utils.output_capturingr(   autor*   configuration_gemma3r,   r-   r1   r>   	EmbeddingrF   Modulerc   rv   r   r   r   r_   r]   r   r^   rD   r   r   r  r#  rF  rG  rI  rr  r-  r   r  rh  r  r  r  r  r  __all__r:   r;   r<   <module>r     s  * % !    & ! . 3 ) I   \  L F & n n G 5  @ 
 9 7 9 9 
 9; 9 90SBLL S		  =BII =(L<BII L<^( *+ ,2	UU\\ 	U# 	U%,, 	U$   %II%<<% 
% <<	%
 LL4'% S[% T\% T\% 5<<%&%D )*K)bii K) +K)\+3 +\ (^O (^ (^V
# 
(CcSVCWY]C]:^ 
 ]
+ ]
 ]
@ L
- L
 L
^!@		 !@HELL %,,Y]J] iniuiu 11<<1 LL4'1 T\	1
 ,,%1 1 
1h 
W
' W

W
t 
B8%:O B8
B8J ^
!*JLa !
!

&FH] 
4r;   