
    ^j                        d dl mZ d dlmZmZ d dlZd dlmZ d dlm	Z	 ddl
mZ ddlmZmZ ddlmZ dd	lmZmZmZmZmZmZmZ dd
lmZmZ ddlmZmZm Z  ddl!m"Z"m#Z# ddl$m%Z%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1 ddl2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z; ddl<m=Z=m>Z>m?Z?m@Z@ ddlAmBZB  e-j                  eD      ZE e+d      e	 G d de1e                    ZF e+d      e	 G d de                    ZG G d de@      ZH G d de=      ZI G d  d!ej                        ZK G d" d#e5      ZL G d$ d%e8      ZM G d& d'e9ej                        ZO G d( d)e3      ZP G d* d+e      ZQdZR G d, d-e7      ZSd.eTd/eeTeTeTeTgeUf   fd0ZV G d1 d2e6      ZW G d3 d4e4      ZX G d5 d6ej                        ZYdLd7ej                  d8ej                  dz  d/ej                  fd9Z\d:ed;ej                  d<ej                  dz  d=edz  d>ej                  dz  d?ej                  d/e]fd@Z^ G dA dBe?      Z_ G dC dDe>      Z` e+dEF       G dG dHeeS             Za G dI dJeeS      Zbg dKZcy)M    )Callable)AnyOptionalN)strict   )initialization)CacheDynamicCache)PreTrainedConfig)_preprocess_mask_argumentsblockwise_overlaycreate_causal_maskcreate_masks_for_generate!create_sliding_window_causal_maskmaybe_pad_block_sequence_idssliding_window_overlay) GenericForSequenceClassificationGradientCheckpointingLayer)BaseModelOutputWithPastBaseModelOutputWithPooling SequenceClassifierOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)maybe_autocast   )Gemma2Config)	Gemma2AttentionGemma2ForCausalLM	Gemma2MLPGemma2ModelGemma2PreTrainedModelGemma2RMSNormGemma2RotaryEmbeddingapply_rotary_pos_embeager_attention_forward)PaliGemmaCausalLMOutputWithPast!PaliGemmaForConditionalGenerationPaliGemmaModelPaligemmaModelOutputWithPast)SiglipVisionConfigzgoogle/gemma-3-4b-it)
checkpointc            
           e Zd ZU dZdZdddddddddd	Zddd	Zd
Zee	d<   dZ
ee	d<   dZee   dz  e	d<   dZedz  e	d<   dZedz  e	d<   dZedz  e	d<   dZedz  e	d<   d Zd Zy)Gemma3TextConfiga  
    query_pre_attn_scalar (`float`, *optional*, defaults to 256):
        scaling factor used on the attention scores
    final_logit_softcapping (`float`, *optional*):
        Scaling factor when applying tanh softcapping on the logits.
    attn_logit_softcapping (`float`, *optional*):
        Scaling factor when applying tanh softcapping on the attention scores.
    use_bidirectional_attention (`bool`, *optional*, defaults to `False`):
        If True, the model will attend to all text tokens instead of using a causal mask. This does not change
        behavior for vision tokens.

    ```python
    >>> from transformers import Gemma3TextModel, Gemma3TextConfig
    >>> # Initializing a Gemma3Text gemma3_text-7b style configuration
    >>> configuration = Gemma3TextConfig()
    >>> # Initializing a model from the gemma3_text-7b style configuration
    >>> model = Gemma3TextModel(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    gemma3_textcolwisereplicated_with_grad_allreducerowwise)	zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.q_normzlayers.*.self_attn.k_normzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projg    .Ag     @)globallocali@  
vocab_sizei   max_position_embeddingsNlayer_typesfinal_logit_softcappingattn_logit_softcappingrope_parametersFuse_bidirectional_attentionc                 N   | j                   r| j                  dz  dz   | _        |j                  dd      | _        | j                  Et        | j                        D cg c]!  }t        |dz   | j                  z        rdnd# c}| _        t        j                  di | y c c}w )Nr"      sliding_window_pattern   sliding_attentionfull_attention )
rA   sliding_windowget_sliding_window_patternr=   rangenum_hidden_layersboolr   __post_init__)selfkwargsis      t/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/gemma3/modular_gemma3.pyrO   zGemma3TextConfig.__post_init__s   s    ++#'#6#6!#;q"@D (.zz2JA'N$# t556  (,QUd6R6R,R'S#Yii D
 	&&00 s   &B"c                 z   |j                  dd       }ddiddid}| j                  | j                  n|| _        || j                  d   j                  |       | j                  j                  d      ddi| j                  d<   | j                  d   j	                  d|j                  d| j
                  d                | j                  j                  d      ddi| j                  d<   | j                  d   j	                  d|j                  d	| j
                  d
                | j                          |S )Nrope_scaling	rope_typedefault)rF   rG   rG   
rope_thetar9   rF   rope_local_base_freqr:   )popr@   updaterJ   
setdefaultdefault_thetastandardize_rope_params)rP   rQ   rU   default_rope_paramss       rS   convert_rope_params_to_dictz,Gemma3TextConfig.convert_rope_params_to_dict   sI   zz.$7
 #.y!9*I6
 8<7K7K7Wt33]p#  !1299,G ##$45=6A95MD  !12-.99&**\43E3Eh3OP	
 ##$78@9Di8PD  !4501<<&**%;T=O=OPW=XY	

 	$$&    )__name__
__module____qualname____doc__
model_typebase_model_tp_planr]   r;   int__annotations__r<   r=   liststrr>   floatr?   r@   dictrA   rN   rO   r`   rH   ra   rS   r4   r4   D   s    , J%.%.%.%E%E%."+ )"+
  +X>MJ#*S*$(KcT!(,0UT\0+/EDL/#'OTD['/441ra   r4   c                        e Zd ZU dZdZddddZeedZdZ	ee
eef   z  dz  ed	<   dZee
eef   z  dz  ed
<   dZedz  ed<   dZedz  ed<   dZedz  ed<   dZedz  ed<   dZedz  ed<   dZedz  ed<    fdZ xZS )Gemma3Configa  
    mm_tokens_per_image (`int`, *optional*, defaults to 256):
        The number of tokens per image embedding.
    boi_token_index (`int`, *optional*, defaults to 255999):
        The begin-of-image token index to wrap the image prompt.
    eoi_token_index (`int`, *optional*, defaults to 256000):
        The end-of-image token index to wrap the image prompt.

    Example:

    ```python
    >>> from transformers import Gemma3ForConditionalGeneration, Gemma3Config, SiglipVisionConfig, Gemma3TextConfig

    >>> # Initializing a Siglip-like vision config
    >>> vision_config = SiglipVisionConfig()

    >>> # Initializing a Gemma3 Text config
    >>> text_config = Gemma3TextConfig()

    >>> # Initializing a Gemma3 gemma-3-4b style configuration
    >>> configuration = Gemma3Config(vision_config, text_config)

    >>> # Initializing a model from the gemma-3-4b style configuration
    >>> model = Gemma3TextConfig(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```gemma3image_token_indexboi_token_indexeoi_token_index)image_token_idboi_token_ideoi_token_id)text_configvision_configNrw   rx      mm_tokens_per_imagei i  i   g{Gz?initializer_rangeTtie_word_embeddingsc                    | j                   %t               | _         t        j                  d       n4t	        | j                   t
              rt        di | j                   | _         t	        | j                  t
              rt        di | j                  | _        n0| j                  $t               | _        t        j                  d       t        | $  di | y )Nz@text_config is None, using default Gemma3TextConfig text config.zFvision_config is None, using default SiglipVisionConfig vision config.rH   )
rw   r4   loggerinfo
isinstancerm   rx   r1   superrO   )rP   rQ   	__class__s     rS   rO   zGemma3Config.__post_init__   s    #/1DKKZ[(($//C$2B2BCDd(($/!3!Id6H6H!ID'!3!5DKK`a''ra   )rb   rc   rd   re   rf   attribute_mapr4   r1   sub_configsrw   rm   rk   r   ri   rx   rz   rh   rr   rs   rq   r{   rl   r|   rN   rO   __classcell__r   s   @rS   ro   ro      s    : J-))M (+K
 =AK!DcN2T9@@DM%S#X6=D&)t)")OS4Z)")OS4Z)$+sTz+&*ut|*'++( (ra   ro   c                       e Zd Zy)Gemma3ModelOutputWithPastNrb   rc   rd   rH   ra   rS   r   r          ra   r   c                       e Zd Zy)Gemma3CausalLMOutputWithPastNr   rH   ra   rS   r   r      r   ra   r   c            	       Z     e Zd ZdZd	dedededef fdZdej                  f fdZ	 xZ
S )
Gemma3TextScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    num_embeddingsembedding_dimpadding_idxembed_scalec                     t         |   |||       || _        | j                  dt	        j
                  |      d       y )Nr   F
persistent)r   __init__scalar_embed_scaleregister_buffertorchtensor)rP   r   r   r   r   r   s        rS   r   z&Gemma3TextScaledWordEmbedding.__init__   s;    D"-]ELL,ERWXra   	input_idsc                     t         |   |      | j                  j                  | j                  j
                        z  S N)r   forwardr   toweightdtype)rP   r   r   s     rS   r   z%Gemma3TextScaledWordEmbedding.forward   s2    wy)D,<,<,?,?@Q@Q,RRRra   )      ?)rb   rc   rd   re   rh   rl   r   r   Tensorr   r   r   s   @rS   r   r      sG    Ys Y3 YS Y_d Y
S S Sra   r   c                   $     e Zd Zdef fdZ xZS )	Gemma3MLPconfigc                 $    t         |   |       y r   r   r   rP   r   r   s     rS   r   zGemma3MLP.__init__   s     ra   )rb   rc   rd   r4   r   r   r   s   @rS   r   r      s    !/ ! !ra   r   c                   *     e Zd Zddedef fdZ xZS )Gemma3RMSNormdimepsc                 (    t         |   ||       y )Nr   r   r   )rP   r   r   r   s      rS   r   zGemma3RMSNorm.__init__   s    Sc*ra   )gư>)rb   rc   rd   rh   rl   r   r   r   s   @rS   r   r      s    +C +e + +ra   r   c                       e Zd ZdefdZe	 	 	 	 ddedz  ded   dedz  dedz  de	d	e
f   f
d
       Z ej                         edd              Zy)Gemma3RotaryEmbeddingr   c                    t         j                  j                  |        |j                  | _        |j                  | _        || _        t        t        |j                              | _	        i | _
        | j                  D ]  }| j                  j                  |   }||d   | j                  |<   | j                  }| j                  |   dk7  rt        | j                  |      } || j                  |      \  }}| j                  | d|d       | j                  | d|j                         d       t!        | | d|        y )	NrV   rW   
layer_type	_inv_freqFr   _original_inv_freq_attention_scaling)nnModuler   r<   max_seq_len_cachedoriginal_max_seq_lenr   rj   setr=   rV   r@   compute_default_rope_parametersr   r   clonesetattr)rP   r   r   rope_paramsrope_init_fncurr_inv_freqcurr_attention_scalings          rS   r   zGemma3RotaryEmbedding.__init__  s<   
		4 "("@"@$*$B$B!F$6$6 78** 	UJ++55jAK")4[)ADNN:&%)%I%IL~~j)Y624>>*3MN4@Yc4d1M1  J<y!9=UZ [  J</A!BMDWDWDYfk lDZL(:;=ST	Ura   Ndeviceztorch.deviceseq_lenr   returnztorch.Tensorc                     | j                   |   d   }t        | dd      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                        j                  |t        j                        |z  z  z  }||fS )	a|  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        rX   head_dimNr   r   r"   r   )r   r   )	r@   getattrhidden_sizenum_attention_headsr   arangeint64r   rl   )r   r   r   r   baser   attention_factorinv_freqs           rS   r   z5Gemma3RotaryEmbedding.compute_default_rope_parameters  s    2 %%j1,?fj$/c63E3EIcIc3c U\\!S!5;;?BB&X]XcXcBdgjjk
 )))ra   c                 N   t        | | d      }t        | | d      }|d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d	      5  |j                         |j                         z  j                  dd
      }	t        j                  |	|	fd      }
|
j                         |z  }|
j                         |z  }d d d        j	                  |j                        j	                  |j                        fS # 1 sw Y   AxY w)Nr   r   r   rC   mpscpuF)device_typeenabledr"   r   r   )r   rl   expandshaper   r   r   typerk   r!   	transposer   catcossinr   )rP   xposition_idsr   r   attention_scalinginv_freq_expandedposition_ids_expandedr   freqsembr   r   s                rS   r   zGemma3RotaryEmbedding.forward>  sl    4J<y!9:#DZL8J*KL$T1d]399;BB<CUCUVWCXZ\^_`ccdedldlm ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfkUC 	0&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')//C'')//C		0 vvAGGv$cff177f&;;;	0 	0s   *A1FF$NNNNr   )rb   rc   rd   r4   r   staticmethodr   rh   rk   tuplerl   r   r   no_gradr   r   rH   ra   rS   r   r     s    U/ U* *.+/"!%	!* 4'!*(!* t!* $J	!*
 
~u$	%!* !*F U]]_<  <ra   r   c                        e Zd Zdedef fdZ	 	 	 ddej                  dej                  dej                  dz  dedz  d	e	e
   d
eej                  ej                  dz  eej                     dz  f   fdZ xZS )Gemma3Attentionr   	layer_idxc                 b   t         |   ||       | j                  dk(  r|j                  nd | _        | j                  dk(  | _        | j
                  j                   | _        t        |j                  |j                        | _        t        |j                  |j                        | _        y )NrF   r   )r   r   r   rI   
is_slidingr   rA   	is_causalr   r   rms_norm_epsq_normk_normrP   r   r   r   s      rS   r   zGemma3Attention.__init__S  s    +7;J]7]f33cg//-@@![[DDD#V=P=PQ#V=P=PQra   Nhidden_statesposition_embeddingsattention_maskpast_key_valuesrQ   r   c                 d   |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }	| j                  |      j                  |      j	                  dd      }
| j                  |      }| j                  |	      }	|\  }}t        ||	||      \  }}	| |j                  |	|
| j                        \  }	}
t        j                  | j                  j                  t               } || ||	|
|f| j"                  r| j$                  nd| j&                  | j(                  d|\  }} |j*                  g |d j-                         }| j/                  |      }||fS )Nr   rC   r"   g        )dropoutscalingrI   )r   r   q_projviewr   k_projv_projr   r   r+   r[   r   r   get_interfacer   _attn_implementationr,   trainingattention_dropoutr   rI   reshape
contiguouso_proj)rP   r   r   r   r   rQ   input_shapehidden_shapequery_states
key_statesvalue_statesr   r   attention_interfaceattn_outputattn_weightss                   rS   r   zGemma3Attention.forward\  s    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST{{<0[[,
&S#7jRUWZ#[ j&'6'='=j,X\XfXf'g$J(?(M(MKK,,.E)
 %8
%
 /3mmD**LL..
%
 
%
!\ *k));;;;FFHkk+.L((ra   )NNN)rb   rc   rd   r4   rh   r   r   r   r	   r   r   r   r   r   r   s   @rS   r   r   R  s    R/ RC R -1.2(,*)||*) #\\*) t+	*)
 *) +,*) 
u||U\\D0%2E2LL	M*)ra   r   c                       e Zd Zdedef fdZ	 	 	 	 ddej                  dej                  dej                  dz  dej                  dz  d	e	dz  d
e
e   deej                  eej                  ej                  f   dz  f   fdZ xZS )Gemma3DecoderLayerr   r   c                    t         |           || _        |j                  | _        || _        t        ||      | _        t        |      | _        t        | j                  |j                        | _        t        | j                  |j                        | _        t        | j                  |j                        | _        t        | j                  |j                        | _        y )N)r   r   r   )r   r   r   r   r   r   	self_attnr   mlpr   r   input_layernormpost_attention_layernormpre_feedforward_layernormpost_feedforward_layernormr   s      rS   r   zGemma3DecoderLayer.__init__  s    !--"()LV$,T-=-=6CVCVW(5d6F6FFL_L_(`%)6t7G7GVM`M`)a&*78H8HfNaNa*b'ra   Nr   r   r   r   r   rQ   r   c           	         |}| j                  |      } | j                  d|||||d|\  }}| j                  |      }||z   }|}| j                  |      }| j	                  |      }| j                  |      }||z   }|S )N)r   r   r   r   r   rH   )r  r  r  r  r  r  )	rP   r   r   r   r   r   rQ   residual_s	            rS   r   zGemma3DecoderLayer.forward  s     !,,];)4>> 
' 3)%+
 
q 55mD =0 66}E/77F =0ra   r   )rb   rc   rd   r4   rh   r   r   r   
LongTensorr	   r   r   r   FloatTensorr   r   r   s   @rS   r  r    s    
c/ 
cC 
c -1.204(,|| #\\ t+	
 &&-  +, 
u  %(9(95;L;L(L"MPT"TT	Ura   r  c                   J    e Zd ZdZdZg dZ ej                         d        Zy)Gemma3PreTrainedModelmodel)imagetext)r  SiglipVisionEmbeddingsSiglipEncoderLayer#SiglipMultiheadAttentionPoolingHeadc                    t        j                  | |       t        |t              r t	        j
                  |j                         y d|j                  j                  v r t	        j
                  |j                         y t        |t              r+t	        j                  |j                  |j                         y t        |t              r|j                  D ]  }|j                   }|j"                  |   dk7  rt$        |j"                  |      } ||j&                  |      \  }}t	        j(                  t+        || d      |       t	        j(                  t+        || d      |        y y )NRMSNormrW   r   r   r   )r   _init_weightsr   Gemma3MultiModalProjectorinitzeros_mm_input_projection_weightr   rb   r   r   	constant_r   r   r   r=   r   rV   r   r   copy_r   )rP   moduler   r   r   r  s         rS   r(  z#Gemma3PreTrainedModel._init_weights  s    %%dF3f78KK99:&**333KK& =>NN6--v/H/HI 56$00 ^
%EE##J/9<#6v7G7G
7S#TL#/*#U q

76j\+CDmT

76j\9K+LM}]^ 7ra   N)	rb   rc   rd   base_model_prefixinput_modalities_no_split_modulesr   r   r(  rH   ra   rS   r  r    s4    ( U]]_^ ^ra   r  rI   r   c           
      P     dt         dt         dt         dt         dt        f
 fd}|S )zA
    Enables a bidirectional mask within the sliding window.
    	batch_idxhead_idxq_idxkv_idxr   c                 &    t        ||z
        k  S )zA token can attend to any other token if their absolute distance is within
        the (exclusive) sliding window size (distance < sliding_window).)abs)r4  r5  r6  r7  rI   s       rS   
inner_maskz1_bidirectional_window_overlay.<locals>.inner_mask  s     56>"^33ra   )rh   rN   )rI   r:  s   ` rS   _bidirectional_window_overlayr;    s3    
4c 4S 4 4c 4d 4
 ra   c                        e Zd ZU eed<   dZdef fdZ	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de
dz  d	ej                  dz  d
edz  dee   defdZ xZS )Gemma3TextModelr   r"  c                     t         |   |       t        |j                  |j                  | j
                  | j                  j                  dz        | _        y )N      ?)r   )r   r   r   r;   r   r   r   embed_tokensr   s     rS   r   zGemma3TextModel.__init__  sM      :v1143C3CQUQ\Q\QhQhjmQm
ra   Nr   r   r   r   inputs_embeds	use_cacherQ   r   c           	         |d u |d uz  rt        d      || j                  |      }|r|t        | j                        }|V||j	                         nd}t        j                  |j                  d   |j                        |z   }|j                  d      }t        |x}	t              sw| j                  ||||d}
|
j                         }| j                  j                  r(d |
d<   t        | j                  j                        |d<   t!        di |
t#        di |d	}	|}i }t%        | j                  j&                        D ]  }| j)                  |||      ||<    t+        | j,                  d | j                  j.                         D ]G  \  }} ||f|	| j                  j&                  |      || j                  j&                  |      ||d
|}I | j1                  |      }t3        ||      S )N:You must specify exactly one of input_ids or inputs_embeds)r   r   rC   r   r   rB  r   r   r   c                  L    t        j                  dt         j                        S )NTr   )r   r   rN   )argss    rS   <lambda>z)Gemma3TextModel.forward.<locals>.<lambda>  s    TY^YcYc@d ra   or_mask_functionrG   rF   )r   r   r   r   )last_hidden_stater   rH   )
ValueErrorrA  r
   r   get_seq_lengthr   r   r   r   	unsqueezer   rm   copyrA   r;  rI   r   r   r   r=   
rotary_emb	enumeratelayersrM   normr   )rP   r   r   r   r   rB  rC  rQ   past_seen_tokenscausal_mask_mappingmask_kwargssliding_mask_kwargsr   r   r   rR   decoder_layers                    rS   r   zGemma3TextModel.forward  s#    -t";<YZZ  --i8M0*$++>OCRC^==?de <<(;(;A(>}G[G[\_ooL'11!4L ?-F ++!."0#2 ,K #."2"2"4{{662d./:WX\XcXcXrXr:s#$67 #5"C{"C%F%]I\%]# & dkk556 	gJ.2oom\[e.f
+	g !*$++6U8U8U*V W 	A})24;;3J3J13MN$78O8OPQ8R$S) / M	 		-0&++
 	
ra   )NNNNNN)rb   rc   rd   r4   ri   r1  r   r   r  r   r	   r  rN   r   r   r   r   r   r   s   @rS   r=  r=    s     
/ 
 .2.204(,26!%C
##d*C
 t+C
 &&-	C

 C
 ((4/C
 $;C
 +,C
 
!C
ra   r=  c                   0     e Zd ZU eed<   def fdZ xZS )Gemma3ForCausalLMr   c                 D    t         |   |       t        |      | _        y r   )r   r   r=  r   r   s     rS   r   zGemma3ForCausalLM.__init__:  s     $V,
ra   )rb   rc   rd   r4   ri   r   r   r   s   @rS   r\  r\  7  s    -/ - -ra   r\  c                   D     e Zd Zdef fdZdej                  fdZ xZS )r)  r   c                    t         |           t        j                  t	        j
                  |j                  j                  |j                  j                              | _	        t        |j                  j                  |j                  j                        | _        t        |j                  j                  |j                  j                  z        | _        t        |j"                  dz        | _        | j                   | j$                  z  | _        t        j(                  | j&                  | j&                        | _        y )Nr  r@  )kernel_sizestride)r   r   r   	Parameterr   zerosrx   r   rw   r,  r   layer_norm_epsmm_soft_emb_normrh   
image_size
patch_sizepatches_per_imagerz   tokens_per_sider`  	AvgPool2davg_poolr   s     rS   r   z"Gemma3MultiModalProjector.__init__@  s    *,,,KK,,88&:L:L:X:XY+
' !.  ,,&2F2F2U2U!
 "%V%9%9%D%DH\H\HgHg%g!h"6#=#=s#BC11T5I5II1A1A$JZJZ[ra   vision_outputsc                    |j                   \  }}}|j                  dd      }|j                  ||| j                  | j                        }|j	                         }| j                  |      }|j                  d      }|j                  dd      }| j                  |      }t        j                  || j                        }|j                  |      S )NrC   r"   )r   r   r  rh  r  rk  flattenre  r   matmulr,  type_as)	rP   rl  
batch_sizer  r   reshaped_vision_outputspooled_vision_outputsnormed_vision_outputsprojected_vision_outputss	            rS   r   z!Gemma3MultiModalProjector.forwardP  s    %3%9%9"
A{"0":":1a"@"9"A"AT%;%;T=S=S#
 #:"D"D"F $.E F 5 = =a @ 5 ? ?1 E $ 5 56K L#(<<0EtGfGf#g '//??ra   )	rb   rc   rd   ro   r   r   r   r   r   r   s   @rS   r)  r)  ?  s#    \| \ @ell @ra   r)  token_type_idsr   c                    | dk(  j                  |      }t        j                  j                  |dd      d d d df   }|| z  }t	        j
                  |j                         d      dz
  }t	        j                  ||d      }|S )NrC   rF  )rC   r   r   )valuer   r   )r   r   
functionalpadr   cumsumrh   where)rv  r   is_imageis_previous_imagenew_image_start	group_idsblock_sequence_idss          rS   get_block_sequence_ids_for_maskr  c  s     !#''v'6H))(F!)DQVL"3!33O_002:Q>IXy"=ra   r   rB  r   r   r   r  c                     | ||||d}t        di |d|i}t        di |ddi\  }}	}	}	}
}	}|r|}nt        |||
|      }t        di |t        |      t	        | j
                        d}||dS )zCreate full_attention and sliding_attention masks with correct composition.

    For global (full attention) layers:  OR(causal, blockwise)
    For local (sliding window) layers:  AND(sliding_window, OR(causal, blockwise))
    rG  r  r   r   )rK  and_mask_functionrL  rH   )r   r   r   r   r   rI   )r   rB  r   r   r   r  rX  	full_mask
early_exitr  	kv_length	kv_offsetpadded_block_sequence_idssliding_masks                 rS   create_masks_for_vision_modelr  n  s     &(*$K #X[XEWXI 4N 4
440J1aAy $6!$@	9%
! & 
*+DE01F1FGL $) ra   c                       e Zd ZdZdef fdZe ed      dej                  de
e   deez  fd	              Zee	 	 	 	 	 	 	 	 	 ddej                  d
z  dej                  d
z  dej                   d
z  dej                  d
z  ded
z  dej                  d
z  dej                  d
z  dej                  d
z  ded
z  de
e   deez  fd              Z xZS )Gemma3ModelFr   c                 (    t         |   |       | `y r   )r   r   text_config_dtyper   s     rS   r   zGemma3Model.__init__  s     "ra   zOProjects the last hidden state from the vision model into language model space.custom_intropixel_valuesrQ   r   c                 t     | j                   d|dd|}|j                  }| j                  |      |_        |S )NT)r  return_dictrH   )vision_towerrM  multi_modal_projectorpooler_output)rP   r  rQ   rl  rM  s        rS   get_image_featureszGemma3Model.get_image_features  sH    
 +**aRVaZ`a*<<'+'A'ABS'T$ra   Nr   r   r   r   rv  rB  labelsrC  	lm_kwargsc
           
      X   |d u |d uz  rt        d      |R| j                  j                  | j                  k\  r/|| j                  j                  k(  }|j	                         }d||<   n|}| | j                         |      }|i| j                  |d      j                  }|j                  |j                  |j                        }| j                  |||      }|j                  ||      }t        |x}t              sR| j                  j                         ||||d}|%t!        ||j                        }t#        dd|i|}nt%        di |} | j&                  d|||||	dd	|
}t)        |j*                  |j,                  |j.                  |j0                  |
      S d 
      S )NrE  r   T)r  )rB  image_featuresrG  rF  r  )r   r   r   rB  rC  r  )rM  r   r   
attentionsimage_hidden_statesrH   )rN  r   rt   r;   r   get_input_embeddingsr  r  r   r   r   get_placeholder_maskmasked_scatterr   rm   get_text_configr  r  r   language_modelr   rM  r   r   r  )rP   r   r  r   r   r   rv  rB  r  rC  r  special_image_maskllm_input_idsr  rW  rX  r  outputss                     rS   r   zGemma3Model.forward  s    -t";<YZZ  T[[%?%?4??%R!*dkk.H.H!H%OO-M01M,-%M 7D557FM #!44\t4TbbN+..}/C/C]EXEXYN!%!:!:~ "; " *889K^\M ?-F++557!."0#2 ,K )%D^\i\p\p%q"&C ''9'!'#
 '@&N+&N#%$%% 
.%+'
 
 )%77#33!//))2>2J
 	

 QU
 	
ra   )	NNNNNNNNN)rb   rc   rd   accepts_loss_kwargsro   r   r   r   r   r  r   r   r   r   r  r  r   r	   rN   r   r   r   r   s   @rS   r  r    ss   #| # !rs!--9?@R9S	+	+ t   .215.204(,2626*.!%G
##d*G
 ''$.G
 t+	G

 &&-G
 G
 ((4/G
 ((4/G
   4'G
 $;G
 ./G
 
*	*G
  G
ra   r  c                       e Zd ZdZee	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  de
dz  dej                  dz  d	ej                  dz  d
ej                  dz  dedz  deej                  z  dee   deez  fd              Z	 	 	 	 	 	 	 	 	 	 d fd	Z	 	 dded	ej                  dej                  dz  de
dz  dej                  dz  dej                  dz  dedz  defdZ xZS )Gemma3ForConditionalGenerationFNr   r  r   r   r   rv  rB  r  rC  logits_to_keepr  r   c                     | j                   d	||||||||	|dd
|}|d   }t        |
t              rt        |
 d      n|
}| j	                  |dd|ddf         }d}|O|j                         }|dddddf   }|dddf   }||dd|j                  d    df   j                  |j                        }||j                  |j                        dk7     j                         }||j                  |j                        dk7     j                         }n |j                         }|j                         }t        j                         }|j                  d| j                  j                  j                        }|j                  d      j                  |j                        } |||      }t!        |||j"                  |j$                  |j&                  |j(                        S )
a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

        >>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma-3-4b-it")
        >>> processor = AutoProcessor.from_pretrained("google/gemma-3-4b-it")

        >>> messages = [
        ...     {
        ...         "role": "system",
        ...         "content": [
        ...             {"type": "text", "text": "You are a helpful assistant."}
        ...         ]
        ...     },
        ...     {
        ...         "role": "user", "content": [
        ...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
        ...             {"type": "text", "text": "Where is the cat standing?"},
        ...         ]
        ...     },
        ... ]

        >>> inputs = processor.apply_chat_template(
        ...     messages,
        ...     tokenize=True,
        ...     return_dict=True,
        ...     return_tensors="pt",
        ...     add_generation_prompt=True
        ... )
        >>> # Generate
        >>> generate_ids = model.generate(**inputs)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
        ```
        T)
r   r  rv  r   r   r   rB  rC  r  r  r   N.r   rC   )losslogitsr   r   r  r  rH   )r   r   rh   slicelm_headrl   r   r   r   r  r   CrossEntropyLossr   r   rw   r;   r   r   r   r  r  )rP   r   r  r   r   r   rv  rB  r  rC  r  r  r  r   slice_indicesr  r  shift_logitsshift_labelsshift_attention_maskloss_fctflat_logitsflat_labelss                          rS   r   z&Gemma3ForConditionalGeneration.forward  s   z $** 
%))%+'
 
  
8B>SV8W~ot4]kmA}a,?@A\\^F!#ssA+.L!#qr'?L) (6a,:L:LQ:O9O9Q6Q'R'U'UV\VcVc'd$+,@,C,CFMM,RVW,WXcce+,@,C,CLDWDW,X\],]^iik+668+668**,H&++B0G0G0R0RSK&++B/22<3F3FGKK5D+#33!//)) ' ; ;
 	
ra   c                 Z    t        |   |f||||||	||d|}|s|s||d<   |S d |d<   |S )N)r   rB  r   r   rC  r  rv  is_first_iterationr  rv  )r   prepare_inputs_for_generation)rP   r   r   rB  r   r  r   rv  rC  r  r  r  rQ   model_inputsr   s                 rS   r  z<Gemma3ForConditionalGeneration.prepare_inputs_for_generationu  sj      w<
+')%))1
 
" Y+7L(
  .2L)*ra   r   r  c                     | j                         ||||d}|$t        ||j                        }	t        dd|	i|S t	        di |S )NrG  rF  r  rH   )r  r  r   r  r   )
r   rB  r   r   r   rv  r  rQ   rX  r  s
             rS   r   z8Gemma3ForConditionalGeneration.create_masks_for_generate  sl     ,,.*,.(
 %!@XeXlXl!m0 #5 
 )7;77ra   )
NNNNNNNNNr   )
NNNNNNTNNF)NF)rb   rc   rd   r  r   r   r   r  r  r   r	   rN   rh   r   r   r   r   r   r  r   rm   r   r   r   s   @rS   r  r    s      .215.204(,2626*.!%-.k
##d*k
 ''$.k
 t+	k

 &&-k
 k
 ((4/k
 ((4/k
   4'k
 $;k
 ell*k
 ./k
 
-	-k
  k
`  '^ /3*/8 8||8 t+8 	8
 llT)8 t+8 !4K8 
8ra   r  z
Gemma3TextForSequenceClassification is a text-only sequence classification model that works with Gemma3TextConfig.
It uses the generic sequence classification implementation for efficiency and consistency.r  c                       e Zd ZU eed<   dZy)#Gemma3TextForSequenceClassificationr   r>  N)rb   rc   rd   r4   ri   r1  rH   ra   rS   r  r    s      ra   r  c                       e Zd Z	 	 	 	 	 	 	 	 ddej                  dz  dej
                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  dej
                  dz  d	ej                  dz  d
ee	   de
f fdZ xZS )Gemma3ForSequenceClassificationNr   r  r   r   r   rv  rB  r  rQ   r   c	                 4    t        
|   d||||||||d|	S )N)r   r   r   r   rB  r  rv  r  rH   )r   r   )rP   r   r  r   r   r   rv  rB  r  rQ   r   s             rS   r   z'Gemma3ForSequenceClassification.forward  s=     w 

)%+'%)

 

 
	
ra   )NNNNNNNN)rb   rc   rd   r   r  r  r   r	   r   r   r   r   r   r   s   @rS   r  r    s     .215.204(,2626*.
##d*
 ''$.
 t+	

 &&-
 
 ((4/
 ((4/
   4'
 +,
 
*
 
ra   r  )	ro   r4   r  r=  r\  r  r  r  r  r   )dcollections.abcr   typingr   r   r   torch.nnr   huggingface_hub.dataclassesr    r   r*  cache_utilsr	   r
   configuration_utilsr   masking_utilsr   r   r   r   r   r   r   modeling_layersr   r   modeling_outputsr   r   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r    utils.genericr!   gemma2.configuration_gemma2r#   gemma2.modeling_gemma2r$   r%   r&   r'   r(   r)   r*   r+   r,   paligemma.modeling_paligemmar-   r.   r/   r0   siglipr1   
get_loggerrb   r~   r4   ro   r   r   	Embeddingr   r   r   r   r   r   r  GEMMA3_START_DOCSTRINGr  rh   rN   r;  r=  r\  r)  r   r   r  rm   r  r  r  r  r  __all__rH   ra   rS   <module>r     s   %     . & . 3   \ u u G & R R + 6
 
 
  ( 
		H	% 12W|%5 W  3Wt 12?(# ?(  3?(D	 < 		#B 	SBLL S!	 !
+M +
J<1299 J<\4)o 4)n+3 +\  ^1 ^<
# 
(CcSVCWY]C]:^ 
O
k O
d-) -!@		 !@HELL %,,Y]J] iniuiu 11<<1 LL4'1 T\	1
 ,,%1 1 
1h\
. \
~v8%F v8r ^
!*JLa !
!

&FH] 
4
ra   