Ë
    ÛÓ^jØÈ  ã                   óB  — d dl mZ d dlmZ d dlmZ d dlZd dlmZ ddl	m
Z ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZmZmZmZmZmZmZ ddl m!Z!m"Z" ddl#m$Z$m%Z%m&Z&m'Z' ddl(m)Z)m*Z* ddl+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1m2Z2m3Z3m4Z4m5Z5 ddl6m7Z7m8Z8 ddl9m:Z: ddl;m<Z< ddl=m>Z>m?Z?  e3d¬«      e G d„ de$«      «       «       Z@ e3d¬«      e G d„ de1«      «       «       ZA G d „ d!ej„                  «      ZC G d"„ d#ejˆ                  «      ZE G d$„ d%ejˆ                  «      ZF G d&„ d'ejˆ                  «      ZGd(„ ZH ed)«      dZd*„«       ZId+ej”                  d,eKd-ej”                  fd.„ZL	 	 	 d[d/ejˆ                  d0ej”                  d1ej”                  d2ej”                  d3ej”                  dz  d4eMeKz  d5eMdz  d6eMdz  d-eNej”                  ej”                  f   fd7„ZO eeI«       G d8„ d9ejˆ                  «      «       ZP G d:„ d;e"«      ZQe3 G d<„ d=e-«      «       ZRd>eKd-eeKeKeKeKgeSf   fd?„ZTe3 G d@„ dAeR«      «       ZUe3 G dB„ dCeRe«      «       ZV G dD„ dEejˆ                  «      ZWd\dFej”                  dGej°                  dz  d-ej”                  fdH„ZYdIedJej”                  d3ej”                  dz  dKedz  dLej”                  dz  dMej”                  d-eZfdN„Z[ e3dO¬«       G dP„ dQeR«      «       Z\ e3dO¬«       G dR„ dSeRe«      «       Z] e3dT¬«       G dU„ dVe!eR«      «       Z^ G dW„ dXe!eR«      Z_g dY¢Z`y)]é    )ÚCallable)Ú	dataclass)ÚOptionalNé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚPreTrainedConfig)ÚGenerationMixin)Úuse_kernel_func_from_hubÚuse_kernelized_func)Ú_preprocess_mask_argumentsÚblockwise_overlayÚcreate_causal_maskÚcreate_masks_for_generateÚ!create_sliding_window_causal_maskÚmaybe_pad_block_sequence_idsÚsliding_window_overlay)Ú GenericForSequenceClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚBaseModelOutputWithPoolingÚCausalLMOutputWithPastÚ SequenceClassifierOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚtorch_compilable_check)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú	AutoModelé   )ÚGemma3ConfigÚGemma3TextConfigzK
    Base class for Gemma3 outputs, with hidden states and attentions.
    ©Úcustom_introc                   ó:   — e Zd ZU dZdZej                  dz  ed<   y)ÚGemma3ModelOutputWithPasta  
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
    NÚimage_hidden_states)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r2   ÚtorchÚFloatTensorÚ__annotations__© ó    úu/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/gemma3/modeling_gemma3.pyr1   r1   <   s   … ñð 59Ð˜×*Ñ*¨TÑ1Ô8r;   r1   zR
    Base class for Gemma3 causal language model (or autoregressive) outputs.
    c                   óú   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   dZej                  dz  ed<   y)	ÚGemma3CausalLMOutputWithPasta8  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.text_config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
        `past_key_values` input) to speed up sequential decoding.
    image_hidden_states (`torch.FloatTensor`, *optional*):
        A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
        image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
    NÚlossÚlogitsÚpast_key_valuesÚhidden_statesÚ
attentionsr2   )r3   r4   r5   r6   r?   r7   r8   r9   r@   rA   r	   rB   ÚtuplerC   r2   r:   r;   r<   r>   r>   L   s‹   … ñð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø'+€FˆE×Ñ Ñ$Ó+Ø$(€O�U˜T‘\Ó(Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ó6Ø48Ð˜×*Ñ*¨TÑ1Ô8r;   r>   c            	       óZ   ‡ — e Zd ZdZd	dedededefˆ fd„Zdej                  fˆ fd„Z	ˆ xZ
S )
ÚGemma3TextScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scalec                 ó„   •— t         ‰| �  |||«       || _        | j                  dt	        j
                  |«      d¬«       y )NrJ   F©Ú
persistent)ÚsuperÚ__init__Úscalar_embed_scaleÚregister_bufferr7   Útensor)ÚselfrG   rH   rI   rJ   Ú	__class__s        €r<   rO   z&Gemma3TextScaledWordEmbedding.__init__o   s;   ø€ Ü‰Ñ˜¨¸ÔDØ"-ˆÔØ×Ñ˜]¬E¯L©L¸Ó,EÐRWÐÕXr;   Ú	input_idsc                 ó‚   •— t         ‰| �  |«      | j                  j                  | j                  j
                  «      z  S ©N)rN   ÚforwardrJ   ÚtoÚweightÚdtype)rS   rU   rT   s     €r<   rX   z%Gemma3TextScaledWordEmbedding.forwardt   s2   ø€ Ü‰w‰˜yÓ)¨D×,<Ñ,<×,?Ñ,?ÀÇÁ×@QÑ@QÓ,RÑRÐRr;   )ç      ð?)r3   r4   r5   r6   ÚintÚfloatrO   r7   ÚTensorrX   Ú__classcell__©rT   s   @r<   rF   rF   j   sG   ø„ ññY sð Y¸3ð YÈSð YÐ_dõ Yð
S §¡÷ Sñ Sr;   rF   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )Ú	Gemma3MLPÚconfigc                 óÊ  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _	        t        |j                     | _        y ©NF©Úbias)rN   rO   rd   Úhidden_sizeÚintermediate_sizeÚnnÚLinearÚ	gate_projÚup_projÚ	down_projr   Úhidden_activationÚact_fn©rS   rd   rT   s     €r<   rO   zGemma3MLP.__init__y   s¦   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×5Ñ5Ñ6ˆ�r;   c                 óˆ   — | j                  | j                  | j                  |«      «      | j                  |«      z  «      }|S rW   )ro   rq   rm   rn   )rS   Úxro   s      r<   rX   zGemma3MLP.forwardƒ   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÑ#SÓTˆ	ØÐr;   )r3   r4   r5   r-   rO   rX   r`   ra   s   @r<   rc   rc   x   s   ø„ ð7Ð/õ 7ör;   rc   c                   ó<   ‡ — e Zd Zddedefˆ fd„Zd„ Zd„ Zd„ Zˆ xZ	S )ÚGemma3RMSNormÚdimÚepsc                 óŠ   •— t         ‰| �  «        || _        t        j                  t        j                  |«      «      | _        y rW   )rN   rO   rx   rk   Ú	Parameterr7   ÚzerosrZ   )rS   rw   rx   rT   s      €r<   rO   zGemma3RMSNorm.__init__‰   s.   ø€ Ü‰ÑÔØˆŒÜ—l‘l¤5§;¡;¨sÓ#3Ó4ˆ�r;   c                 óŒ   — |t        j                  |j                  d«      j                  dd¬«      | j                  z   «      z  S )Nr)   éÿÿÿÿT)Úkeepdim)r7   ÚrsqrtÚpowÚmeanrx   )rS   rt   s     r<   Ú_normzGemma3RMSNorm._normŽ   s4   € Ø”5—;‘;˜qŸu™u Q›xŸ}™}¨R¸˜}Ó>ÀÇÁÑIÓJÑJÐJr;   c                 ó¢   — | j                  |j                  «       «      }|d| j                  j                  «       z   z  }|j                  |«      S )Nr\   )r‚   r^   rZ   Útype_as)rS   rt   Úoutputs      r<   rX   zGemma3RMSNorm.forward‘   sC   € Ø—‘˜AŸG™G›IÓ&ˆð ˜3 §¡×!2Ñ!2Ó!4Ñ4Ñ5ˆØ�~‰~˜aÓ Ð r;   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)rD   rZ   Úshaperx   )rS   s    r<   Ú
extra_reprzGemma3RMSNorm.extra_repr˜   s'   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$·(±(°Ð<Ð<r;   )g�íµ ÷Æ°>)
r3   r4   r5   r]   r^   rO   r‚   rX   rˆ   r`   ra   s   @r<   rv   rv   ˆ   s&   ø„ ñ5˜Cð 5 eõ 5ò
Kò!ö=r;   rv   c                   óÐ   ‡ — e Zd ZU ej                  ed<   defˆ fd„Ze	 	 	 	 ddedz  de	d   de
dz  dedz  d	ed
ef   f
d„«       Z ej                  «       edd„«       «       Zˆ xZS )ÚGemma3RotaryEmbeddingÚinv_freqrd   c                 ót  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        t        t        |j                  «      «      | _        i | _	        | j                  D ]Ç  }| j
                  j                  |   }|€Œ|d   | j                  |<   | j                  }| j                  |   dk7  rt        | j                  |      } || j
                  |¬«      \  }}| j                  |› d�|d¬«       | j                  |› d�|j                  «       d¬«       t        | |› d�|«       ŒÉ y )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqFrL   Ú_original_inv_freqÚ_attention_scaling)rN   rO   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrd   ÚlistÚsetÚlayer_typesr�   Úrope_parametersÚcompute_default_rope_parametersr   rQ   ÚcloneÚsetattr)rS   rd   r�   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingrT   s          €r<   rO   zGemma3RotaryEmbedding.__init__Ÿ   s6  ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØˆŒÜ¤ F×$6Ñ$6Ó 7Ó8ˆÔØˆŒØ×*Ñ*ò 	UˆJØŸ+™+×5Ñ5°jÑAˆKØÐ"Øà)4°[Ñ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÑ)¨YÒ6Ü2°4·>±>À*Ñ3MÑN�Ù4@ÀÇÁÐYcÔ4dÑ1ˆMÐ1Ø× Ñ  J <¨yÐ!9¸=ÐUZÐ Ô[Ø× Ñ  J <Ð/AÐ!BÀM×DWÑDWÓDYÐfkÐ ÔlÜ�D˜Z˜LÐ(:Ð;Ð=SÕTñ	Ur;   NÚdeviceztorch.deviceÚseq_lenr�   Úreturnztorch.Tensorc                 ó   — | j                   |   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a|  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNr\   r   r)   ©r[   )r¢   r[   )	rš   Úgetattrri   Únum_attention_headsr7   ÚarangeÚint64rY   r^   )rd   r¢   r£   r�   Úbaserw   Úattention_factorr‹   s           r<   r›   z5Gemma3RotaryEmbedding.compute_default_rope_parameters´   s•   € ð2 ×%Ñ% jÑ1°,Ñ?ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r;   c                 óN  — t        | |› d�«      }t        | |› d�«      }|d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬	«      5  |j                  «       |j                  «       z  j                  dd
«      }	t        j                  |	|	fd¬«      }
|
j                  «       |z  }|
j                  «       |z  }d d d «       j	                  |j                  ¬«      j	                  |j                  ¬«      fS # 1 sw Y   ŒAxY w)Nr‘   r“   r   r}   r+   ÚmpsÚcpuF)Údevice_typeÚenabledr)   ©rw   r¨   )r©   r^   Úexpandr‡   rY   r¢   Ú
isinstanceÚtypeÚstrr&   Ú	transposer7   ÚcatÚcosÚsinr[   )rS   rt   Úposition_idsr�   r‹   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedr²   ÚfreqsÚembr»   r¼   s                r<   rX   zGemma3RotaryEmbedding.forwardØ   sl  € ô ˜4 J <¨yÐ!9Ó:ˆÜ# D¨Z¨LÐ8JÐ*KÓLÐà$ Tª1¨d ]Ñ3×9Ñ9Ó;×BÑBÀ<×CUÑCUÐVWÑCXÐZ\Ð^_Ó`×cÑcÐde×dlÑdlÓmÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	0Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)Ð/Ñ/ˆCØ—'‘'“)Ð/Ñ/ˆC÷		0ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	0ð 	0ús   Ã*A1FÆF$©NNNNrW   )r3   r4   r5   r7   r_   r9   r-   rO   Ústaticmethodr   r]   r¸   rD   r^   r›   Úno_gradr   rX   r`   ra   s   @r<   rŠ   rŠ   œ   s«   ø… Ø�l‰lÓðUÐ/õ Uð* à*.Ø+/Ø"Ø!%ñ	!*Ø  4Ñ'ð!*à˜Ñ(ð!*ð �t‘ð!*ð ˜$‘Jð	!*ð
 
ˆ~˜uÐ$Ñ	%ò!*ó ð!*ðF €U‡]�]ƒ_Øò<ó ó ô<r;   rŠ   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Nr}   r)   r´   )r‡   r7   rº   )rt   Úx1Úx2s      r<   Úrotate_halfrÉ   ë   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r;   Úrotary_pos_embc                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezerÉ   )ÚqÚkr»   r¼   Úunsqueeze_dimÚq_embedÚk_embeds          r<   Úapply_rotary_pos_embrÒ   ò   sY   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐr;   rB   Ún_repr¤   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r+   N)r‡   rµ   Úreshape)rB   rÓ   ÚbatchÚnum_key_value_headsÚslenr§   s         r<   Ú	repeat_kvrÙ     so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr;   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚdropoutÚscalingÚsoftcapc                 ó|  — |€| j                   dz  }t        || j                  «      }	t        || j                  «      }
t        j                  ||	j                  dd«      «      |z  }|�||z  }t        j                  |«      }||z  }|�||z   }t        j                  j                  |dt        j                  ¬«      j                  |j                  «      }t        j                  j                  ||| j                  ¬«      }t        j                  ||
«      }|j                  dd«      j                  «       }||fS )Nç      à¿r)   r   r}   )rw   r[   )ÚpÚtrainingr+   )r§   rÙ   Únum_key_value_groupsr7   Úmatmulr¹   Útanhrk   Ú
functionalÚsoftmaxÚfloat32rY   r[   rß   rå   Ú
contiguous)rÚ   rÛ   rÜ   rÝ   rÞ   rß   rà   rá   ÚkwargsÚ
key_statesÚvalue_statesÚattn_weightsÚattn_outputs                r<   Úeager_attention_forwardrò     s  € ð €Ø—/‘/ 4Ñ'ˆä˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LàÐØ# gÑ-ˆÜ—z‘z ,Ó/ˆØ# gÑ-ˆØÐ!Ø# nÑ4ˆô —=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØ˜Ð$Ð$r;   c                   óü   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej                  dej                  dej                  dz  d	e	dz  d
e
e   deej                  ej                  dz  eej                     dz  f   fd„Zˆ xZS )ÚGemma3Attentionz=Multi-headed attention from 'Attention Is All You Need' paperrd   Ú	layer_idxc                 ó   •— t         ‰| �  «        t        |d«      r|j                  |   nd | _        || _        || _        t        |d|j                  |j                  z  «      | _
        |j                  |j                  z  | _        |j                  dz  | _        | j
                  j                  | _        | j
                  j                    | _        t%        j&                  |j                  |j                  | j                  z  |j(                  ¬«      | _        t%        j&                  |j                  |j                  | j                  z  |j(                  ¬«      | _        t%        j&                  |j                  |j                  | j                  z  |j(                  ¬«      | _        t%        j&                  |j                  | j                  z  |j                  |j(                  ¬«      | _        | j
                  j2                  | _        | j                  dk(  r|j4                  nd | _        | j                  dk(  | _        t9        |j                  |j:                  ¬«      | _        t9        |j                  |j:                  ¬«      | _        y )Nr™   r§   rã   rg   Úsliding_attention)rw   rx   ) rN   rO   Úhasattrr™   r�   rd   rõ   r©   ri   rª   r§   r×   ræ   Úquery_pre_attn_scalarrà   Úattention_dropoutÚuse_bidirectional_attentionÚ	is_causalrk   rl   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚattn_logit_softcappingÚsliding_windowÚ
is_slidingrv   Úrms_norm_epsÚq_normÚk_norm©rS   rd   rõ   rT   s      €r<   rO   zGemma3Attention.__init__>  sð  ø€ Ü‰ÑÔÜ;BÀ6È=Ô;Y˜&×,Ñ,¨YÒ7Ð_cˆŒØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø×3Ñ3°TÑ9ˆŒØ!%§¡×!>Ñ!>ˆÔØ!Ÿ[™[×DÑDÐDˆŒä—i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒð '+§k¡k×&HÑ&HˆÔ#Ø7;·±ÐJ]Ò7]˜f×3Ò3ÐcgˆÔØŸ/™/Ð-@Ñ@ˆŒä#¨¯©¸V×=PÑ=PÔQˆŒÜ#¨¯©¸V×=PÑ=PÔQˆ�r;   NrB   Úposition_embeddingsrÞ   rA   rí   r¤   c                 ód  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
| j                  |«      }| j                  |	«      }	|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t         «      } || ||	|
|f| j"                  r| j$                  nd| j&                  | j(                  dœ|¤Ž\  }} |j*                  g |¢d‘­Ž j-                  «       }| j/                  |«      }||fS )Nr}   r+   r)   ç        )rß   rà   r  )r‡   r§   rþ   Úviewr¹   rÿ   r   r  r  rÒ   Úupdaterõ   r   Úget_interfacerd   Ú_attn_implementationrò   rå   rú   rà   r  rÕ   rì   r  )rS   rB   r	  rÞ   rA   rí   Úinput_shapeÚhidden_shapeÚquery_statesrî   rï   r»   r¼   Úattention_interfacerñ   rð   s                   r<   rX   zGemma3Attention.forward\  s¶  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà—{‘{ <Ó0ˆØ—[‘[ Ó,ˆ
à&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð /3¯mªm�D×*Ò*ÀØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r;   )NNN)r3   r4   r5   r6   r-   r]   rO   r7   r_   r	   r    r"   rD   rX   r`   ra   s   @r<   rô   rô   :  sª   ø„ áGðRÐ/ð R¸Cõ RðB -1Ø.2Ø(,ñ*)à—|‘|ð*)ð #Ÿ\™\ð*)ð Ÿ™ tÑ+ð	*)ð
  ™ð*)ð Ð+Ñ,ð*)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷*)r;   rô   c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej                  dej                  dej                  dz  dej                  dz  d	e	dz  d
e
e   deej                  eej                  ej                  f   dz  f   fd„Zˆ xZS )ÚGemma3DecoderLayerrd   rõ   c                 óÔ  •— t         ‰| �  «        || _        |j                  | _        || _        t        ||¬«      | _        t        |«      | _        t        | j                  |j                  ¬«      | _        t        | j                  |j                  ¬«      | _        t        | j                  |j                  ¬«      | _        t        | j                  |j                  ¬«      | _        y )N)rd   rõ   ©rx   )rN   rO   rd   ri   rõ   rô   Ú	self_attnrc   Úmlprv   r  Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormr  s      €r<   rO   zGemma3DecoderLayer.__init__Š  s°   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ"ˆŒÜ(°À)ÔLˆŒÜ˜VÓ$ˆŒÜ,¨T×-=Ñ-=À6×CVÑCVÔWˆÔÜ(5°d×6FÑ6FÈF×L_ÑL_Ô(`ˆÔ%Ü)6°t×7GÑ7GÈV×M`ÑM`Ô)aˆÔ&Ü*7¸×8HÑ8HÈf×NaÑNaÔ*bˆÕ'r;   NrB   r	  rÞ   r½   rA   rí   r¤   c           	      ó  — |}| j                  |«      } | j                  d|||||dœ|¤Ž\  }}| j                  |«      }||z   }|}| j                  |«      }| j	                  |«      }| j                  |«      }||z   }|S )N)rB   r	  rÞ   r½   rA   r:   )r  r  r  r  r  r  )	rS   rB   r	  rÞ   r½   rA   rí   ÚresidualÚ_s	            r<   rX   zGemma3DecoderLayer.forward–  s«   € ð !ˆà×,Ñ,¨]Ó;ˆà)˜4Ÿ>™>ð 
Ø'Ø 3Ø)Ø%Ø+ñ
ð ñ
Ñˆ�qð ×5Ñ5°mÓDˆØ  =Ñ0ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ  =Ñ0ˆàÐr;   rÃ   )r3   r4   r5   r-   r]   rO   r7   r_   Ú
LongTensorr	   r    r"   rD   r8   rX   r`   ra   s   @r<   r  r  ‰  sÃ   ø„ ð
cÐ/ð 
c¸Cõ 
cð -1Ø.2Ø04Ø(,ñà—|‘|ðð #Ÿ\™\ðð Ÿ™ tÑ+ð	ð
 ×&Ñ&¨Ñ-ðð  ™ðð Ð+Ñ,ðð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷r;   r  c                   óŠ   ‡ — e Zd ZU eed<   dZdZg d¢ZdgZdZ	dZ
dZdZdZeedœZdZ ej&                  «       ˆ fd„«       Zˆ xZS )	ÚGemma3PreTrainedModelrd   ÚmodelT)r  ÚSiglipVisionEmbeddingsÚSiglipEncoderLayerÚ#SiglipMultiheadAttentionPoolingHeadrA   )rB   rC   )ÚimageÚtextc                 óÐ  •— t         ‰| �  |«       t        |t        «      r t	        j
                  |j                  «       y d|j                  j                  v r t	        j
                  |j                  «       y t        |t        «      r+t	        j                  |j                  |j                  «       y t        |t        «      r£|j                  D ]“  }|j                   }|j"                  |   dk7  rt$        |j"                  |      } ||j&                  |¬«      \  }}t	        j(                  t+        ||› d�«      |«       t	        j(                  t+        ||› d�«      |«       Œ• y y )NÚRMSNormrŽ   r�   r‘   r’   )rN   Ú_init_weightsr¶   ÚGemma3MultiModalProjectorÚinitÚzeros_Úmm_input_projection_weightrT   r3   rZ   rF   Ú	constant_rJ   rP   rŠ   r™   r›   r�   r   rd   Úcopy_r©   )rS   rÚ   r�   rŸ   r    r   rT   s         €r<   r,  z#Gemma3PreTrainedModel._init_weightsÏ  s  ø€ ä‰Ñ˜fÔ%Ü�fÔ7Ô8Ü�K‰K˜×9Ñ9Õ:à˜&×*Ñ*×3Ñ3Ñ3Ü�K‰K˜Ÿ™Õ&Ü˜Ô =Ô>Ü�N‰N˜6×-Ñ-¨v×/HÑ/HÕIÜ˜Ô 5Ô6Ø$×0Ñ0ò ^�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ò<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ô#UÑ �˜qÜ—
‘
œ7 6¨j¨\¸Ð+CÓDÀmÔTÜ—
‘
œ7 6¨j¨\Ð9KÐ+LÓMÈ}Õ]ñ^ð 7r;   )r3   r4   r5   r,   r9   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr  rô   Ú_can_record_outputsÚinput_modalitiesr7   rÅ   r,  r`   ra   s   @r<   r#  r#  ·  sw   ø… àÓØÐØ&*Ð#òÐð $5Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà+Ø%ñÐð )Ðà€U‡]�]ƒ_ó^ó ô^r;   r#  r  c           
      óP   ‡ — dt         dt         dt         dt         dt        f
ˆ fd„}|S )zA
    Enables a bidirectional mask within the sliding window.
    Ú	batch_idxÚhead_idxÚq_idxÚkv_idxr¤   c                 ó&   •— t        ||z
  «      ‰k  S )z“A token can attend to any other token if their absolute distance is within
        the (exclusive) sliding window size (distance < sliding_window).)Úabs)r?  r@  rA  rB  r  s       €r<   Ú
inner_maskz1_bidirectional_window_overlay.<locals>.inner_maskè  s   ø€ ô �5˜6‘>Ó" ^Ñ3Ð3r;   )r]   Úbool)r  rE  s   ` r<   Ú_bidirectional_window_overlayrG  ã  s3   ø€ ð
4œcð 4¬Sð 4¼ð 4Äcð 4Ìdõ 4ð
 Ðr;   c                   ó  ‡ — e Zd ZU eed<   dZdefˆ fd„Zeee		 	 	 	 	 	 dde
j                  dz  de
j                  dz  de
j                  dz  dedz  d	e
j                  dz  d
edz  dee   defd„«       «       «       Zˆ xZS )ÚGemma3TextModelrd   ©r)  c           	      ó(  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        |j                  |j                  | j                  | j                  j                  dz  ¬«      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                   ¬«      | _        t%        |«      | _        d| _        | j+                  «        y c c}w )Nç      à?)rJ   r  F)rN   rO   Úpad_token_idrI   Ú
vocab_sizerF   ri   rd   Úembed_tokensrk   Ú
ModuleListÚrangeÚnum_hidden_layersr  Úlayersrv   r  ÚnormrŠ   Ú
rotary_embÚgradient_checkpointingÚ	post_initr  s      €r<   rO   zGemma3TextModel.__init__õ  sÝ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒô :Ø×Ñ˜v×1Ñ1°4×3CÑ3CÐQU×Q\ÑQ\×QhÑQhÐjmÑQmô
ˆÔô —m‘mÜDIÈ&×JbÑJbÓDcÖd°yÔ ¨	Õ2Òdó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÔNˆŒ	Ü/°Ó7ˆŒØ&+ˆÔ#ð 	�‰Õùò es   Â"DNrU   rÞ   r½   rA   Úinputs_embedsÚ	use_cacherí   r¤   c           	      óê  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      sw| j                  ||||dœ}
|
j                  «       }| j                  j                  r(d„ |
d<   t        | j                  j                  «      |d<   t!        di |
¤Žt#        di |¤Žd	œ}	|}i }t%        | j                  j&                  «      D ]  }| j)                  |||«      ||<   Œ t+        | j,                  d | j                  j.                   «      D ]G  \  }} ||f|	| j                  j&                  |      || j                  j&                  |      ||d
œ|¤Ž}ŒI | j1                  |«      }t3        ||¬«      S )Nú:You must specify exactly one of input_ids or inputs_embeds©rd   r   r+   ©r¢   ©rd   rX  rÞ   rA   r½   c                  óL   — t        j                  dt         j                  ¬«      S )NTr¨   )r7   rR   rF  )Úargss    r<   ú<lambda>z)Gemma3TextModel.forward.<locals>.<lambda>0  s   € ÄÇÁÈTÔY^×YcÑYcÔ@d€ r;   Úor_mask_function©Úfull_attentionr÷   )rÞ   r	  r½   rA   )Úlast_hidden_staterA   r:   )Ú
ValueErrorrO  r
   rd   Úget_seq_lengthr7   r«   r‡   r¢   rÌ   r¶   ÚdictÚcopyrû   rG  r  r   r   r˜   r™   rU  Ú	enumeraterS  rR  rT  r   )rS   rU   rÞ   r½   rA   rX  rY  rí   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsÚsliding_mask_kwargsrB   r	  r�   ÚiÚdecoder_layers                    r<   rX   zGemma3TextModel.forward  s#  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLô °Ð?Ð-ÄÔFð Ÿ+™+Ø!.Ø"0Ø#2Ø ,ñˆKð #.×"2Ñ"2Ó"4Ðà�{‰{×6Ò6Ù2d�Ð.Ñ/Ü:WÐX\×XcÑXc×XrÑXrÓ:sÐ#Ð$6Ñ7ô #5Ñ"C°{Ñ"CÜ%FÑ%]ÐI\Ñ%]ñ#Ðð &ˆØ ÐÜ˜dŸk™k×5Ñ5Ó6ò 	gˆJØ.2¯o©o¸mÈ\Ð[eÓ.fÐ 
Ò+ð	gô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÓ Wò 	ÑˆAˆ}Ù)Øðà2°4·;±;×3JÑ3JÈ1Ñ3MÑNØ$7¸¿¹×8OÑ8OÐPQÑ8RÑ$SØ)Ø /ñð ñ‰Mð	ð Ÿ	™	 -Ó0ˆä&Ø+Ø+ô
ð 	
r;   )NNNNNN)r3   r4   r5   r-   r9   r=  rO   r'   r(   r#   r7   r!  r_   r	   r8   rF  r    r"   r   rX   r`   ra   s   @r<   rI  rI  ð  sä   ø… àÓØ ÐðÐ/õ ð&  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñC
à×#Ñ# dÑ*ðC
ð Ÿ™ tÑ+ðC
ð ×&Ñ&¨Ñ-ð	C
ð
  ™ðC
ð ×(Ñ(¨4Ñ/ðC
ð ˜$‘;ðC
ð Ð+Ñ,ðC
ð 
!òC
ó ó ó  ôC
r;   rI  c                   óT  ‡ — e Zd ZU ddiZddiZddgdgfiZeed<   defˆ fd„Ze	e
	 	 	 	 	 	 	 	 dd
ej                  d	z  dej                  d	z  dej                  d	z  ded	z  dej                  d	z  dej                  d	z  ded	z  deej                  z  dee   defd„«       «       Zˆ xZS )ÚGemma3ForCausalLMúlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrB   r@   rd   c                 óè   •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  |j                  d¬«      | _        | j                  «        y rf   )
rN   rO   rI  r$  rN  rk   rl   ri   rt  rW  rr   s     €r<   rO   zGemma3ForCausalLM.__init__X  sU   ø€ Ü‰Ñ˜Ô Ü$ VÓ,ˆŒ
Ø ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Õr;   NrU   rÞ   r½   rA   rX  ÚlabelsrY  Úlogits_to_keeprí   r¤   c	           
      ó  —  | j                   d||||||dœ|	¤Ž}
|
j                  }t        |t        «      rt	        | d«      n|}| j                  |dd…|dd…f   «      }| j                  j                  �G|| j                  j                  z  }t        j                  |«      }|| j                  j                  z  }d}|� | j                  ||| j                  fi |	¤Ž}t        |||
j                  |
j                  |
j                  ¬«      S )a‚  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, Gemma3ForCausalLM

        >>> model = Gemma3ForCausalLM.from_pretrained("google/gemma-2-9b")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-9b")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```)rU   rÞ   r½   rA   rX  rY  N)r?   r@   rA   rB   rC   r:   )r$  re  r¶   r]   Úslicert  rd   Úfinal_logit_softcappingr7   rè   Úloss_functionrN  r   rA   rB   rC   )rS   rU   rÞ   r½   rA   rX  rw  rY  rx  rí   ÚoutputsrB   Úslice_indicesr@   r?   s                  r<   rX   zGemma3ForCausalLM.forwarda  s  € ð@ ,6¨4¯:©:ð ,
ØØ)Ø%Ø+Ø'Øñ,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆØ�;‰;×.Ñ.Ð:Ø˜dŸk™k×AÑAÑAˆFÜ—Z‘Z Ó'ˆFØ˜dŸk™k×AÑAÑAˆFàˆØÐØ%�4×%Ñ% f¨f°d·o±oÑPÈÑPˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r;   )NNNNNNNr   )r3   r4   r5   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr-   r9   rO   r$   r#   r7   r!  r_   r	   r8   rF  r]   r    r"   r   rX   r`   ra   s   @r<   rr  rr  Q  s%  ø… à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÓðÐ/õ ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ;
à×#Ñ# dÑ*ð;
ð Ÿ™ tÑ+ð;
ð ×&Ñ&¨Ñ-ð	;
ð
  ™ð;
ð ×(Ñ(¨4Ñ/ð;
ð × Ñ  4Ñ'ð;
ð ˜$‘;ð;
ð ˜eŸl™lÑ*ð;
ð Ð+Ñ,ð;
ð 
 ò;
ó ó ô;
r;   rr  c                   óD   ‡ — e Zd Zdefˆ fd„Zdej                  fd„Zˆ xZS )r-  rd   c                 óŽ  •— t         ‰| �  «        t        j                  t	        j
                  |j                  j                  |j                  j                  «      «      | _	        t        |j                  j                  |j                  j                  ¬«      | _        t        |j                  j                  |j                  j                  z  «      | _        t        |j"                  dz  «      | _        | j                   | j$                  z  | _        t        j(                  | j&                  | j&                  ¬«      | _        y )Nr  rL  )Úkernel_sizeÚstride)rN   rO   rk   rz   r7   r{   Úvision_configri   Útext_configr0  rv   Úlayer_norm_epsÚmm_soft_emb_normr]   Ú
image_sizeÚ
patch_sizeÚpatches_per_imageÚmm_tokens_per_imageÚtokens_per_sider„  Ú	AvgPool2dÚavg_poolrr   s     €r<   rO   z"Gemma3MultiModalProjector.__init__¢  sì   ø€ Ü‰ÑÔä*,¯,©,Ü�K‰K˜×,Ñ,×8Ñ8¸&×:LÑ:L×:XÑ:XÓYó+
ˆÔ'ô !.Ø× Ñ ×,Ñ,°&×2FÑ2F×2UÑ2Uô!
ˆÔô "% V×%9Ñ%9×%DÑ%DÈ×H\ÑH\×HgÑHgÑ%gÓ!hˆÔÜ" 6×#=Ñ#=¸sÑ#BÓCˆÔØ×1Ñ1°T×5IÑ5IÑIˆÔÜŸ™°×1AÑ1AÈ$×JZÑJZÔ[ˆ�r;   Úvision_outputsc                 ó¢  — |j                   \  }}}|j                  dd«      }|j                  ||| j                  | j                  «      }|j	                  «       }| j                  |«      }|j                  d«      }|j                  dd«      }| j                  |«      }t        j                  || j                  «      }|j                  |«      S )Nr+   r)   )r‡   r¹   rÕ   rŒ  rì   r�  Úflattenr‰  r7   rç   r0  r„   )	rS   r‘  Ú
batch_sizer   ri   Úreshaped_vision_outputsÚpooled_vision_outputsÚnormed_vision_outputsÚprojected_vision_outputss	            r<   rX   z!Gemma3MultiModalProjector.forward²  sÍ   € Ø%3×%9Ñ%9Ñ"ˆ
�A�{à"0×":Ñ":¸1¸aÓ"@ÐØ"9×"AÑ"AØ˜ T×%;Ñ%;¸T×=SÑ=Só#
Ðð #:×"DÑ"DÓ"FÐà $§¡Ð.EÓ FÐØ 5× =Ñ =¸aÓ @ÐØ 5× ?Ñ ?ÀÀ1Ó EÐà $× 5Ñ 5Ð6KÓ LÐä#(§<¡<Ð0EÀt×GfÑGfÓ#gÐ Ø'×/Ñ/°Ó?Ð?r;   )	r3   r4   r5   r,   rO   r7   r_   rX   r`   ra   s   @r<   r-  r-  ¡  s#   ø„ ð\˜|õ \ð @ e§l¡l÷ @r;   r-  Útoken_type_idsr¢   c                 ó  — | dk(  j                  |¬«      }t        j                  j                  |dd¬«      d d …d d…f   }|| z  }t	        j
                  |j                  «       d¬«      dz
  }t	        j                  ||d«      }|S )Nr+   r]  )r+   r   r   )rÝ   r}   r´   )rY   rk   ré   Úpadr7   Úcumsumr]   Úwhere)r™  r¢   Úis_imageÚis_previous_imageÚnew_image_startÚ	group_idsÚblock_sequence_idss          r<   Úget_block_sequence_ids_for_maskr£  Å  sŠ   € ð  !Ñ#×'Ñ'¨vÐ'Ó6€HÜŸ™×)Ñ)¨(°FÀ!Ð)ÓDÂQÈÈÈÀVÑLÐØÐ"3Ð!3Ñ3€OÜ—‘˜_×0Ñ0Ó2¸Ô:¸QÑ>€IÜŸ™ X¨y¸"Ó=ÐØÐr;   rd   rX  rA   r½   r¢  c                 óæ   — | ||||dœ}t        di |¤d|i¤Ž}t        di |¤ddi¤Ž\  }}	}	}	}
}	}|r|}nt        |||
|«      }t        di |¤t        |«      t	        | j
                  «      dœ¤Ž}||dœS )zãCreate full_attention and sliding_attention masks with correct composition.

    For global (full attention) layers:  OR(causal, blockwise)
    For local (sliding window) layers:  AND(sliding_window, OR(causal, blockwise))
    r^  r¢  rõ   r   )rb  Úand_mask_functionrc  r:   )r   r   r   r   r   r  )rd   rX  rÞ   rA   r½   r¢  rm  Ú	full_maskÚ
early_exitr   Ú	kv_lengthÚ	kv_offsetÚpadded_block_sequence_idsÚsliding_masks                 r<   Úcreate_masks_for_vision_modelr¬  Ð  s¼   € ð Ø&Ø(Ø*Ø$ñ€Kô #ÑX [ÑXÐEWÒX€Iô 4Nñ 4Ø
ñ4àò4Ñ0€J��1�a˜ A yñ Ø$6Ñ!ä$@Ø °	¸9ó%
Ð!ô &ñ Ø
ðä*Ð+DÓEÜ0°×1FÑ1FÓGó€Lð $Ø)ñð r;   zy
    The Base Gemma3 model which consists of a vision backbone and a language model without language modeling head.,
    c                   óø  ‡ — e Zd ZdZdefˆ fd„Ze ed¬«      dej                  de
e   deez  fd	„«       «       Zd
ej                  dej                  dej                  fd„Zee	 	 	 	 	 	 	 	 	 dd
ej                  dz  dej                  dz  dej"                  dz  dej                  dz  dedz  dej                  dz  dej                  dz  dej                  dz  dedz  de
e   deez  fd„«       «       Zˆ xZS )ÚGemma3ModelFrd   c                 ó2  •— t         ‰| �  |«       t        j                  |j                  ¬«      | _        t        |«      | _        |j                  j                  | _	        t        j                  |j                  ¬«      }|| _
        | j                  «        y )Nr\  )rN   rO   r*   Úfrom_configr†  Úvision_towerr-  Úmulti_modal_projectorr‡  rN  Úlanguage_modelrW  )rS   rd   r³  rT   s      €r<   rO   zGemma3Model.__init__  sq   ø€ Ü‰Ñ˜Ô Ü%×1Ñ1¸×9MÑ9MÔNˆÔÜ%>¸vÓ%FˆÔ"Ø ×,Ñ,×7Ñ7ˆŒä"×.Ñ.°f×6HÑ6HÔIˆØ,ˆÔØ�‰Õr;   zOProjects the last hidden state from the vision model into language model space.r.   Úpixel_valuesrí   r¤   c                 ót   —  | j                   d|ddœ|¤Ž}|j                  }| j                  |«      |_        |S )NT)r´  Úreturn_dictr:   )r±  re  r²  Úpooler_output)rS   r´  rí   r‘  re  s        r<   Úget_image_featureszGemma3Model.get_image_features  sH   € ð
 +˜×*Ñ*Ða¸ÐRVÑaÐZ`ÑaˆØ*×<Ñ<ÐØ'+×'AÑ'AÐBSÓ'TˆÔ$àÐr;   rU   rX  Úimage_featuresc                 ó.  — |€m| | j                  «       t        j                  | j                  j                  t        j
                  |j                  ¬«      «      k(  }|j                  d«      }n|| j                  j                  k(  }|j                  «       }|j                  d   |j                  d   z  }|j                  d«      j                  |j                  «      }t        ||j                  d   z  |j                  «       k(  d|› d|› �«       |S )zï
        Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
        equal to the length of multimodal features. If the lengths are different, an error is raised.
        )r[   r¢   r}   r   r+   z6Image features and image tokens do not match, tokens: z, features: )Úget_input_embeddingsr7   rR   rd   Úimage_token_idÚlongr¢   ÚallÚsumr‡   rÌ   rY   r%   Únumel)rS   rU   rX  r¹  Úspecial_image_maskÚn_image_tokensÚn_image_featuress          r<   Úget_placeholder_maskz Gemma3Model.get_placeholder_mask"  s  € ð ÐØ!.Ð2M°$×2KÑ2KÓ2MÜ—‘˜TŸ[™[×7Ñ7¼u¿z¹zÐR_×RfÑRfÔgó3ñ "Ðð "4×!7Ñ!7¸Ó!;Ñà!*¨d¯k©k×.HÑ.HÑ!HÐà+×/Ñ/Ó1ˆØ)×/Ñ/°Ñ2°^×5IÑ5IÈ!Ñ5LÑLÐØ/×9Ñ9¸"Ó=×@Ñ@À×AUÑAUÓVÐÜØ˜]×0Ñ0°Ñ4Ñ4¸×8LÑ8LÓ8NÑNØDÀ^ÐDTÐT`ÐaqÐ`rÐsô	
ð "Ð!r;   NrÞ   r½   rA   r™  rw  rY  Ú	lm_kwargsc
           
      óX  — |du |duz  rt        d«      ‚|�R| j                  j                  | j                  k\  r/|| j                  j                  k(  }|j	                  «       }d||<   n|}|€ | j                  «       |«      }|�i| j                  |d¬«      j                  }|j                  |j                  |j                  «      }| j                  |||¬«      }|j                  ||«      }t        |x}t        «      sR| j                  j                  «       ||||dœ}|�%t!        ||j                  ¬«      }t#        dd	|i|¤Ž}nt%        di |¤Ž} | j&                  d|||||	dd
œ|
¤Ž}t)        |j*                  |j,                  |j.                  |j0                  |�¬«      S d¬«      S )a®  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

        >>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma32-3b-mix-224")
        >>> processor = AutoProcessor.from_pretrained("google/gemma32-3b-mix-224")

        >>> prompt = "Where is the cat standing?"
        >>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"
        >>> with httpx.stream("GET", url) as response:
        ...     image = Image.open(BytesIO(response.read()))

        >>> inputs = processor(images=image, text=prompt,  return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(**inputs,)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Where is the cat standing?\nsnow"
        ```Nr[  r   T)r¶  )rX  r¹  r^  r]  r¢  )rÞ   r½   rA   rX  rY  r¶  )re  rA   rB   rC   r2   r:   )rf  rd   r¼  rN  rœ   r»  r¸  r·  rY   r¢   r[   rÄ  Úmasked_scatterr¶   rh  Úget_text_configr£  r¬  r   r³  r1   re  rA   rB   rC   )rS   rU   r´  rÞ   r½   rA   r™  rX  rw  rY  rÅ  rÁ  Úllm_input_idsr¹  rl  rm  r¢  r}  s                     r<   rX   zGemma3Model.forward:  sû  € ðX ˜Ð -°tÐ";Ò<ÜÐYÓZÐZð Ð  T§[¡[×%?Ñ%?À4Ç?Á?Ò%RØ!*¨d¯k©k×.HÑ.HÑ!HÐØ%ŸO™OÓ-ˆMØ01ˆMÐ,Ò-à%ˆMàÐ Ø7˜D×5Ñ5Ó7¸ÓFˆMð Ð#Ø!×4Ñ4°\ÈtÐ4ÓT×bÑbˆNØ+×.Ñ.¨}×/CÑ/CÀ]×EXÑEXÓYˆNØ!%×!:Ñ!:Ø¨À~ð ";ó "Ðð *×8Ñ8Ð9KÈ^Ó\ˆMô °Ð?Ð-ÄÔFàŸ+™+×5Ñ5Ó7Ø!.Ø"0Ø#2Ø ,ñˆKð Ð)Ü%DÀ^Ð\i×\pÑ\pÔ%qÐ"Ü&Cñ 'Ø'9ð'à!ñ'Ñ#ô
 '@Ñ&NÀ+Ñ&NÐ#à%�$×%Ñ%ð 
Ø.Ø%Ø+Ø'ØØñ
ð ñ
ˆô )Ø%×7Ñ7Ø#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø2>Ð2J ô
ð 	
ð
 QUô
ð 	
r;   )	NNNNNNNNN)r3   r4   r5   Úaccepts_loss_kwargsr,   rO   r$   r#   r7   r8   r    r"   rD   r   r¸  r!  rÄ  r_   r	   rF  r1   rX   r`   ra   s   @r<   r®  r®    s¤  ø„ ð  Ðð˜|õ ð ÙÐ!rÔsðØ!×-Ñ-ðØ9?Ð@RÑ9Sðà	Ð+Ñ	+òó tó ðð"Ø×)Ñ)ð"Ø:?×:KÑ:Kð"Ø]b×]nÑ]nó"ð0 Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.Ø!%ñd
à×#Ñ# dÑ*ðd
ð ×'Ñ'¨$Ñ.ðd
ð Ÿ™ tÑ+ð	d
ð
 ×&Ñ&¨Ñ-ðd
ð  ™ðd
ð ×(Ñ(¨4Ñ/ðd
ð ×(Ñ(¨4Ñ/ðd
ð × Ñ  4Ñ'ðd
ð ˜$‘;ðd
ð Ð.Ñ/ðd
ð 
Ð*Ñ	*òd
ó ó ôd
r;   r®  c                   ót  ‡ — e Zd ZddiZdZdefˆ fd„Zedej                  de
e   fd„«       Zee	 	 	 	 	 	 	 	 	 	 dd
ej                  d	z  dej                  d	z  dej                  d	z  dej                  d	z  ded	z  dej                  d	z  dej                  d	z  dej                  d	z  ded	z  deej                  z  de
e   deez  fd„«       «       Z	 	 	 	 	 	 	 	 	 	 dˆ fd„	Ze	 	 ddedej                  dej                  d	z  ded	z  dej                  d	z  dej                  d	z  ded	z  defd„«       Zˆ xZS )ÚGemma3ForConditionalGenerationrs  z(model.language_model.embed_tokens.weightFrd   c                 óî   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  j                  |j                  j                  d¬«      | _	        | j                  «        y rf   )rN   rO   r®  r$  rk   rl   r‡  ri   rN  rt  rW  rr   s     €r<   rO   z'Gemma3ForConditionalGeneration.__init__®  sS   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ü—y‘y ×!3Ñ!3×!?Ñ!?À×ASÑAS×A^ÑA^ÐejÔkˆŒØ�‰Õr;   r´  rí   c                 ó<   —  | j                   j                  |fi |¤ŽS rW   )r$  r¸  )rS   r´  rí   s      r<   r¸  z1Gemma3ForConditionalGeneration.get_image_features´  s   € à,ˆt�z‰z×,Ñ,¨\ÑD¸VÑDÐDr;   NrU   rÞ   r½   rA   r™  rX  rw  rY  rx  rÅ  r¤   c                 óÔ  —  | j                   d	||||||||	|ddœ
|¤Ž}|d   }t        |
t        «      rt        |
 d«      n|
}| j	                  |dd…|dd…f   «      }d}|��O|j                  «       }|ddd…dd…f   }|ddd…f   }|�‘|dd…|j                  d    d…f   j                  |j                  «      }||j                  |j                  «      dk7     j                  «       }||j                  |j                  «      dk7     j                  «       }n |j                  «       }|j                  «       }t        j                  «       }|j                  d| j                  j                  j                  «      }|j                  d«      j                  |j                  «      } |||«      }t!        |||j"                  |j$                  |j&                  |j(                  ¬«      S )
a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

        Example:

        ```python
        >>> from PIL import Image
        >>> import httpx
        >>> from io import BytesIO
        >>> from transformers import AutoProcessor, Gemma3ForConditionalGeneration

        >>> model = Gemma3ForConditionalGeneration.from_pretrained("google/gemma-3-4b-it")
        >>> processor = AutoProcessor.from_pretrained("google/gemma-3-4b-it")

        >>> messages = [
        ...     {
        ...         "role": "system",
        ...         "content": [
        ...             {"type": "text", "text": "You are a helpful assistant."}
        ...         ]
        ...     },
        ...     {
        ...         "role": "user", "content": [
        ...             {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"},
        ...             {"type": "text", "text": "Where is the cat standing?"},
        ...         ]
        ...     },
        ... ]

        >>> inputs = processor.apply_chat_template(
        ...     messages,
        ...     tokenize=True,
        ...     return_dict=True,
        ...     return_tensors="pt",
        ...     add_generation_prompt=True
        ... )
        >>> # Generate
        >>> generate_ids = model.generate(**inputs)
        >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "user\nYou are a helpful assistant.\n\n\n\n\n\nWhere is the cat standing?\nmodel\nBased on the image, the cat is standing in a snowy area, likely outdoors. It appears to"
        ```
        T)
rU   r´  r™  rÞ   r½   rA   rX  rY  rw  r¶  r   N.r}   r+   )r?   r@   rA   rB   rC   r2   r:   )r$  r¶   r]   rz  rt  r^   r‡   rY   r¢   rì   rk   ÚCrossEntropyLossr  rd   r‡  rN  r>   rA   rB   rC   r2   )rS   rU   r´  rÞ   r½   rA   r™  rX  rw  rY  rx  rÅ  r}  rB   r~  r@   r?   Úshift_logitsÚshift_labelsÚshift_attention_maskÚloss_fctÚflat_logitsÚflat_labelss                          r<   rX   z&Gemma3ForConditionalGeneration.forward¸  sú  € ðz �$—*‘*ð 
ØØ%Ø)Ø)Ø%Ø+Ø'ØØØñ
ð ñ
ˆð   ™
ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÑà—\‘\“^ˆFØ! # s¨ sªA +Ñ.ˆLØ! # q¡r '™?ˆLØÐ)ð (6²a¸,×:LÑ:LÈQÑ:OÐ9OÑ9QÐ6QÑ'R×'UÑ'UÐV\×VcÑVcÓ'dÐ$Ø+Ð,@×,CÑ,CÀFÇMÁMÓ,RÐVWÑ,WÑX×cÑcÓe�Ø+Ð,@×,CÑ,CÀL×DWÑDWÓ,XÐ\]Ñ,]Ñ^×iÑiÓk‘à+×6Ñ6Ó8�Ø+×6Ñ6Ó8�ä×*Ñ*Ó,ˆHà&×+Ñ+¨B°·±×0GÑ0G×0RÑ0RÓSˆKØ&×+Ñ+¨BÓ/×2Ñ2°<×3FÑ3FÓGˆKÙ˜K¨Ó5ˆDä+ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø '× ;Ñ ;ô
ð 	
r;   c                 óZ   •— t        ‰| �  |f||||||	||dœ|¤Ž}|s|s||d<   |S d |d<   |S )N)rA   rX  rÞ   r½   rY  rx  r™  Úis_first_iterationr´  r™  )rN   Úprepare_inputs_for_generation)rS   rU   rA   rX  r½   r´  rÞ   r™  rY  rx  rw  rØ  rí   Úmodel_inputsrT   s                 €r<   rÙ  z<Gemma3ForConditionalGeneration.prepare_inputs_for_generation'  sj   ø€ ô  ‘wÑ<Øð
à+Ø'Ø)Ø%ØØ)Ø)Ø1ñ
ð ñ
ˆñ" ¡YØ+7ˆL˜Ñ(ð
 Ðð .2ˆLÐ)Ñ*àÐr;   rØ  c                 ó�   — | j                  «       ||||dœ}|�$t        ||j                  ¬«      }	t        dd|	i|¤ŽS t	        di |¤ŽS )Nr^  r]  r¢  r:   )rÈ  r£  r¢   r¬  r   )
rd   rX  rÞ   rA   r½   r™  rØ  rí   rm  r¢  s
             r<   r   z8Gemma3ForConditionalGeneration.create_masks_for_generateP  sl   € ð ×,Ñ,Ó.Ø*Ø,Ø.Ø(ñ
ˆð Ð%Ü!@ÀÐXe×XlÑXlÔ!mÐÜ0ñ Ø#5ðàñð ô
 )Ñ7¨;Ñ7Ð7r;   )
NNNNNNNNNr   )
NNNNNNTNNF)NF)r3   r4   r5   r  rÊ  r,   rO   r#   r7   r8   r    r"   r¸  r$   r!  r_   r	   rF  r]   rD   r>   rX   rÙ  rÄ   r   rh  r   r`   ra   s   @r<   rÌ  rÌ  £  s5  ø„ ð +Ð,VÐWÐð  Ðð˜|õ ð ðE¨u×/@Ñ/@ð EÈFÐSeÑLfò Eó ðEð Øð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.Ø!%Ø-.ñk
à×#Ñ# dÑ*ðk
ð ×'Ñ'¨$Ñ.ðk
ð Ÿ™ tÑ+ð	k
ð
 ×&Ñ&¨Ñ-ðk
ð  ™ðk
ð ×(Ñ(¨4Ñ/ðk
ð ×(Ñ(¨4Ñ/ðk
ð × Ñ  4Ñ'ðk
ð ˜$‘;ðk
ð ˜eŸl™lÑ*ðk
ð Ð.Ñ/ðk
ð 
Ð-Ñ	-òk
ó ó ðk
ð` ØØØØØØØØØ õ'ðR ð /3Ø*/ñ8Ø ð8à—|‘|ð8ð Ÿ™ tÑ+ð8ð  ™ð	8ð
 —l‘l TÑ)ð8ð Ÿ™ tÑ+ð8ð ! 4™Kð8ð 
ò8ó ô8r;   rÌ  zÎ
Gemma3TextForSequenceClassification is a text-only sequence classification model that works with Gemma3TextConfig.
It uses the generic sequence classification implementation for efficiency and consistency.c                   ó   — e Zd ZU eed<   dZy)Ú#Gemma3TextForSequenceClassificationrd   rJ  N)r3   r4   r5   r-   r9   r=  r:   r;   r<   rÝ  rÝ  m  s   … ð ÓØ Ñr;   rÝ  c                   ó  ‡ — e Zd Z	 	 	 	 	 	 	 	 ddej                  dz  dej
                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  dej
                  dz  d	ej                  dz  d
ee	   de
fˆ fd„Zˆ xZS )ÚGemma3ForSequenceClassificationNrU   r´  rÞ   r½   rA   r™  rX  rw  rí   r¤   c	                 ó4   •— t        ‰
| �  d||||||||dœ|	¤ŽS )N)rU   rÞ   r½   rA   rX  r´  r™  rw  r:   )rN   rX   )rS   rU   r´  rÞ   r½   rA   r™  rX  rw  rí   rT   s             €r<   rX   z'Gemma3ForSequenceClassification.forwardx  s=   ø€ ô ‰w‰ð 

ØØ)Ø%Ø+Ø'Ø%Ø)Øñ

ð ñ

ð 
	
r;   )NNNNNNNN)r3   r4   r5   r7   r!  r8   r_   r	   r    r"   r   rX   r`   ra   s   @r<   rß  rß  w  sÞ   ø„ ð .2Ø15Ø.2Ø04Ø(,Ø26Ø26Ø*.ñ
à×#Ñ# dÑ*ð
ð ×'Ñ'¨$Ñ.ð
ð Ÿ™ tÑ+ð	
ð
 ×&Ñ&¨Ñ-ð
ð  ™ð
ð ×(Ñ(¨4Ñ/ð
ð ×(Ñ(¨4Ñ/ð
ð × Ñ  4Ñ'ð
ð Ð+Ñ,ð
ð 
*÷
ñ 
r;   rß  )r#  rI  rr  rÌ  r®  rß  rÝ  )r+   )r  NNrW   )aÚcollections.abcr   Údataclassesr   Útypingr   r7   Útorch.nnrk   Ú r   r.  Úactivationsr   Úcache_utilsr	   r
   Úconfiguration_utilsr   Ú
generationr   Úintegrationsr   r   Úmasking_utilsr   r   r   r   r   r   r   Úmodeling_layersr   r   Úmodeling_outputsr   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr    Úutilsr!   r"   r#   r$   r%   Úutils.genericr&   r'   Úutils.output_capturingr(   Úautor*   Úconfiguration_gemma3r,   r-   r1   r>   Ú	EmbeddingrF   ÚModulerc   rv   rŠ   rÉ   rÒ   r_   r]   rÙ   r^   rD   rò   rô   r  r#  rF  rG  rI  rr  r-  r¢   r£  rh  r¬  r®  rÌ  rÝ  rß  Ú__all__r:   r;   r<   ú<module>rù     së  ðõ* %Ý !Ý ã Ý å &Ý !ß .Ý 3Ý )ß I÷÷ ñ ÷ \÷ó ÷ Lß FÝ &ß nÕ nß GÝ 5Ý ß @ñ ðôð
 ô9Ð 7ó 9ó óð9ñ ðôð
 ô9 ;ó 9ó óð9ô0S B§L¡Lô Sô�—	‘	ô ô =�B—I‘Iô =ô(L<˜BŸI™Iô L<ò^(ñ Ð*Ó+òó ,ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð$ Ø Ø ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �S‰[ð%ð �T‰\ð%ð �T‰\ð%ð ˆ5�<‰<˜Ÿ™Ð%Ñ&ó%ñD Ð)Ó*ôK)�b—i‘ió K)ó +ðK)ô\+Ð3ô +ð\ ô(^˜Oó (^ó ð(^ðV
°#ð 
¸(ÀCÈÈcÐSVÐCWÐY]ÐC]Ñ:^ó 
ð ô]
Ð+ó ]
ó ð]
ð@ ôL
Ð-¨ó L
ó ðL
ô^!@ §	¡	ô !@ñH°E·L±Lð È%Ï,É,ÐY]ÑJ]ð Ðin×iuÑiuó ð1Øð1à—<‘<ð1ð —L‘L 4Ñ'ð1ð ˜T‘\ð	1ð
 —,‘, Ñ%ð1ð Ÿ™ð1ð 
ó1ñh ðôô
W
Ð'ó W
óð
W
ñt ðôô
B8Ð%:¸Oó B8óð
B8ñJ ð^ôô
!Ð*JÐLaó !óð
!ô

Ð&FÐH]ô 
ò4�r;   