Ë
    ÛÓ^jëe  ã                   ó–  — d dl mZ d dlmZ d dlZd dlmZ d dlmZmZm	Z	 ddl
mZ ddlmZ dd	lmZmZmZ dd
lmZ ddlmZ ddlmZmZmZmZ ddlmZmZ ddlmZm Z  ddl!m"Z" ddl#m$Z$ ddl%m&Z&m'Z'm(Z(m)Z) ddl*m+Z+ ddl,m-Z-  ed«       G d„ dej\                  «      «       Z/d„ Z0 ed«      d;d„«       Z1dejd                  de3dejd                  fd„Z4	 d<dej\                  d ejd                  d!ejd                  d"ejd                  d#ejd                  dz  d$e5d%e5d&e"e&   fd'„Z6 ee1«       G d(„ d)ej\                  «      «       Z7 G d*„ d+ej\                  «      Z8 G d,„ d-e«      Z9e$ G d.„ d/e «      «       Z: G d0„ d1ej\                  «      Z;e$ G d2„ d3e:«      «       Z<e$ G d4„ d5e:«      «       Z=e$ G d6„ d7e:«      «       Z>e$ G d8„ d9e:«      «       Z?g d:¢Z@y)=é    )ÚCallable)ÚOptionalN)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )ÚACT2FN)ÚCache)Úuse_kernel_forward_from_hubÚuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚMaskedLMOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úauto_docstring)ÚTransformersKwargsÚcan_return_tupleÚmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚEuroBertConfigÚRMSNormc                   ó`   ‡ — e Zd Zddˆ fd„Zdej
                  dej
                  fd„Zd„ Zˆ xZS )ÚEuroBertRMSNormÚreturnc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)z>
        EuroBertRMSNorm is equivalent to T5LayerNorm
        N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s      €úy/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/eurobert/modeling_eurobert.pyr(   zEuroBertRMSNorm.__init__.   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕó    Úhidden_statesc                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S )Né   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtor*   Úfloat32ÚpowÚmeanÚrsqrtr-   r,   )r.   r4   Úinput_dtypeÚvariances       r2   ÚforwardzEuroBertRMSNorm.forward6   sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r3   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)Útupler,   Úshaper-   ©r.   s    r2   Ú
extra_reprzEuroBertRMSNorm.extra_repr=   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr3   )gñhãˆµøä>)r%   N)	Ú__name__Ú
__module__Ú__qualname__r(   r*   ÚTensorrA   rF   Ú__classcell__©r1   s   @r2   r$   r$   ,   s)   ø„ ö$ð; U§\¡\ð ;°e·l±ló ;öJr3   r$   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Nr7   r6   ©Údim)rD   r*   Úcat)ÚxÚx1Úx2s      r2   Úrotate_halfrT   A   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r3   Úrotary_pos_embc                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezerT   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚq_embedÚk_embeds          r2   Úapply_rotary_pos_embr_   H   sY   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐr3   r4   Ún_repr%   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r    N)rD   ÚexpandÚreshape)r4   r`   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         r2   Ú	repeat_kvrh   b   so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr3   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t        || j                  «      }t        || j                  «      }	t        j                  ||j	                  dd«      «      |z  }
|�|
|z   }
t
        j                  j                  |
dt        j                  ¬«      j                  |j                  «      }
t
        j                  j                  |
|| j                  ¬«      }
t        j                  |
|	«      }|j	                  dd«      j                  «       }||
fS )Nr6   r	   r7   )rO   r9   )ÚpÚtrainingr    )rh   Únum_key_value_groupsr*   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr;   r:   r9   ro   rs   Ú
contiguous)ri   rj   rk   rl   rm   rn   ro   rp   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r2   Úeager_attention_forwardr~   n   sà   € ô ˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r3   c                   óø   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej                  de	ej                  ej                  f   dz  dej                  dz  d	e
dz  d
ee   de	ej                  ej                  f   fd„Zˆ xZS )ÚEuroBertAttentionz=Multi-headed attention from 'Attention Is All You Need' paperÚconfigÚ	layer_idxc                 ód  •— t         ‰| �  «        || _        || _        t	        |d|j
                  |j                  z  «      | _        |j                  |j                  z  | _	        | j                  dz  | _
        |j                  | _        d| _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j                  | j                  z  |j
                  |j                  ¬«      | _        y )Nrg   g      à¿F©Úbias)r'   r(   r�   r‚   Úgetattrr/   Únum_attention_headsrg   re   rt   rn   Úattention_dropoutÚ	is_causalr   ÚLinearÚattention_biasÚq_projÚk_projÚv_projÚo_proj©r.   r�   r‚   r1   s      €r2   r(   zEuroBertAttention.__init__‹   sM  ø€ Ü‰ÑÔØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆ�r3   Nr4   Úposition_embeddingsrm   Úpast_key_valuesrp   r%   c                 ó
  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t        «      } || ||	|
|f| j                  sdn| j                   | j"                  dœ|¤Ž\  }} |j$                  g |¢d‘­Ž j'                  «       }| j)                  |«      }||fS )Nr7   r    r6   ç        )ro   rn   )rD   rg   rŒ   Úviewrv   r�   rŽ   r_   Úupdater‚   r   Úget_interfacer�   Ú_attn_implementationr~   rs   rˆ   rn   rc   ry   r�   )r.   r4   r‘   rm   r’   rp   Úinput_shapeÚhidden_shapeÚquery_statesrz   r{   rZ   r[   Úattention_interfacer}   r|   s                   r2   rA   zEuroBertAttention.forward¢   s‘  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r3   ©NNN)rG   rH   rI   Ú__doc__r!   Úintr(   r*   rJ   rC   r   r   r   rA   rK   rL   s   @r2   r€   r€   ‡   s¥   ø„ áGð
˜~ð 
¸#õ 
ð4 IMØ.2Ø(,ñ&)à—|‘|ð&)ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð&)ð Ÿ™ tÑ+ð	&)ð
  ™ð&)ð Ð+Ñ,ð&)ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷&)r3   r€   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEuroBertMLPc                 ó  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        t        j                  | j                  | j                  |j                  ¬«      | _        t        j                  | j                  | j                  |j                  ¬«      | _	        t        j                  | j                  | j                  |j                  ¬«      | _
        t        |j                     | _        y )Nr„   )r'   r(   r�   r/   Úintermediate_sizer   rŠ   Úmlp_biasÚ	gate_projÚup_projÚ	down_projr
   Ú
hidden_actÚact_fn©r.   r�   r1   s     €r2   r(   zEuroBertMLP.__init__Ì   s¸   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRX×RaÑRaÔbˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPV×P_ÑP_Ô`ˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRX×RaÑRaÔbˆŒÜ˜V×.Ñ.Ñ/ˆ�r3   c                 óˆ   — | j                  | j                  | j                  |«      «      | j                  |«      z  «      }|S ©N)r§   r©   r¥   r¦   )r.   rQ   r§   s      r2   rA   zEuroBertMLP.forwardÖ   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÑ#SÓTˆ	ØÐr3   )rG   rH   rI   r(   rA   rK   rL   s   @r2   r¡   r¡   Ë   s   ø„ ô0ör3   r¡   c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
eej                  ej                  f   dz  dee   dej                  fd„Zˆ xZS )ÚEuroBertDecoderLayerr�   r‚   c                 ó   •— t         ‰| �  «        |j                  | _        t        ||¬«      | _        t        |«      | _        t        |j                  |j                  ¬«      | _	        t        |j                  |j                  ¬«      | _
        y )N)r�   r‚   ©r0   )r'   r(   r/   r€   Ú	self_attnr¡   Úmlpr$   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormr�   s      €r2   r(   zEuroBertDecoderLayer.__init__Ü   sm   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä*°&ÀIÔNˆŒä˜vÓ&ˆŒÜ.¨v×/AÑ/AÀv×GZÑGZÔ[ˆÔÜ(7¸×8JÑ8JÐPV×PcÑPcÔ(dˆÕ%r3   Nr4   rm   Úposition_idsr’   Ú	use_cacher‘   rp   r%   c           
      óÀ   — |}| j                  |«      } | j                  d||||||dœ|¤Ž\  }}	||z   }|}| j                  |«      }| j                  |«      }||z   }|S )N)r4   rm   r¶   r’   r·   r‘   © )r´   r±   rµ   r²   )
r.   r4   rm   r¶   r’   r·   r‘   rp   ÚresidualÚ_s
             r2   rA   zEuroBertDecoderLayer.forwardæ   s�   € ð !ˆØ×,Ñ,¨]Ó;ˆà)˜4Ÿ>™>ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
Ñˆ�qð ! =Ñ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ  =Ñ0ˆØÐr3   )NNNFN)rG   rH   rI   r!   rŸ   r(   r*   rJ   Ú
LongTensorr   ÚboolrC   r   r   rA   rK   rL   s   @r2   r®   r®   Û   s¾   ø„ ðe˜~ð e¸#õ eð /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð+Ñ,ðð 
�‰÷r3   r®   c                   óJ   — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZeedœZy)ÚEuroBertPreTrainedModelr�   ÚmodelTr®   r’   )r4   Ú
attentionsN)rG   rH   rI   r!   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr®   r€   Ú_can_record_outputsr¹   r3   r2   r¿   r¿     sQ   … àÓØÐØ&*Ð#Ø/Ð0ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà-Ø'ñÑr3   r¿   c                   óÄ   ‡ — e Zd ZU ej                  ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d   de
dz  ded	ef   fd
„«       Z ej                  «       ed„ «       «       Zˆ xZS )ÚEuroBertRotaryEmbeddingÚinv_freqNr�   c                 ó®  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |«      \  }| _
        | j                  d|d¬«       | j                  d|j                  «       d¬«       y )NÚ	rope_typeÚdefaultrÏ   F)Ú
persistentÚoriginal_inv_freq)r'   r(   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr�   Úrope_parametersrÑ   Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)r.   r�   ÚdeviceÚrope_init_fnrÏ   r1   s        €r2   r(   z EuroBertRotaryEmbedding.__init__  s¶   ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àˆŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÒ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆ�$Ô(à×Ñ˜Z¨¸eÐÔDØ×ÑÐ0°(·.±.Ó2BÈuÐÕUr3   rÝ   ztorch.deviceÚseq_lenr%   ztorch.Tensorc                 ó  — | j                   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetarg   Ng      ð?r   r6   ©r9   )rÝ   r9   )	rØ   r†   r/   r‡   r*   ÚarangeÚint64r:   Úfloat)r�   rÝ   rß   ÚbaserO   Úattention_factorrÏ   s          r2   rÙ   z7EuroBertRotaryEmbedding.compute_default_rope_parameters,  s�   € ð& ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r3   c                 óN  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  ||fd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j	                  |j                   ¬
«      	j	                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   r7   r    ÚmpsÚcpuF)Údevice_typeÚenabledr6   rN   râ   )rÏ   rå   rb   rD   r:   rÝ   Ú
isinstanceÚtypeÚstrr   rv   r*   rP   rZ   rÚ   r[   r9   )
r.   rQ   r¶   Úinv_freq_expandedÚposition_ids_expandedrë   ÚfreqsÚembrZ   r[   s
             r2   rA   zEuroBertRotaryEmbedding.forwardJ  sR  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   ÃBFÆF$r¬   r�   )rG   rH   rI   r*   rJ   rÂ   r!   r(   Ústaticmethodr   rŸ   rC   rå   rÙ   Úno_gradr   rA   rK   rL   s   @r2   rÎ   rÎ     s˜   ø… Ø�l‰lÓñV˜~õ Vð  à(,Ø+/Ø"ñ*Ø Ñ%ð*à˜Ñ(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ñ	%ò	*ó ð*ð: €U‡]�]ƒ_Øñ<ó ó ô<r3   rÎ   c                   óÚ   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 ddej                  dej                  dz  dej                  dz  dej                  dz  dee   d	eez  fd
„«       «       «       Zˆ xZS )ÚEuroBertModelr�   c           	      ó  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                  ¬«      | _        t#        |¬«      | _        d| _        | j)                  «        y c c}w )Nr°   )r�   F)r'   r(   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr/   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersr®   Úlayersr$   r³   ÚnormrÎ   Ú
rotary_embÚgradient_checkpointingÚ	post_initr�   s      €r2   r(   zEuroBertModel.__init__\  sÆ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m‘mÜFKÈF×LdÑLdÓFeÖf¸Ô! &¨)Õ4Òfó
ˆŒô $ F×$6Ñ$6¸F×<OÑ<OÔPˆŒ	Ü1¸Ô@ˆŒØ&+ˆÔ#ð 	�‰Õùò gs   ÂDNÚ	input_idsrm   r¶   Úinputs_embedsrp   r%   c                 óÌ  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|€=t        j                  |j                  d   |j
                  ¬«      j                  d«      }t        | j                  ||¬«      }|}| j                  ||¬«      }| j                  d | j                  j                   D ]  }	 |	|f|||dœ|¤Ž}Œ | j                  |«      }t        |¬«      S )	Nz:You must specify exactly one of input_ids or inputs_embedsr    )rÝ   r   )r�   r  rm   )r¶   )rm   r‘   r¶   )Úlast_hidden_state)Ú
ValueErrorrý   r*   rã   rD   rÝ   rW   r   r�   r  r  r   r  r   )
r.   r  rm   r¶   r  rp   Úbidirectional_maskr4   r‘   Úencoder_layers
             r2   rA   zEuroBertModel.forwardl  s  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø*.×*;Ñ*;¸IÓ*FˆMàÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\×fÑfÐghÓiˆLä6Ø—;‘;Ø'Ø)ô
Ðð &ˆØ"Ÿo™o¨mÈ,˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÐIò 	ˆMÙ)Øðà1Ø$7Ø)ñ	ð
 ñ‰Mð	ð Ÿ	™	 -Ó0ˆÜØ+ô
ð 	
r3   )NNNN)rG   rH   rI   r!   r(   r   r   r   r*   r¼   rJ   ÚFloatTensorr   r   rC   r   rA   rK   rL   s   @r2   r÷   r÷   Z  s«   ø„ ð˜~õ ð   ØØð '+Ø.2Ø04Ø26ñ&
à×#Ñ#ð&
ð Ÿ™ tÑ+ð&
ð ×&Ñ&¨Ñ-ð	&
ð
 ×(Ñ(¨4Ñ/ð&
ð Ð+Ñ,ð&
ð 
�Ñ	 ò&
ó ó ó  ô&
r3   r÷   c                   ó0  ‡ — e Zd ZddiZddiZddgdgfiZdefˆ fd„Zee		 	 	 	 	 dd
e
j                  d	z  de
j                  d	z  de
j                  d	z  de
j                  d	z  de
j                  d	z  dee   dee
j                     ez  fd„«       «       Zˆ xZS )ÚEuroBertForMaskedLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr4   Úlogitsr�   c                 óØ   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  |j                  «      | _	        | j                  «        y r¬   )r'   r(   r÷   rÀ   r   rŠ   r/   rû   r¤   r  r  rª   s     €r2   r(   zEuroBertForMaskedLM.__init__ž  sL   ø€ Ü‰Ñ˜Ô Ü" 6Ó*ˆŒ
Ü—y‘y ×!3Ñ!3°V×5FÑ5FÈÏÉÓXˆŒð 	�‰Õr3   Nr  rm   r¶   r  Úlabelsrp   r%   c                 ó  —  | j                   d||||dœ|¤Ž}| j                  |j                  «      }d}	|�* | j                  d||| j                  j
                  dœ|¤Ž}	t        |	||j                  |j                  ¬«      S )a)  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, EuroBertForMaskedLM

        >>> model = EuroBertForMaskedLM.from_pretrained("EuroBERT/EuroBERT-210m")
        >>> tokenizer = AutoTokenizer.from_pretrained("EuroBERT/EuroBERT-210m")

        >>> text = "The capital of France is <|mask|>."
        >>> inputs = tokenizer(text, return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> # To get predictions for the mask:
        >>> masked_index = inputs["input_ids"][0].tolist().index(tokenizer.mask_token_id)
        >>> predicted_token_id = outputs.logits[0, masked_index].argmax(axis=-1)
        >>> predicted_token = tokenizer.decode(predicted_token_id)
        >>> print("Predicted token:", predicted_token)
        Predicted token:  Paris
        ```)r  rm   r¶   r  N)r  r  rû   ©Úlossr  r4   rÁ   r¹   )	rÀ   r  r	  Úloss_functionr�   rû   r   r4   rÁ   )
r.   r  rm   r¶   r  r  rp   Úoutputsr  r  s
             r2   rA   zEuroBertForMaskedLM.forward¦  sž   € ð> $. 4§:¡:ð $
ØØ)Ø%Ø'ñ	$
ð
 ñ$
ˆð —‘˜g×7Ñ7Ó8ˆØˆØÐØ%�4×%Ñ%Ðp¨V¸FÈtÏ{É{×OeÑOeÑpÐioÑpˆDäØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r3   ©NNNNN)rG   rH   rI   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr!   r(   r   r   r*   r¼   rJ   r  r   r   rC   r   rA   rK   rL   s   @r2   r  r  ˜  sô   ø„ à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€Hð˜~õ ð Øð .2Ø.2Ø04Ø26Ø*.ñ/
à×#Ñ# dÑ*ð/
ð Ÿ™ tÑ+ð/
ð ×&Ñ&¨Ñ-ð	/
ð
 ×(Ñ(¨4Ñ/ð/
ð × Ñ  4Ñ'ð/
ð Ð+Ñ,ð/
ð 
ˆu�|‰|Ñ	˜~Ñ	-ò/
ó ó ô/
r3   r  c                   ó  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ee   d
eej                     ez  fd„«       «       Zˆ xZS )Ú!EuroBertForSequenceClassificationr�   c                 ó–  •— t         ‰| �  |«       |j                  | _        |j                  | _        t	        |«      | _        t        j                  |j                  |j                  «      | _	        t        j                  «       | _        t        j                  |j                  | j                  «      | _        | j                  «        y r¬   )r'   r(   Ú
num_labelsÚclassifier_poolingr÷   rÀ   r   rŠ   r/   ÚdenseÚGELUÚ
activationÚ
classifierr  rª   s     €r2   r(   z*EuroBertForSequenceClassification.__init__Ü  sˆ   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØ"(×";Ñ";ˆÔä" 6Ó*ˆŒ
Ü—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'™'›)ˆŒÜŸ)™) F×$6Ñ$6¸¿¹ÓHˆŒØ�‰Õr3   Nr  rm   r¶   r  r  rp   r%   c                 óÎ  —  | j                   |f|||dœ|¤Ž}|d   }| j                  dv rÆ| j                  dk(  r
|d d …df   }	ny| j                  dk(  rj|€|j                  d¬«      }	nU|j                  |j                  «      }||j                  d«      z  j                  d¬«      }	|	|j                  dd	¬
«      z  }	| j                  	«      }	| j                  |	«      }	| j                  |	«      }
n¬| j                  dk(  r�| j                  |«      }| j                  |«      }| j                  |«      }
|€|
j                  d¬«      }
nU|j                  |
j                  «      }|
|j                  d«      z  j                  d¬«      }
|
|j                  dd	¬
«      z  }
d }|��¢|j                  
j                  «      }| j                  j                  €�| j                  dk(  rd| j                  _        nl| j                  dkD  rL|j                  t        j                  k(  s|j                  t        j                   k(  rd| j                  _        nd| j                  _        | j                  j                  dk(  rIt#        «       }| j                  dk(  r& ||
j%                  «       |j%                  «       «      }nŒ ||
|«      }n‚| j                  j                  dk(  r=t'        «       } ||
j)                  d| j                  «      |j)                  d«      «      }n,| j                  j                  dk(  rt+        «       } ||
|«      }t-        |
|j.                  |j0                  ¬«      S )N©rm   r¶   r  r   )Úbosr=   r)  r=   r    rN   r7   T)rO   r8   ÚlateÚ
regressionÚsingle_label_classificationÚmulti_label_classificationr  )rÀ   r"  r=   r:   rÝ   rW   Úsumr#  r%  r&  r�   Úproblem_typer!  r9   r*   ÚlongrŸ   r   Úsqueezer   r•   r   r   r4   rÁ   )r.   r  rm   r¶   r  r  rp   Úencoder_outputr	  Úpooled_outputr  rQ   r  Úloss_fcts                 r2   rA   z)EuroBertForSequenceClassification.forwardç  s  € ð $˜Ÿ™Øð
à)Ø%Ø'ñ	
ð
 ñ
ˆð +¨1Ñ-Ðà×"Ñ" oÑ5Ø×&Ñ&¨%Ò/Ø 1²!°Q°$Ñ 7‘à×(Ñ(¨FÒ2Ø!Ð)Ø$5×$:Ñ$:¸qÐ$:Ó$A‘Mà%3×%6Ñ%6Ð7H×7OÑ7OÓ%P�NØ%6¸×9QÑ9QÐRTÓ9UÑ%U×$ZÑ$ZÐ_`Ð$ZÓ$a�MØ! ^×%7Ñ%7¸AÀtÐ%7Ó%LÑL�Mà ŸJ™J }Ó5ˆMØ ŸO™O¨MÓ:ˆMØ—_‘_ ]Ó3‰Fà×$Ñ$¨Ò.Ø—
‘
Ð,Ó-ˆAØ—‘ Ó"ˆAØ—_‘_ QÓ'ˆFØÐ%ØŸ™¨˜Ó+‘à!/×!2Ñ!2°6·=±=Ó!A�Ø  >×#;Ñ#;¸BÓ#?Ñ?×DÑDÈÐDÓK�Ø˜.×,Ñ,°¸DÐ,ÓAÑA�àˆØÑØ—Y‘Y˜vŸ}™}Ó-ˆFØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ä'ØØØ(×6Ñ6Ø%×0Ñ0ô	
ð 	
r3   r  )rG   rH   rI   r!   r(   r   r   r*   r¼   rJ   r  r   r   rC   r   rA   rK   rL   s   @r2   r  r  Ú  sÓ   ø„ ð	˜~õ 	ð Øð .2Ø.2Ø04Ø26Ø*.ñJ
à×#Ñ# dÑ*ðJ
ð Ÿ™ tÑ+ðJ
ð ×&Ñ&¨Ñ-ð	J
ð
 ×(Ñ(¨4Ñ/ðJ
ð × Ñ  4Ñ'ðJ
ð Ð+Ñ,ðJ
ð 
ˆu�|‰|Ñ	Ð7Ñ	7òJ
ó ó ôJ
r3   r  c                   ó  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  dee   deez  fd„«       «       Zˆ xZS )ÚEuroBertForTokenClassificationr�   c                 óä   •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r¬   )
r'   r(   r!  r÷   rÀ   r   rŠ   r/   r&  r  rª   s     €r2   r(   z'EuroBertForTokenClassification.__init__8  sQ   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ" 6Ó*ˆŒ
äŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒØ�‰Õr3   c                 ó.   — | j                   j                  S r¬   ©rÀ   rý   rE   s    r2   Úget_input_embeddingsz3EuroBertForTokenClassification.get_input_embeddings@  s   € Ø�z‰z×&Ñ&Ð&r3   c                 ó&   — || j                   _        y r¬   r9  )r.   rl   s     r2   Úset_input_embeddingsz3EuroBertForTokenClassification.set_input_embeddingsC  s   € Ø"'ˆ�
‰
Õr3   Nr  rm   r¶   r  r  rp   r%   c                 ó"  —  | j                   |f|||dœ|¤Ž}|d   }| j                  |«      }	d}
|�<t        «       } ||	j                  d| j                  «      |j                  d«      «      }
t        |
|	|j                  |j                  ¬«      S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        r(  r   Nr7   r  )rÀ   r&  r   r•   r!  r   r4   rÁ   )r.   r  rm   r¶   r  r  rp   r  Úsequence_outputr  r  r4  s               r2   rA   z&EuroBertForTokenClassification.forwardF  s£   € ð" �$—*‘*Øð
à)Ø%Ø'ñ	
ð
 ñ
ˆð " !™*ˆØ—‘ Ó1ˆàˆØÐÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r3   r  )rG   rH   rI   r!   r(   r:  r<  r   r   r*   r¼   rJ   r  r   r   rC   r   rA   rK   rL   s   @r2   r6  r6  6  sË   ø„ ð˜~õ ò'ò(ð Øð .2Ø.2Ø04Ø26Ø*.ñ#
à×#Ñ# dÑ*ð#
ð Ÿ™ tÑ+ð#
ð ×&Ñ&¨Ñ-ð	#
ð
 ×(Ñ(¨4Ñ/ð#
ð × Ñ  4Ñ'ð#
ð Ð+Ñ,ð#
ð 
Ð&Ñ	&ò#
ó ó ô#
r3   r6  )r¿   r÷   r  r  r6  )r    )r”   )AÚcollections.abcr   Útypingr   r*   r   Útorch.nnr   r   r   Úactivationsr
   Úcache_utilsr   Úintegrationsr   r   r   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   Úutils.genericr   r   r   r   Úutils.output_capturingr   Úconfiguration_eurobertr!   ÚModuler$   rT   r_   rJ   rŸ   rh   rå   r~   r€   r¡   r®   r¿   rÎ   r÷   r  r  r6  Ú__all__r¹   r3   r2   ú<module>rQ     s  ðõ, %Ý ã Ý ß AÑ Aå !Ý  ß fÑ fÝ 6Ý 9ß pÓ pß Kß FÝ &Ý #ß mÓ mÝ 5Ý 2ñ ˜YÓ'ôJ�b—i‘ió Jó (ðJò((ñ Ð*Ó+òó ,ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(ó%ñ2 Ð)Ó*ô@)˜Ÿ	™	ó @)ó +ð@)ôF�"—)‘)ô ô (Ð5ô (ðV ô˜oó ó ðô$><˜bŸi™iô ><ðB ô:
Ð+ó :
ó ð:
ðz ô>
Ð1ó >
ó ð>
ðB ôX
Ð(?ó X
ó ðX
ðv ô4
Ð%<ó 4
ó ð4
òn�r3   