Ë
    ÛÓ^j¶4  ã                   ó¬  — d dl Z d dl mZ d dlmZmZmZ ddlmZ ddlm	Z	 ddl
mZmZmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZmZ  ed¬«      e G d„ de«      «       «       Z G d„ de«      Z  G d„ de«      Z! G d„ de«      Z" G d„ de«      Z#e G d„ de"«      «       Z$e G d„ de"«      «       Z%e G d„ d e"«      «       Z&g d!¢Z'y)"é    N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Ústrict)Úcreate_bidirectional_mask)ÚBaseModelOutputÚMaskedLMOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚRopeParameters)ÚUnpack)Úauto_docstring)ÚTransformersKwargsÚcan_return_tupleé   )ÚLlamaConfig)ÚLlamaAttentionÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRMSNormzEuroBERT/EuroBERT-210m)Ú
checkpointc                   ó˜  ‡ — e Zd ZU dZdZdZeed<   dZeed<   dZ	eed<   d	Z
eed
<   d	Zeed<   dZedz  ed<   dZeed<   dZeed<   dZeed<   dZeed<   dZedz  ed<   dZeee   z  dz  ed<   dZedz  ed<   dZeed<   dZeed<   dZeed <   dZeez  dz  ed!<   dZeed"<   d#Zeez  ed$<   dZ eed%<   dZ!edz  ed&<   d'Z"eed(<   ˆ fd)„Z#ˆ xZ$S )*ÚEuroBertConfiga�  
    mask_token_id (`int`, *optional*, defaults to 128002):
        Mask token id.
    classifier_pooling (`str`, *optional*, defaults to `"late"`):
        The pooling strategy to use for the classifier. Can be one of ['bos', 'mean', 'late'].

    ```python
    >>> from transformers import EuroBertModel, EuroBertConfig

    >>> # Initializing a EuroBert eurobert-base style configuration
    >>> configuration = EuroBertConfig()

    >>> # Initializing a model from the eurobert-base style configuration
    >>> model = EuroBertModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Úeuroberti õ Ú
vocab_sizei   Úhidden_sizei   Úintermediate_sizeé   Únum_hidden_layersÚnum_attention_headsNÚnum_key_value_headsÚsiluÚ
hidden_acti    Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeçñhãˆµøä>Úrms_norm_epsi ô Úbos_token_idiô Úeos_token_idÚpad_token_idiô Úmask_token_idé   Úpretraining_tpFÚtie_word_embeddingsÚrope_parametersÚattention_biasg        Úattention_dropoutÚmlp_biasÚhead_dimÚlateÚclassifier_poolingc                 ó^   •— | j                   €| j                  | _         t        ‰| �  di |¤Ž y )N© )r#   r"   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úx/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/eurobert/modular_eurobert.pyr;   zEuroBertConfig.__post_init__N   s-   ø€ Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$Ü‰ÑÑ' Ó'ó    )%Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typer   ÚintÚ__annotations__r   r   r!   r"   r#   r%   Ústrr&   r'   Úfloatr)   r*   r+   Úlistr,   r-   r/   r0   Úboolr1   r   Údictr2   r3   r4   r5   r7   r;   Ú__classcell__©r>   s   @r?   r   r      s)  ø… ñð& €Jà€J�ÓØ€K�ÓØ!Ð�sÓ!ØÐ�sÓØ!Ð˜Ó!Ø&*Ð˜˜t™Ó*Ø€J�ÓØ#'Ð˜SÓ'Ø#Ð�uÓ#Ø€L�%ÓØ%€L�#˜‘*Ó%Ø+1€L�#˜˜S™	‘/ DÑ(Ó1Ø%€L�#˜‘*Ó%Ø€M�3ÓØ€N�CÓØ %Ð˜Ó%Ø48€O�^ dÑ*¨TÑ1Ó8Ø €N�DÓ Ø%(Ð�s˜U‘{Ó(Ø€HˆdÓØ€Hˆc�D‰jÓØ$Ð˜Ó$÷(ð (r@   r   c                   ó    ‡ — e Zd Zdˆ fd„	Zˆ xZS )ÚEuroBertRMSNormc                 ó&   •— t         ‰| �  ||«       y ©N)r:   Ú__init__)r<   r   Úepsr>   s      €r?   rS   zEuroBertRMSNorm.__init__U   s   ø€ Ü‰Ñ˜ cÕ*r@   )r(   )rA   rB   rC   rS   rM   rN   s   @r?   rP   rP   T   s   ø„ ÷+ñ +r@   rP   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚEuroBertAttentionÚconfigÚ	layer_idxc                 ó4   •— t         ‰| �  ||«       d| _        y )NF)r:   rS   Ú	is_causal)r<   rW   rX   r>   s      €r?   rS   zEuroBertAttention.__init__Z   s   ø€ Ü‰Ñ˜ Ô+Øˆ�r@   )rA   rB   rC   r   rF   rS   rM   rN   s   @r?   rV   rV   Y   s   ø„ ð˜~ð ¸#÷ ñ r@   rV   c                   ó   — e Zd Zy)ÚEuroBertPreTrainedModelN)rA   rB   rC   r9   r@   r?   r\   r\   _   s   „ Ør@   r\   c                   ó¤   — e Zd Z	 	 	 	 d	dej                  dej
                  dz  dej                  dz  dej                  dz  dee   de	e
z  fd„Zy)
ÚEuroBertModelNÚ	input_idsÚattention_maskÚposition_idsÚinputs_embedsr=   Úreturnc                 óÌ  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|€=t        j                  |j                  d   |j
                  ¬«      j                  d«      }t        | j                  ||¬«      }|}| j                  ||¬«      }| j                  d | j                  j                   D ]  }	 |	|f|||dœ|¤Ž}Œ | j                  |«      }t        |¬«      S )	Nz:You must specify exactly one of input_ids or inputs_embedsr.   )Údevicer   )rW   rb   r`   )ra   )r`   Úposition_embeddingsra   )Úlast_hidden_state)Ú
ValueErrorÚembed_tokensÚtorchÚarangeÚshapere   Ú	unsqueezer	   rW   Ú
rotary_embÚlayersr!   Únormr
   )
r<   r_   r`   ra   rb   r=   Úbidirectional_maskÚhidden_statesrf   Úencoder_layers
             r?   ÚforwardzEuroBertModel.forwardd   s  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø*.×*;Ñ*;¸IÓ*FˆMàÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\×fÑfÐghÓiˆLä6Ø—;‘;Ø'Ø)ô
Ðð &ˆØ"Ÿo™o¨mÈ,˜oÓWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÐIò 	ˆMÙ)Øðà1Ø$7Ø)ñ	ð
 ñ‰Mð	ð Ÿ	™	 -Ó0ˆÜØ+ô
ð 	
r@   )NNNN)rA   rB   rC   rj   Ú
LongTensorÚTensorÚFloatTensorr   r   Útupler
   rt   r9   r@   r?   r^   r^   c   s   „ ð '+Ø.2Ø04Ø26ñ&
à×#Ñ#ð&
ð Ÿ™ tÑ+ð&
ð ×&Ñ&¨Ñ-ð	&
ð
 ×(Ñ(¨4Ñ/ð&
ð Ð+Ñ,ð&
ð 
�Ñ	 ô&
r@   r^   c                   ó0  ‡ — e Zd ZddiZddiZddgdgfiZdefˆ fd„Zee		 	 	 	 	 dd
e
j                  d	z  de
j                  d	z  de
j                  d	z  de
j                  d	z  de
j                  d	z  dee   dee
j                     ez  fd„«       «       Zˆ xZS )ÚEuroBertForMaskedLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrr   ÚlogitsrW   c                 óØ   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  |j                  «      | _	        | j                  «        y rR   )r:   rS   r^   Úmodelr   ÚLinearr   r   r4   r{   Ú	post_init©r<   rW   r>   s     €r?   rS   zEuroBertForMaskedLM.__init__“   sL   ø€ Ü‰Ñ˜Ô Ü" 6Ó*ˆŒ
Ü—y‘y ×!3Ñ!3°V×5FÑ5FÈÏÉÓXˆŒð 	�‰Õr@   Nr_   r`   ra   rb   Úlabelsr=   rc   c                 ó  —  | j                   d||||dœ|¤Ž}| j                  |j                  «      }d}	|�* | j                  d||| j                  j
                  dœ|¤Ž}	t        |	||j                  |j                  ¬«      S )a)  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, EuroBertForMaskedLM

        >>> model = EuroBertForMaskedLM.from_pretrained("EuroBERT/EuroBERT-210m")
        >>> tokenizer = AutoTokenizer.from_pretrained("EuroBERT/EuroBERT-210m")

        >>> text = "The capital of France is <|mask|>."
        >>> inputs = tokenizer(text, return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> # To get predictions for the mask:
        >>> masked_index = inputs["input_ids"][0].tolist().index(tokenizer.mask_token_id)
        >>> predicted_token_id = outputs.logits[0, masked_index].argmax(axis=-1)
        >>> predicted_token = tokenizer.decode(predicted_token_id)
        >>> print("Predicted token:", predicted_token)
        Predicted token:  Paris
        ```)r_   r`   ra   rb   N)r}   rƒ   r   ©Úlossr}   rr   Ú
attentionsr9   )	r   r{   rg   Úloss_functionrW   r   r   rr   r‡   )
r<   r_   r`   ra   rb   rƒ   r=   Úoutputsr}   r†   s
             r?   rt   zEuroBertForMaskedLM.forward›   sž   € ð> $. 4§:¡:ð $
ØØ)Ø%Ø'ñ	$
ð
 ñ$
ˆð —‘˜g×7Ñ7Ó8ˆØˆØÐØ%�4×%Ñ%Ðp¨V¸FÈtÏ{É{×OeÑOeÑpÐioÑpˆDäØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r@   ©NNNNN)rA   rB   rC   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr   rS   r   r   rj   ru   rv   rw   r   r   rx   r   rt   rM   rN   s   @r?   rz   rz   �   sô   ø„ à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€Hð˜~õ ð Øð .2Ø.2Ø04Ø26Ø*.ñ/
à×#Ñ# dÑ*ð/
ð Ÿ™ tÑ+ð/
ð ×&Ñ&¨Ñ-ð	/
ð
 ×(Ñ(¨4Ñ/ð/
ð × Ñ  4Ñ'ð/
ð Ð+Ñ,ð/
ð 
ˆu�|‰|Ñ	˜~Ñ	-ò/
ó ó ô/
r@   rz   c                   ó  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ee   d
eej                     ez  fd„«       «       Zˆ xZS )Ú!EuroBertForSequenceClassificationrW   c                 ó–  •— t         ‰| �  |«       |j                  | _        |j                  | _        t	        |«      | _        t        j                  |j                  |j                  «      | _	        t        j                  «       | _        t        j                  |j                  | j                  «      | _        | j                  «        y rR   )r:   rS   Ú
num_labelsr7   r^   r   r   r€   r   ÚdenseÚGELUÚ
activationÚ
classifierr�   r‚   s     €r?   rS   z*EuroBertForSequenceClassification.__init__Ñ   sˆ   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØ"(×";Ñ";ˆÔä" 6Ó*ˆŒ
Ü—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'™'›)ˆŒÜŸ)™) F×$6Ñ$6¸¿¹ÓHˆŒØ�‰Õr@   Nr_   r`   ra   rb   rƒ   r=   rc   c                 óÎ  —  | j                   |f|||dœ|¤Ž}|d   }| j                  dv rÆ| j                  dk(  r
|d d …df   }	ny| j                  dk(  rj|€|j                  d¬«      }	nU|j                  |j                  «      }||j                  d«      z  j                  d¬«      }	|	|j                  dd	¬
«      z  }	| j                  	«      }	| j                  |	«      }	| j                  |	«      }
n¬| j                  dk(  r�| j                  |«      }| j                  |«      }| j                  |«      }
|€|
j                  d¬«      }
nU|j                  |
j                  «      }|
|j                  d«      z  j                  d¬«      }
|
|j                  dd	¬
«      z  }
d }|��¢|j                  
j                  «      }| j                  j                  €�| j                  dk(  rd| j                  _        nl| j                  dkD  rL|j                  t        j                  k(  s|j                  t        j                   k(  rd| j                  _        nd| j                  _        | j                  j                  dk(  rIt#        «       }| j                  dk(  r& ||
j%                  «       |j%                  «       «      }nŒ ||
|«      }n‚| j                  j                  dk(  r=t'        «       } ||
j)                  d| j                  «      |j)                  d«      «      }n,| j                  j                  dk(  rt+        «       } ||
|«      }t-        |
|j.                  |j0                  ¬«      S )N©r`   ra   rb   r   )ÚbosÚmeanr˜   r™   r.   )ÚdiméÿÿÿÿT)rš   Úkeepdimr6   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr…   )r   r7   r™   Útore   rm   Úsumr’   r”   r•   rW   Úproblem_typer‘   Údtyperj   ÚlongrF   r   Úsqueezer   Úviewr   r   rr   r‡   )r<   r_   r`   ra   rb   rƒ   r=   Úencoder_outputrg   Úpooled_outputr}   Úxr†   Úloss_fcts                 r?   rt   z)EuroBertForSequenceClassification.forwardÜ   s  € ð $˜Ÿ™Øð
à)Ø%Ø'ñ	
ð
 ñ
ˆð +¨1Ñ-Ðà×"Ñ" oÑ5Ø×&Ñ&¨%Ò/Ø 1²!°Q°$Ñ 7‘à×(Ñ(¨FÒ2Ø!Ð)Ø$5×$:Ñ$:¸qÐ$:Ó$A‘Mà%3×%6Ñ%6Ð7H×7OÑ7OÓ%P�NØ%6¸×9QÑ9QÐRTÓ9UÑ%U×$ZÑ$ZÐ_`Ð$ZÓ$a�MØ! ^×%7Ñ%7¸AÀtÐ%7Ó%LÑL�Mà ŸJ™J }Ó5ˆMØ ŸO™O¨MÓ:ˆMØ—_‘_ ]Ó3‰Fà×$Ñ$¨Ò.Ø—
‘
Ð,Ó-ˆAØ—‘ Ó"ˆAØ—_‘_ QÓ'ˆFØÐ%ØŸ™¨˜Ó+‘à!/×!2Ñ!2°6·=±=Ó!A�Ø  >×#;Ñ#;¸BÓ#?Ñ?×DÑDÈÐDÓK�Ø˜.×,Ñ,°¸DÐ,ÓAÑA�àˆØÑØ—Y‘Y˜vŸ}™}Ó-ˆFØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ä'ØØØ(×6Ñ6Ø%×0Ñ0ô	
ð 	
r@   rŠ   )rA   rB   rC   r   rS   r   r   rj   ru   rv   rw   r   r   rx   r   rt   rM   rN   s   @r?   r�   r�   Ï   sÓ   ø„ ð	˜~õ 	ð Øð .2Ø.2Ø04Ø26Ø*.ñJ
à×#Ñ# dÑ*ðJ
ð Ÿ™ tÑ+ðJ
ð ×&Ñ&¨Ñ-ð	J
ð
 ×(Ñ(¨4Ñ/ðJ
ð × Ñ  4Ñ'ðJ
ð Ð+Ñ,ðJ
ð 
ˆu�|‰|Ñ	Ð7Ñ	7òJ
ó ó ôJ
r@   r�   c                   ó  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zee	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  dee   deez  fd„«       «       Zˆ xZS )ÚEuroBertForTokenClassificationrW   c                 óä   •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y rR   )
r:   rS   r‘   r^   r   r   r€   r   r•   r�   r‚   s     €r?   rS   z'EuroBertForTokenClassification.__init__-  sQ   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ" 6Ó*ˆŒ
äŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒØ�‰Õr@   c                 ó.   — | j                   j                  S rR   ©r   ri   )r<   s    r?   Úget_input_embeddingsz3EuroBertForTokenClassification.get_input_embeddings5  s   € Ø�z‰z×&Ñ&Ð&r@   c                 ó&   — || j                   _        y rR   r¯   )r<   Úvalues     r?   Úset_input_embeddingsz3EuroBertForTokenClassification.set_input_embeddings8  s   € Ø"'ˆ�
‰
Õr@   Nr_   r`   ra   rb   rƒ   r=   rc   c                 ó"  —  | j                   |f|||dœ|¤Ž}|d   }| j                  |«      }	d}
|�<t        «       } ||	j                  d| j                  «      |j                  d«      «      }
t        |
|	|j                  |j                  ¬«      S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        r—   r   Nr›   r…   )r   r•   r   r¦   r‘   r   rr   r‡   )r<   r_   r`   ra   rb   rƒ   r=   r‰   Úsequence_outputr}   r†   rª   s               r?   rt   z&EuroBertForTokenClassification.forward;  s£   € ð" �$—*‘*Øð
à)Ø%Ø'ñ	
ð
 ñ
ˆð " !™*ˆØ—‘ Ó1ˆàˆØÐÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r@   rŠ   )rA   rB   rC   r   rS   r°   r³   r   r   rj   ru   rv   rw   r   r   rx   r   rt   rM   rN   s   @r?   r¬   r¬   +  sË   ø„ ð˜~õ ò'ò(ð Øð .2Ø.2Ø04Ø26Ø*.ñ#
à×#Ñ# dÑ*ð#
ð Ÿ™ tÑ+ð#
ð ×&Ñ&¨Ñ-ð	#
ð
 ×(Ñ(¨4Ñ/ð#
ð × Ñ  4Ñ'ð#
ð Ð+Ñ,ð#
ð 
Ð&Ñ	&ò#
ó ó ô#
r@   r¬   )r   r\   r^   rz   r�   r¬   )(rj   r   Útorch.nnr   r   r   Úconfiguration_utilsr   Úmasking_utilsr	   Úmodeling_outputsr
   r   r   r   Úmodeling_rope_utilsr   Úprocessing_utilsr   Úutilsr   Úutils.genericr   r   Úllama.configuration_llamar   Úllama.modeling_llamar   r   r   r   r   rP   rV   r\   r^   rz   r�   r¬   Ú__all__r9   r@   r?   ú<module>rÁ      sñ   ðó  Ý ß AÑ Aå )Ý 6ß pÓ pÝ 1Ý &Ý #ß AÝ 3ß aÓ añ Ð3Ô4Øô0(�[ó 0(ó ó 5ð0(ôf+�lô +ô
˜ô ô	Ð2ô 	ô'
�Jô '
ðT ô>
Ð1ó >
ó ð>
ðB ôX
Ð(?ó X
ó ðX
ðv ô4
Ð%<ó 4
ó ð4
òn�r@   