Ë
    ÜÓ^j¢“  ã                   óÄ  — d dl Z d dlmZ d dlmZmZ d dlZd dlmZ d dlm	Z	 d dl
mZmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZmZ ddlmZ ddlmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z,m-Z-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5 ddl6m7Z7m8Z8  e.jr                  e:«      Z; e-d¬«      e G d„ de«      «       «       Z< G d„ de	jz                  «      Z> G d„ de	jz                  «      Z? G d „ d!e7«      Z@ ed"«      d>d#„«       ZA eeA«       G d$„ d%e	jz                  «      «       ZB G d&„ d'e«      ZCe- G d(„ d)e(«      «       ZDe- G d*„ d+eD«      «       ZE G d,„ d-e	jz                  «      ZF e-d.¬/«       G d0„ d1eD«      «       ZG e-d2¬/«       G d3„ d4eD«      «       ZH e-d5¬/«       G d6„ d7eD«      «       ZIe- G d8„ d9eD«      «       ZJ e-d:¬/«       G d;„ d<eD«      «       ZKg d=¢ZLy)?é    N)ÚCallable)ÚLiteralÚOptional)Ústrict)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)ÚPreTrainedConfig)Úuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚ(create_bidirectional_sliding_window_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONS)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )Úeager_attention_forward)ÚGemma3RotaryEmbeddingÚrotate_halfzanswerdotai/ModernBERT-base)Ú
checkpointc                   ó´  ‡ — e Zd ZU dZdZdgZdddœZdZee	d<   d	Z
ee	d
<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZedz  e	d<   d Zeee   z  dz  e	d!<   d"Zedz  e	d#<   d"Zedz  e	d$<   d Zedz  e	d%<   dZee	d&<   d'Zeez  e	d(<   dZee   dz  e	d)<   dZ e!e"d*   e!f   dz  e	d+<   d,Z#ee	d-<   d'Z$eez  e	d.<   dZ%ee	d/<   d'Z&eez  e	d0<   d1Z'ee	d2<   d3Z(e"d4   e	d5<   d'Z)eez  e	d6<   dZ*ee	d7<   dZ+ee	d8<   dZ,ee	d9<   dZ-ee	d:<   d;Z.ee	d<<   d1Z/ee	d=<   ˆ fd>„Z0d?„ Z1ˆ fd@„Z2e3dA„ «       Z4e4jj                  dB„ «       Z4ˆ xZ6S )CÚModernBertConfiga+  
    initializer_cutoff_factor (`float`, *optional*, defaults to 2.0):
        The cutoff factor for the truncated_normal_initializer for initializing all weight matrices.
    norm_eps (`float`, *optional*, defaults to 1e-05):
        The epsilon used by the rms normalization layers.
    norm_bias (`bool`, *optional*, defaults to `False`):
        Whether to use bias in the normalization layers.
    local_attention (`int`, *optional*, defaults to 128):
        The window size for local attention.
    mlp_dropout (`float`, *optional*, defaults to 0.0):
        The dropout ratio for the MLP layers.
    decoder_bias (`bool`, *optional*, defaults to `True`):
        Whether to use bias in the decoder layers.
    classifier_pooling (`str`, *optional*, defaults to `"cls"`):
        The pooling method for the classifier. Should be either `"cls"` or `"mean"`. In local attention layers, the
        CLS token doesn't attend to all tokens on long sequences.
    classifier_bias (`bool`, *optional*, defaults to `False`):
        Whether to use bias in the classifier.
    classifier_activation (`str`, *optional*, defaults to `"gelu"`):
        The activation function for the classifier.
    deterministic_flash_attn (`bool`, *optional*, defaults to `False`):
        Whether to use deterministic flash attention. If `False`, inference will be faster but not deterministic.
    sparse_prediction (`bool`, *optional*, defaults to `False`):
        Whether to use sparse prediction for the masked language model instead of returning the full dense logits.
    sparse_pred_ignore_index (`int`, *optional*, defaults to -100):
        The index to ignore for the sparse prediction.

    Examples:

    ```python
    >>> from transformers import ModernBertModel, ModernBertConfig

    >>> # Initializing a ModernBert style configuration
    >>> configuration = ModernBertConfig()

    >>> # Initializing a model from the modernbert-base style configuration
    >>> model = ModernBertModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Ú
modernbertÚpast_key_valuesg     ˆAg     ˆÃ@)ÚglobalÚlocaliÀÄ  Ú
vocab_sizei   Úhidden_sizei€  Úintermediate_sizeé   Únum_hidden_layersé   Únum_attention_headsÚgeluÚhidden_activationi    Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeç       @Úinitializer_cutoff_factorgñhãˆµøä>Únorm_epsFÚ	norm_biasikÄ  NÚpad_token_idijÄ  Úeos_token_idiiÄ  Úbos_token_idÚcls_token_idÚsep_token_idÚattention_biasç        Úattention_dropoutÚlayer_types©Úfull_attentionÚsliding_attentionÚrope_parametersé€   Úlocal_attentionÚembedding_dropoutÚmlp_biasÚmlp_dropoutTÚdecoder_biasÚcls)rQ   ÚmeanÚclassifier_poolingÚclassifier_dropoutÚclassifier_biasÚclassifier_activationÚdeterministic_flash_attnÚsparse_predictioniœÿÿÿÚsparse_pred_ignore_indexÚtie_word_embeddingsc                 óÚ   •— |j                  dd«      }| j                  €8t        | j                  «      D �cg c]  }t	        ||z  «      rdnd‘Œ c}| _        t        ‰| �  di |¤Ž y c c}w )NÚglobal_attn_every_n_layersr   rI   rH   © )ÚgetrF   Úranger3   ÚboolÚsuperÚ__post_init__)ÚselfÚkwargsr\   ÚiÚ	__class__s       €ú|/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/modernbert/modular_modernbert.pyrb   zModernBertConfig.__post_init__‡   st   ø€ à%+§Z¡ZÐ0LÈaÓ%PÐ"Ø×ÑÐ#ô ˜t×5Ñ5Ó6ö àô (,¨AÐ0JÑ,JÔ'KÑ#ÐQaÑaò ˆDÔô
 	‰ÑÑ' Ó'ùò s   ·A(c                 ó¶  — |j                  dd «      }ddiddidœ}| j                  �| j                  n|| _        |�<| j                  d   j                  |«       | j                  d   j                  |«       | j                  j                  d«      €ddi| j                  d<   | j                  d   j	                  d|j                  d| j
                  d	   «      «       | j                  j                  d«      €ddi| j                  d<   | j                  d   j	                  d|j                  d
| j
                  d   «      «       | j                  «        |S )NÚrope_scalingÚ	rope_typeÚdefault)rI   rH   rH   rI   Ú
rope_thetaÚglobal_rope_thetar-   Úlocal_rope_thetar.   )ÚpoprJ   Úupdater^   Ú
setdefaultÚdefault_thetaÚstandardize_rope_params)rc   rd   ri   Údefault_rope_paramss       rg   Úconvert_rope_params_to_dictz,ModernBertConfig.convert_rope_params_to_dict’   sc  € Ø—z‘z .°$Ó7ˆð
 #.¨yÐ!9Ø*¨IÐ6ñ
Ðð 8<×7KÑ7KÐ7W˜t×3Ò3Ð]pˆÔØÐ#Ø× Ñ Ð!1Ñ2×9Ñ9¸,ÔGØ× Ñ Ð!4Ñ5×<Ñ<¸\ÔJð ×Ñ×#Ñ#Ð$4Ó5Ð=Ø6AÀ9Ð5MˆD× Ñ Ð!1Ñ2Ø×ÑÐ-Ñ.×9Ñ9Ø˜&Ÿ*™*Ð%8¸$×:LÑ:LÈXÑ:VÓWô	
ð ×Ñ×#Ñ#Ð$7Ó8Ð@Ø9DÀiÐ8PˆD× Ñ Ð!4Ñ5Ø×ÑÐ0Ñ1×<Ñ<Ø˜&Ÿ*™*Ð%7¸×9KÑ9KÈGÑ9TÓUô	
ð
 	×$Ñ$Ô&Øˆó    c                 óH   •— t         ‰| �  «       }|j                  dd «       |S )NÚreference_compile)ra   Úto_dictro   )rc   Úoutputrf   s     €rg   ry   zModernBertConfig.to_dict°   s#   ø€ Ü‘‘Ó"ˆØ�
‰
Ð&¨Ô-Øˆrv   c                 ó    — | j                   dz  S )zKHalf-window size: `local_attention` is the total window, so we divide by 2.r$   ©rL   ©rc   s    rg   Úsliding_windowzModernBertConfig.sliding_windowµ   s   € ð ×#Ñ# qÑ(Ð(rv   c                 ó   — |dz  | _         y)z<Set sliding_window by updating local_attention to 2 * value.r$   Nr|   ©rc   Úvalues     rg   r~   zModernBertConfig.sliding_windowº   s   € ð  % q™yˆÕrv   )7Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferencerr   r/   ÚintÚ__annotations__r0   r1   r3   r5   r7   Ústrr8   r9   Úfloatr;   r<   r=   r`   r>   r?   Úlistr@   rA   rB   rC   rE   rF   rJ   Údictr   rL   rM   rN   rO   rP   rS   rT   rU   rV   rW   rX   rY   rZ   rb   ru   ry   Úpropertyr~   ÚsetterÚ__classcell__©rf   s   @rg   r*   r*   4   s  ø… ñ(ðT €JØ#4Ð"5ÐØ(°8Ñ<€Mà€J�ÓØ€K�ÓØ!Ð�sÓ!ØÐ�sÓØ!Ð˜Ó!Ø#Ð�sÓ#Ø#'Ð˜SÓ'Ø#Ð�uÓ#Ø'*Ð˜uÓ*Ø€HˆeÓØ€IˆtÓØ$€L�#˜‘*Ó$Ø+0€L�#˜˜S™	‘/ DÑ(Ó0Ø$€L�#˜‘*Ó$Ø$€L�#˜‘*Ó$Ø$€L�#˜‘*Ó$Ø €N�DÓ Ø%(Ð�u˜s‘{Ó(Ø$(€K��c‘˜TÑ!Ó(ØY]€O�T˜'Ð"GÑHÈ$ÐNÑOÐRVÑVÓ]Ø€O�SÓØ%(Ð�u˜s‘{Ó(Ø€HˆdÓØ"€K�˜‘Ó"Ø€L�$ÓØ16Ð˜ Ñ.Ó6Ø&)Ð˜ ™Ó)Ø!€O�TÓ!Ø!'Ð˜3Ó'Ø%*Ð˜dÓ*Ø#Ð�tÓ#Ø$(Ð˜cÓ(Ø $Ð˜Ó$ô	(òô<ð
 ñ)ó ð)ð ×Ññ)ó ô)rv   r*   c                   óˆ   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej                  dz  dej                  dz  dej                  fd„Z	ˆ xZ
S )
ÚModernBertEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    Úconfigc                 ód  •— t         ‰| �  «        || _        t        j                  |j
                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _        y )N)Úpadding_idx©ÚepsÚbias)ra   Ú__init__r”   r   Ú	Embeddingr/   r0   r>   Útok_embeddingsÚ	LayerNormr<   r=   ÚnormÚDropoutrM   Údrop©rc   r”   rf   s     €rg   rš   zModernBertEmbeddings.__init__Å   sw   ø€ Ü‰ÑÔØˆŒÜ Ÿl™l¨6×+<Ñ+<¸f×>PÑ>PÐ^d×^qÑ^qÔrˆÔÜ—L‘L ×!3Ñ!3¸¿¹Èv×O_ÑO_Ô`ˆŒ	Ü—J‘J˜v×7Ñ7Ó8ˆ�	rv   NÚ	input_idsÚinputs_embedsÚreturnc                 ó¬   — |�"| j                  | j                  |«      «      }|S | j                  | j                  | j                  |«      «      «      }|S ©N)r    rž   rœ   )rc   r¢   r£   Úhidden_statess       rg   ÚforwardzModernBertEmbeddings.forwardÌ   sS   € ð Ð$Ø ŸI™I d§i¡i°Ó&>Ó?ˆMð Ðð !ŸI™I d§i¡i°×0CÑ0CÀIÓ0NÓ&OÓPˆMØÐrv   ©NN)r‚   rƒ   r„   r…   r*   rš   ÚtorchÚ
LongTensorÚTensorr¨   r�   r‘   s   @rg   r“   r“   À   sR   ø„ ñð9Ð/õ 9ð _cñØ×)Ñ)¨DÑ0ðØHMÏÉÐW[ÑH[ðà	�‰÷rv   r“   c                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚModernBertMLPa6  Applies the GLU at the end of each ModernBERT layer.

    Compared to the default BERT architecture, this block replaces :class:`~transformers.model.bert.modeling_bert.BertIntermediate`
    and :class:`~transformers.model.bert.modeling_bert.SelfOutput` with a single module that has similar functionality.
    r”   c                 ó¬  •— t         ‰| �  «        || _        t        j                  |j
                  t        |j                  «      dz  |j                  ¬«      | _	        t        |j                     | _        t        j                  |j                  «      | _        t        j                  |j                  |j
                  |j                  ¬«      | _        y )Nr$   ©r™   )ra   rš   r”   r   ÚLinearr0   rˆ   r1   rN   ÚWir   r7   ÚactrŸ   rO   r    ÚWor¡   s     €rg   rš   zModernBertMLP.__init__Ý   s“   ø€ Ü‰ÑÔØˆŒÜ—)‘)˜F×.Ñ.´°F×4LÑ4LÓ0MÐPQÑ0QÐX^×XgÑXgÔhˆŒÜ˜&×2Ñ2Ñ3ˆŒÜ—J‘J˜v×1Ñ1Ó2ˆŒ	Ü—)‘)˜F×4Ñ4°f×6HÑ6HÈvÏÉÔ_ˆ�rv   r§   r¤   c                 ó°   — | j                  |«      j                  dd¬«      \  }}| j                  | j                  | j	                  |«      |z  «      «      S )Nr$   éÿÿÿÿ©Údim)r²   Úchunkr´   r    r³   )rc   r§   ÚinputÚgates       rg   r¨   zModernBertMLP.forwardå   sI   € Ø—g‘g˜mÓ,×2Ñ2°1¸"Ð2Ó=‰ˆˆtØ�w‰w�t—y‘y §¡¨%£°4Ñ!7Ó8Ó9Ð9rv   )
r‚   rƒ   r„   r…   r*   rš   rª   r¬   r¨   r�   r‘   s   @rg   r®   r®   Ö   s2   ø„ ñð`Ð/õ `ð: U§\¡\ð :°e·l±l÷ :rv   r®   c                   ó|   ‡ — e Zd Zddefˆ fd„Ze	 	 	 	 ddedz  ded   dedz  dedz  de	d	e
f   f
ˆ fd
„«       Zˆ xZS )ÚModernBertRotaryEmbeddingNr”   c                 ó&   •— t         ‰| �  ||«       y r¦   )ra   rš   )rc   r”   Údevicerf   s      €rg   rš   z"ModernBertRotaryEmbedding.__init__ë   s   ø€ Ü‰Ñ˜ Õ(rv   r¿   ztorch.deviceÚseq_lenÚ
layer_typer¤   ztorch.Tensorc                 ó(   •— t         ‰| �  | |||«      S r¦   )ra   Úcompute_default_rope_parameters)r”   r¿   rÀ   rÁ   rf   s       €rg   rÃ   z9ModernBertRotaryEmbedding.compute_default_rope_parametersî   s   ø€ ô ‰wÑ6°v¸vÀwÐPZÓ[Ð[rv   r¦   ©NNNN)r‚   rƒ   r„   r*   rš   Ústaticmethodr   rˆ   rŠ   Útupler‹   rÃ   r�   r‘   s   @rg   r½   r½   ê   s�   ø„ ñ)Ð/õ )ð à*.Ø+/Ø"Ø!%ñ	\Ø  4Ñ'ð\à˜Ñ(ð\ð �t‘ð\ð ˜$‘Jð	\ð
 
ˆ~˜uÐ$Ñ	%ô\ó ô\rv   r½   Úrotary_pos_embc                 ób  — | j                   }|j                  |«      }|j                  |«      }| j                  «       |z  t        | j                  «       «      |z  z   }|j                  «       |z  t        |j                  «       «      |z  z   }|j	                  |«      |j	                  |«      fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )ÚdtypeÚ	unsqueezer‹   r'   Úto)ÚqÚkÚcosÚsinÚunsqueeze_dimÚoriginal_dtypeÚq_embedÚk_embeds           rg   Úapply_rotary_pos_embrÔ   ø   sŽ   € ð& —W‘W€NØ
�-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�w‰w‹y˜3‰¤;¨q¯w©w«yÓ#9¸CÑ#?Ñ@€GØ�w‰w‹y˜3‰¤;¨q¯w©w«yÓ#9¸CÑ#?Ñ@€GØ�:‰:�nÓ% w§z¡z°.Ó'AÐAÐArv   c                   óú   ‡ — e Zd ZdZddededz  fˆ fd„Z	 	 ddej                  de	ej                  ej                  f   dz  dej                  dz  d	e
e   d
e	ej                  ej                  dz  f   f
d„Zˆ xZS )ÚModernBertAttentiona‚  Performs multi-headed self attention on a batch of unpadded sequences.

    If Flash Attention 2 is installed, this module uses Flash Attention to improve throughput.
    If Flash Attention 2 is not installed, the implementation will use PyTorch's SDPA kernel,
    which requires padding and unpadding inputs, adding some overhead.

    See `forward` method for additional details.
    Nr”   Ú	layer_idxc                 óR  •— t         ‰| �  «        || _        || _        |j                  |j
                  z  dk7  r&t        d|j                  › d|j
                  › d�«      ‚|j                  | _        |j                  | _        |j                  |j
                  z  | _	        t        j                  |j                  d| j                  z  |j
                  z  |j                  ¬«      | _        |j                  |   dk(  r|j                  dz   | _        nd | _        d	| _        t        j                  |j                  |j                  |j                  ¬«      | _        |j                  d
kD  r%t        j$                  |j                  «      | _        y t        j&                  «       | _        y )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (ú)r   r°   rI   é   FrD   )ra   rš   r”   r×   r0   r5   Ú
ValueErrorrE   rW   Úhead_dimr   r±   rC   ÚWqkvrF   r~   Ú	is_causalr´   rŸ   ÚIdentityÚout_drop©rc   r”   r×   rf   s      €rg   rš   zModernBertAttention.__init__  sz  ø€ Ü‰ÑÔØˆŒØ"ˆŒà×Ñ × :Ñ :Ñ:¸aÒ?ÜØ# F×$6Ñ$6Ð#7Ð7mÐnt÷  oIñ  oIð  nJð  JKð  Lóð ð "(×!9Ñ!9ˆÔØ(.×(GÑ(GˆÔ%Ø×*Ñ*¨f×.HÑ.HÑHˆŒÜ—I‘IØ×Ñ  D§M¡MÑ 1°F×4NÑ4NÑ NÐU[×UjÑUjô
ˆŒ	ð ×Ñ˜iÑ(Ð,?Ò?ð #)×"7Ñ"7¸!Ñ";ˆDÕà"&ˆDÔàˆŒä—)‘)˜F×.Ñ.°×0BÑ0BÈ×I^ÑI^Ô_ˆŒØ@F×@XÑ@XÐ[^Ò@^œŸ
™
 6×#;Ñ#;Ó<ˆ�Ôdf×doÑdoÓdqˆ�rv   r§   Úposition_embeddingsÚattention_maskrd   r¤   c                 óÒ  — |j                   d d }| j                  |«      } |j                  g |¢d‘d‘| j                  ‘­Ž }|j	                  d¬«      \  }}}	|j                  dd«      }|j                  dd«      }|	j                  dd«      }	|\  }
}t        |||
|d¬«      \  }}t        j                  | j                  j                  t        «      } || |||	|f| j                  r| j                  nd| j                  d	z  | j                  | j                  d
œ|¤Ž\  }} |j                   g |¢d‘­Ž j#                  «       }| j%                  | j'                  |«      «      }||fS )Nr¶   r   éýÿÿÿr·   rÚ   r$   )rÐ   rD   ç      à¿)ÚdropoutÚscalingr~   Údeterministic)ÚshaperÝ   ÚviewrÜ   ÚunbindÚ	transposerÔ   r   Úget_interfacer”   Ú_attn_implementationr%   ÚtrainingrE   r~   rW   ÚreshapeÚ
contiguousrà   r´   )rc   r§   râ   rã   rd   Úinput_shapeÚqkvÚquery_statesÚ
key_statesÚvalue_statesrÎ   rÏ   Úattention_interfaceÚattn_outputÚattn_weightss                  rg   r¨   zModernBertAttention.forward;  s‚  € ð $×)Ñ)¨#¨2Ð.ˆà�i‰i˜Ó&ˆØˆc�h‰hÐ:˜Ð: QÐ:¨Ð:¨D¯M©MÒ:ˆØ14·±À°Ó1CÑ.ˆ�j ,à#×-Ñ-¨a°Ó3ˆØ×)Ñ)¨!¨QÓ/ˆ
Ø#×-Ñ-¨a°Ó3ˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÐjkÔ#lÑ ˆ�jä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð%
ð /3¯mªm�D×*Ò*ÀØ—M‘M 4Ñ'Ø×.Ñ.Ø×7Ñ7ñ%
ð ñ%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—m‘m D§G¡G¨KÓ$8Ó9ˆØ˜LÐ(Ð(rv   r¦   r©   )r‚   rƒ   r„   r…   r*   rˆ   rš   rª   r¬   rÆ   r   r   r¨   r�   r‘   s   @rg   rÖ   rÖ     s¤   ø„ ññrÐ/ð r¸CÀ$¹Jõ rð@ IMØ.2ñ	')à—|‘|ð')ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð')ð Ÿ™ tÑ+ð	')ð
 Ð+Ñ,ð')ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷')rv   rÖ   c                   ó´   ‡ — e Zd Zddededz  fˆ fd„Z	 	 ddej                  dej                  dz  dej                  dz  dee	   d	ej                  f
d
„Z
ˆ xZS )ÚModernBertEncoderLayerNr”   r×   c                 óÔ  •— t         ‰| �  «        || _        || _        |dk(  rt	        j
                  «       | _        n;t	        j                  |j                  |j                  |j                  ¬«      | _        t        ||¬«      | _        t	        j                  |j                  |j                  |j                  ¬«      | _        t        |«      | _        |j                   |   | _        y )Nr   r—   )r”   r×   )ra   rš   r”   r×   r   rß   Ú	attn_normr�   r0   r<   r=   rÖ   ÚattnÚmlp_normr®   ÚmlprF   Úattention_typerá   s      €rg   rš   zModernBertEncoderLayer.__init__f  s§   ø€ Ü‰ÑÔØˆŒØ"ˆŒØ˜Š>ÜŸ[™[›]ˆD�NäŸ\™\¨&×*<Ñ*<À&Ç/Á/ÐX^×XhÑXhÔiˆDŒNÜ'¨vÀÔKˆŒ	ÜŸ™ V×%7Ñ%7¸V¿_¹_ÐSY×ScÑScÔdˆŒÜ  Ó(ˆŒØ$×0Ñ0°Ñ;ˆÕrv   r§   rã   râ   rd   r¤   c                 ó¦   —  | j                   | j                  |«      f||dœ|¤Ž\  }}||z   }|| j                  | j                  |«      «      z   }|S )N)râ   rã   )rÿ   rþ   r  r   )rc   r§   rã   râ   rd   rù   Ú_s          rg   r¨   zModernBertEncoderLayer.forwards  sg   € ð #˜Ÿ™Ø�N‰N˜=Ó)ð
à 3Ø)ñ
ð ñ	
‰ˆ�Qð &¨Ñ3ˆØ%¨¯©°·±¸}Ó1MÓ(NÑNˆØÐrv   r¦   r©   )r‚   rƒ   r„   r*   rˆ   rš   rª   r¬   r   r   r¨   r�   r‘   s   @rg   rü   rü   e  sx   ø„ ñ<Ð/ð <¸CÀ$¹Jõ <ð  /3Ø37ñ	à—|‘|ðð Ÿ™ tÑ+ðð #Ÿ\™\¨DÑ0ð	ð
 Ð+Ñ,ðð 
�‰÷rv   rü   c                   ó–   ‡ — e Zd ZU eed<   dZdZddgZdZdZ	dZ
dZeedœZ ej                   «       dej$                  fˆ fd„«       Zˆ xZS )	ÚModernBertPreTrainedModelr”   ÚmodelTr“   rü   )r§   Ú
attentionsÚmodulec                 óR  •‡— t         ‰	| �  |«       | j                  j                  Š‰€dŠdt        j
                  dt        fˆfd„}| j                  j                  | j                  j                  t        j                  d| j                  j                  z  «      z  | j                  j                  | j                  j                  dz  dœ}t        |t        «      r ||j                  |d   «       y t        |t        «      r- ||j                   |d	   «        ||j"                  |d
   «       y t        |t$        «      r- ||j&                  |d	   «        ||j"                  |d
   «       y t        |t(        «      r ||j*                  |d
   «       y t        |t,        «      r ||j.                  |d
   «       y t        |t0        t2        t4        t6        f«      r ||j8                  |d   «       y t        |t:        «      r£|j<                  D ]“  }|j>                  }|j@                  |   dk7  rtB        |j@                  |      } ||j                  |¬«      \  }}tE        jF                  tI        ||› d�«      |«       tE        jF                  tI        ||› d�«      |«       Œ• y y )Nr   r	  Ústdc                 óê   •— t        j                  | j                  d|‰ |z  ‰|z  ¬«       t        | t        j
                  «      r-| j                  � t        j                  | j                  «       y y y )NrD   )rR   r  ÚaÚb)ÚinitÚtrunc_normal_ÚweightÚ
isinstancer   r±   r™   Úzeros_)r	  r  Úcutoff_factors     €rg   Úinit_weightz<ModernBertPreTrainedModel._init_weights.<locals>.init_weightœ  sd   ø€ Ü×ÑØ—‘ØØØ �. 3Ñ&Ø #Ñ%õô ˜&¤"§)¡)Ô,Ø—;‘;Ð*Ü—K‘K §¡Õ,ð +ð -rv   r:   ræ   )ÚinÚoutÚ	embeddingÚ	final_outr  r  r  r  rk   )rÁ   Ú	_inv_freqÚ_original_inv_freq)%ra   Ú_init_weightsr”   r;   r   ÚModuler‹   r9   ÚmathÚsqrtr3   r0   r  r“   rœ   r®   r²   r´   rÖ   rÝ   ÚModernBertPredictionHeadÚdenseÚModernBertForMaskedLMÚdecoderÚ#ModernBertForSequenceClassificationÚModernBertForMultipleChoiceÚ ModernBertForTokenClassificationÚModernBertForQuestionAnsweringÚ
classifierr½   rF   rÃ   rj   r   r  Úcopy_Úgetattr)
rc   r	  r  ÚstdsrÁ   Úrope_init_fnÚcurr_inv_freqr  r  rf   s
           @€rg   r  z'ModernBertPreTrainedModel._init_weights•  s4  ù€ ä‰Ñ˜fÔ%ØŸ™×=Ñ=ˆØÐ ØˆMð	-¤§	¡	ð 	-´õ 	-ð —+‘+×/Ñ/Ø—;‘;×0Ñ0´4·9±9¸SÀ4Ç;Á;×C`ÑC`Ñ=`Ó3aÑaØŸ™×6Ñ6ØŸ™×0Ñ0°$Ñ6ñ	
ˆô �fÔ2Ô3Ù˜×-Ñ-¨t°KÑ/@ÕAÜ˜¤Ô.Ù˜Ÿ	™	 4¨¡:Ô.Ù˜Ÿ	™	 4¨¡;Õ/Ü˜Ô 3Ô4Ù˜Ÿ™ T¨$¡ZÔ0Ù˜Ÿ	™	 4¨¡;Õ/Ü˜Ô 8Ô9Ù˜Ÿ™ d¨5¡kÕ2Ü˜Ô 5Ô6Ù˜Ÿ™¨¨U©Õ4ÜØä3Ü+Ü0Ü.ð	ô
ñ ˜×)Ñ)¨4°Ñ+<Õ=Ü˜Ô 9Ô:Ø$×0Ñ0ò ^�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ò<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ô#UÑ �˜qÜ—
‘
œ7 6¨j¨\¸Ð+CÓDÀmÔTÜ—
‘
œ7 6¨j¨\Ð9KÐ+LÓMÈ}Õ]ñ^ð ;rv   )r‚   rƒ   r„   r*   r‰   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendrü   rÖ   Ú_can_record_outputsrª   Úno_gradr   r  r  r�   r‘   s   @rg   r  r  …  sr   ø… àÓØÐØ&*Ð#Ø/Ð1IÐJÐØÐØ€NØÐØ"&Ðð 0Ø)ñÐð
 €U‡]�]ƒ_ð7^ B§I¡Iô 7^ó ô7^rv   r  c                   óæ   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zeee		 	 	 	 dde
j                  dz  de
j                  dz  de
j                  dz  d	e
j                  dz  d
ee   defd„«       «       «       Zˆ xZS )ÚModernBertModelr”   c           	      ó¬  •— t         ‰| �  |«       || _        t        |«      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _
        t        j                  |j                  |j                  |j                  ¬«      | _        t!        |¬«      | _        d| _        | j'                  «        y c c}w )Nr—   )r”   F)ra   rš   r”   r“   Ú
embeddingsr   Ú
ModuleListr_   r3   rü   Úlayersr�   r0   r<   r=   Ú
final_normr½   Ú
rotary_embÚgradient_checkpointingÚ	post_initrá   s      €rg   rš   zModernBertModel.__init__Ò  sŸ   ø€ Ü‰Ñ˜Ô ØˆŒÜ.¨vÓ6ˆŒÜ—m‘mÜHMÈf×NfÑNfÓHgÖh¸9Ô# F¨IÕ6Òhó
ˆŒô Ÿ,™, v×'9Ñ'9¸v¿¹ÐU[×UeÑUeÔfˆŒÜ3¸6ÔBˆŒØ&+ˆÔ#Ø�‰Õùò is   ÁCc                 ó.   — | j                   j                  S r¦   ©r:  rœ   r}   s    rg   Úget_input_embeddingsz$ModernBertModel.get_input_embeddingsÞ  s   € Ø�‰×-Ñ-Ð-rv   c                 ó&   — || j                   _        y r¦   rB  r€   s     rg   Úset_input_embeddingsz$ModernBertModel.set_input_embeddingsá  s   € Ø).ˆ�‰Õ&rv   Nr¢   rã   Úposition_idsr£   rd   r¤   c                 ó¦  — |d u |d uz  rt        d«      ‚|�|j                  d   n|j                  d   }|�|j                  n|j                  }|€&t        j                  ||¬«      j                  d«      }| j                  ||¬«      }t        |x}	t        «      s'| j                  ||dœ}
t        d
i |
¤Žt        d
i |
¤Ždœ}	i }t        | j                  j                  «      D ]  }| j                  |||«      ||<   Œ | j                  D ](  } ||f|	|j                      ||j                      dœ|¤Ž}Œ* | j#                  |«      }t%        |¬	«      S )Nz:You must specify exactly one of input_ids or inputs_embedsrÚ   ©r¿   r   )r¢   r£   )r”   r£   rã   rG   )rã   râ   )Úlast_hidden_stater]   )rÛ   rê   r¿   rª   ÚarangerÊ   r:  r  r�   r”   r   r   ÚsetrF   r>  r<  r  r=  r   )rc   r¢   rã   rF  r£   rd   rÀ   r¿   r§   Úattention_mask_mappingÚmask_kwargsrâ   rÁ   Úencoder_layers                 rg   r¨   zModernBertModel.forwardä  s|  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZà,9Ð,E�-×%Ñ% aÒ(È9Ï?É?Ð[\ÑK]ˆØ%.Ð%:�×!Ò!À×@TÑ@TˆàÐÜ Ÿ<™<¨¸Ô?×IÑIÈ!ÓLˆLàŸ™°)È=˜ÓYˆä°NÐBÐ0ÄDÔIàŸ+™+Ø!.Ø"0ñˆKô #<Ñ"J¸kÑ"JÜ%MÑ%\ÐP[Ñ%\ñ&Ð"ð
 !ÐÜ˜dŸk™k×5Ñ5Ó6ò 	gˆJØ.2¯o©o¸mÈ\Ð[eÓ.fÐ 
Ò+ð	gð "Ÿ[™[ò 	ˆMÙ)Øðà5°m×6RÑ6RÑSØ$7¸×8TÑ8TÑ$Uñð ñ	‰Mð	ð Ÿ™¨Ó6ˆä°Ô?Ð?rv   rÄ   )r‚   rƒ   r„   r*   rš   rC  rE  r"   r#   r   rª   r«   r¬   r   r   r   r¨   r�   r‘   s   @rg   r8  r8  Ð  s¼   ø„ ð
Ð/õ 
ò.ò/ð  ØØð .2Ø.2Ø04Ø-1ñ,@à×#Ñ# dÑ*ð,@ð Ÿ™ tÑ+ð,@ð ×&Ñ&¨Ñ-ð	,@ð
 —|‘| dÑ*ð,@ð Ð+Ñ,ð,@ð 
ò,@ó ó ó  ô,@rv   r8  c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )r   r”   c                 óJ  •— t         ‰| �  «        || _        t        j                  |j
                  |j
                  |j                  «      | _        t        |j                     | _
        t        j                  |j
                  |j                  |j                  ¬«      | _        y )Nr—   )ra   rš   r”   r   r±   r0   rU   r!  r   rV   r³   r�   r<   r=   rž   r¡   s     €rg   rš   z!ModernBertPredictionHead.__init__  sq   ø€ Ü‰ÑÔØˆŒÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÀv×G]ÑG]Ó^ˆŒ
Ü˜&×6Ñ6Ñ7ˆŒÜ—L‘L ×!3Ñ!3¸¿¹Èv×O_ÑO_Ô`ˆ�	rv   r§   r¤   c                 ó`   — | j                  | j                  | j                  |«      «      «      S r¦   )rž   r³   r!  )rc   r§   s     rg   r¨   z ModernBertPredictionHead.forward  s#   € Ø�y‰y˜Ÿ™ $§*¡*¨]Ó";Ó<Ó=Ð=rv   )	r‚   rƒ   r„   r*   rš   rª   r¬   r¨   r�   r‘   s   @rg   r   r     s-   ø„ ðaÐ/õ að> U§\¡\ð >°e·l±l÷ >rv   r   zd
    The ModernBert Model with a decoder head on top that is used for masked language modeling.
    )Úcustom_introc                   ó>  ‡ — e Zd ZddiZdefˆ fd„Zd„ Zdej                  fd„Z	e
e	 	 	 	 	 dd	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dee   deej                     ez  fd„«       «       Zˆ xZS )r"  zdecoder.weightz&model.embeddings.tok_embeddings.weightr”   c                 ót  •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        t        j                  |j                  |j                  |j                  ¬«      | _        | j                  j                  | _        | j                  j                  | _        | j                  «        y )Nr°   )ra   rš   r”   r8  r  r   Úheadr   r±   r0   r/   rP   r#  rX   rY   r@  r¡   s     €rg   rš   zModernBertForMaskedLM.__init__*  s…   ø€ Ü‰Ñ˜Ô ØˆŒÜ$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—y‘y ×!3Ñ!3°V×5FÑ5FÈV×M`ÑM`ÔaˆŒà!%§¡×!>Ñ!>ˆÔØ(,¯©×(LÑ(LˆÔ%ð 	�‰Õrv   c                 ó   — | j                   S r¦   ©r#  r}   s    rg   Úget_output_embeddingsz+ModernBertForMaskedLM.get_output_embeddings7  s   € Ø�|‰|Ðrv   Únew_embeddingsc                 ó   — || _         y r¦   rW  )rc   rY  s     rg   Úset_output_embeddingsz+ModernBertForMaskedLM.set_output_embeddings:  s	   € Ø%ˆ�rv   Nr¢   rã   rF  r£   Úlabelsrd   r¤   c                 óÊ  —  | j                   d||||dœ|¤Ž}|d   }| j                  rK|�I|j                  d«      }|j                  |j                  d   d«      }|| j                  k7  }	||	   }||	   }| j                  | j                  |«      «      }
d }|�* | j                  |
|fd| j                  j                  i|¤Ž}t        ||
|j                  |j                  ¬«      S )N©r¢   rã   rF  r£   r   r¶   r/   ©ÚlossÚlogitsr§   r  r]   )r  rX   rë   rê   rY   r#  rU  Úloss_functionr”   r/   r   r§   r  )rc   r¢   rã   rF  r£   r\  rd   ÚoutputsrI  Úmask_tokensra  r`  s               rg   r¨   zModernBertForMaskedLM.forward=  s	  € ð �$—*‘*ð 
ØØ)Ø%Ø'ñ	
ð
 ñ
ˆð $ A™JÐà×!Ò! fÐ&8à—[‘[ “_ˆFØ 1× 6Ñ 6°v·|±|ÀA±ÈÓ KÐð ! D×$AÑ$AÑAˆKØ 1°+Ñ >ÐØ˜KÑ(ˆFà—‘˜dŸi™iÐ(9Ó:Ó;ˆàˆØÐØ%�4×%Ñ% f¨fÑbÀÇÁ×AWÑAWÐbÐ[aÑbˆDäØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rv   ©NNNNN)r‚   rƒ   r„   Ú_tied_weights_keysr*   rš   rX  r   r±   r[  r!   r   rª   r«   r¬   r   r   rÆ   r   r¨   r�   r‘   s   @rg   r"  r"  "  sç   ø„ ð +Ð,TÐUÐðÐ/õ òð&°B·I±Ió &ð Øð .2Ø.2Ø,0Ø-1Ø&*ñ'
à×#Ñ# dÑ*ð'
ð Ÿ™ tÑ+ð'
ð —l‘l TÑ)ð	'
ð
 —|‘| dÑ*ð'
ð —‘˜tÑ#ð'
ð Ð+Ñ,ð'
ð 
ˆu�|‰|Ñ	˜~Ñ	-ò'
ó ó ô'
rv   r"  z`
    The ModernBert Model with a sequence classification head on top that performs pooling.
    c                   ó  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e
e   d
eej                     ez  fd„«       «       Zˆ xZS )r$  r”   c                 ón  •— t         ‰| �  |«       |j                  | _        || _        t	        |«      | _        t        |«      | _        t        j                  j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j!                  «        y r¦   )ra   rš   Ú
num_labelsr”   r8  r  r   rU  rª   r   rŸ   rT   r    r±   r0   r(  r@  r¡   s     €rg   rš   z,ModernBertForSequenceClassification.__init__o  s‚   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØˆŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õrv   Nr¢   rã   rF  r£   r\  rd   r¤   c                 ód  —  | j                   d||||dœ|¤Ž}|d   }| j                  j                  dk(  r
|dd…df   }n�| j                  j                  dk(  rw|€=t        j                  |j
                  dd |j                  t        j                  ¬«      }||j                  d«      z  j                  d	¬
«      |j                  d	d¬«      z  }| j                  |«      }	| j                  |	«      }	| j                  |	«      }
d}|��‡| j                  j                  €�| j                  d	k(  rd| j                  _        nl| j                  d	kD  rL|j                  t        j                   k(  s|j                  t        j"                  k(  rd| j                  _        nd| j                  _        | j                  j                  dk(  rIt%        «       }| j                  d	k(  r& ||
j'                  «       |j'                  «       «      }nŒ ||
|«      }n‚| j                  j                  dk(  r=t)        «       } ||
j+                  d| j                  «      |j+                  d«      «      }n,| j                  j                  dk(  rt-        «       } ||
|«      }t/        ||
|j0                  |j2                  ¬«      S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        r^  r   rQ   NrR   r$   )r¿   rÉ   r¶   rÚ   r·   T©r¸   ÚkeepdimÚ
regressionÚsingle_label_classificationÚmulti_label_classificationr_  r]   )r  r”   rS   rª   Úonesrê   r¿   r`   rÊ   ÚsumrU  r    r(  Úproblem_typeri  rÉ   Úlongrˆ   r
   Úsqueezer	   rë   r   r   r§   r  )rc   r¢   rã   rF  r£   r\  rd   rc  rI  Úpooled_outputra  r`  Úloss_fcts                rg   r¨   z+ModernBertForSequenceClassification.forward|  se  € ð" �$—*‘*ð 
ØØ)Ø%Ø'ñ	
ð
 ñ
ˆð $ A™JÐà�;‰;×)Ñ)¨UÒ2Ø 1²!°Q°$Ñ 7ÑØ�[‰[×+Ñ+¨vÒ5ØÐ%Ü!&§¡Ø%×+Ñ+¨B¨QÐ/Ð8I×8PÑ8PÔX]×XbÑXbô"�ð "3°^×5MÑ5MÈbÓ5QÑ!Q× VÑ VÐ[\Ð VÓ ]Ð`n×`rÑ`rØ˜tð asó añ !Ðð Ÿ	™	Ð"3Ó4ˆØŸ	™	 -Ó0ˆØ—‘ Ó/ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rv   re  )r‚   rƒ   r„   r*   rš   r!   r   rª   r«   r¬   r   r   rÆ   r   r¨   r�   r‘   s   @rg   r$  r$  i  sÎ   ø„ ðÐ/õ ð Øð .2Ø.2Ø,0Ø-1Ø&*ñC
à×#Ñ# dÑ*ðC
ð Ÿ™ tÑ+ðC
ð —l‘l TÑ)ð	C
ð
 —|‘| dÑ*ðC
ð —‘˜tÑ#ðC
ð Ð+Ñ,ðC
ð 
ˆu�|‰|Ñ	Ð7Ñ	7òC
ó ó ôC
rv   r$  zv
    The ModernBert Model with a token classification head on top, e.g. for Named Entity Recognition (NER) tasks.
    c                   ó  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e
e   d
eej                     ez  fd„«       «       Zˆ xZS )r&  r”   c                 ó`  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        |«      | _        t        j                  j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r¦   ©ra   rš   ri  r8  r  r   rU  rª   r   rŸ   rT   r    r±   r0   r(  r@  r¡   s     €rg   rš   z)ModernBertForTokenClassification.__init__Ê  s{   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õrv   Nr¢   rã   rF  r£   r\  rd   r¤   c                 óf  —  | j                   d||||dœ|¤Ž}|d   }| j                  |«      }| j                  |«      }| j                  |«      }	d}
|�<t	        «       } ||	j                  d| j                  «      |j                  d«      «      }
t        |
|	|j                  |j                  ¬«      S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        r^  r   Nr¶   r_  r]   )
r  rU  r    r(  r	   rë   ri  r   r§   r  )rc   r¢   rã   rF  r£   r\  rd   rc  rI  ra  r`  rv  s               rg   r¨   z(ModernBertForTokenClassification.forwardÖ  sÅ   € ð �$—*‘*ð 
ØØ)Ø%Ø'ñ	
ð
 ñ
ˆð $ A™JÐà ŸI™IÐ&7Ó8ÐØ ŸI™IÐ&7Ó8ÐØ—‘Ð!2Ó3ˆàˆØÐÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rv   re  )r‚   rƒ   r„   r*   rš   r!   r   rª   r«   r¬   r   r   rÆ   r   r¨   r�   r‘   s   @rg   r&  r&  Ä  sÅ   ø„ ð
Ð/õ 
ð Øð .2Ø.2Ø,0Ø-1Ø&*ñ$
à×#Ñ# dÑ*ð$
ð Ÿ™ tÑ+ð$
ð —l‘l TÑ)ð	$
ð
 —|‘| dÑ*ð$
ð —‘˜tÑ#ð$
ð Ð+Ñ,ð$
ð 
ˆu�|‰|Ñ	Ð4Ñ	4ò$
ó ó ô$
rv   r&  c                   ó  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
   d
eej                     ez  fd„«       «       Zˆ xZS )r'  r”   c                 ó`  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        |«      | _        t        j                  j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r¦   ry  r¡   s     €rg   rš   z'ModernBertForQuestionAnswering.__init__  sy   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒà�‰Õrv   Nr¢   rã   rF  Ústart_positionsÚend_positionsrd   r¤   c                 óÆ  —  | j                   |f||dœ|¤Ž}|d   }| j                  |«      }| j                  |«      }| j                  |«      }	|	j	                  dd¬«      \  }
}|
j                  d«      j                  «       }
|j                  d«      j                  «       }d }|�|� | j                  |
|||fi |¤Ž}t        ||
||j                  |j                  ¬«      S )N)rã   rF  r   rÚ   r¶   r·   )r`  Ústart_logitsÚ
end_logitsr§   r  )r  rU  r    r(  Úsplitrt  rò   rb  r   r§   r  )rc   r¢   rã   rF  r}  r~  rd   rc  rI  ra  r€  r�  r`  s                rg   r¨   z&ModernBertForQuestionAnswering.forward  s  € ð �$—*‘*Øð
à)Ø%ñ
ð ñ	
ˆð $ A™JÐà ŸI™IÐ&7Ó8ÐØ ŸI™IÐ&7Ó8ÐØ—‘Ð!2Ó3ˆà#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆØÐ&¨=Ð+DØ%�4×%Ñ% l°JÀÐQ^ÑiÐbhÑiˆDä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
rv   re  )r‚   rƒ   r„   r*   rš   r!   r   rª   r¬   r   r   rÆ   r   r¨   r�   r‘   s   @rg   r'  r'  ÿ  sÃ   ø„ ð	Ð/õ 	ð Øð *.Ø.2Ø,0Ø/3Ø-1ñ#
à—<‘< $Ñ&ð#
ð Ÿ™ tÑ+ð#
ð —l‘l TÑ)ð	#
ð
 Ÿ™¨Ñ,ð#
ð —|‘| dÑ*ð#
ð Ð+Ñ,ð#
ð 
ˆu�|‰|Ñ	Ð;Ñ	;ò#
ó ó ô#
rv   r'  z«
    The ModernBert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks.
    c                   ó  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e
e   d
eej                     ez  fd„«       «       Zˆ xZS )r%  r”   c                 ó8  •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        t        j                  j                  |j                  «      | _        t        j                  |j                  d«      | _        | j                  «        y )NrÚ   )ra   rš   r”   r8  r  r   rU  rª   r   rŸ   rT   r    r±   r0   r(  r@  r¡   s     €rg   rš   z$ModernBertForMultipleChoice.__init__:  so   ø€ Ü‰Ñ˜Ô ØˆŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)™) F×$6Ñ$6¸Ó:ˆŒð 	�‰Õrv   Nr¢   rã   rF  r£   r\  rd   r¤   c                 óÔ  — |�|j                   d   n|j                   d   }|�!|j                  d|j                  d«      «      nd}|�!|j                  d|j                  d«      «      nd}|�!|j                  d|j                  d«      «      nd}|�1|j                  d|j                  d«      |j                  d«      «      nd} | j                  d||||dœ|¤Ž}|d   }	| j                  j
                  dk(  r”t        j                  |	j                   d   |	j                  ¬«      }
|�,|j                  d¬	«      j                  |	j                  «      }n0t        j                  dt        j                  |	j                  ¬
«      }|	|
|f   }	nS| j                  j
                  dk(  r:|j                  dd¬«      }|	|j                  d«      z  j                  d¬	«      |z  }	| j                  |	«      }| j!                  |«      }| j#                  |«      }|j                  d|«      }d}|�t%        j&                  «       } |||«      }t)        |||j*                  |j,                  ¬«      S )a&  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors.
        NrÚ   r¶   éþÿÿÿr^  r   rQ   rH  r·   )rÉ   r¿   rR   Trk  r_  r]   )rê   rë   Úsizer  r”   rS   rª   rJ  r¿   ÚargmaxrË   Útensorrs  rq  rÊ   rU  r    r(  r   r	   r   r§   r  )rc   r¢   rã   rF  r£   r\  rd   Únum_choicesrc  rI  Ú	indices_0Úcls_maskÚnum_non_pad_tokensru  ra  Úreshaped_logitsr`  rv  s                     rg   r¨   z#ModernBertForMultipleChoice.forwardF  sn  € ð  -6Ð,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆà>GÐ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÐG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ð(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð �$—*‘*ð 
ØØ)Ø%Ø'ñ	
ð
 ñ
ˆð $ A™JÐð �;‰;×)Ñ)¨UÒ2ÜŸ™Ð%6×%<Ñ%<¸QÑ%?ÐHY×H`ÑH`ÔaˆIàÐ)Ø)×0Ñ0°RÐ0Ó8×;Ñ;Ð<M×<TÑ<TÓU‘ô !Ÿ<™<¨´·±ÐDU×D\ÑD\Ô]�à 1°)¸XÐ2EÑ FÑð �[‰[×+Ñ+¨vÒ5Ø!/×!3Ñ!3¸À4Ð!3Ó!HÐØ!2°^×5MÑ5MÈbÓ5QÑ!Q× VÑ VÐ[\Ð VÓ ]Ð`rÑ rÐàŸ	™	Ð"3Ó4ˆØŸ	™	 -Ó0ˆØ—‘ Ó/ˆà Ÿ+™+ b¨+Ó6ˆàˆØÐÜ×*Ñ*Ó,ˆHÙ˜O¨VÓ4ˆDä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
rv   re  )r‚   rƒ   r„   r*   rš   r!   r   rª   r«   r¬   r   r   rÆ   r   r¨   r�   r‘   s   @rg   r%  r%  4  sÎ   ø„ ð
Ð/õ 
ð Øð .2Ø.2Ø,0Ø-1Ø&*ñC
à×#Ñ# dÑ*ðC
ð Ÿ™ tÑ+ðC
ð —l‘l TÑ)ð	C
ð
 —|‘| dÑ*ðC
ð —‘˜tÑ#ðC
ð Ð+Ñ,ðC
ð 
ˆu�|‰|Ñ	Ð8Ñ	8òC
ó ó ôC
rv   r%  )r*   r8  r  r"  r$  r&  r'  r%  )rÚ   )Mr  Úcollections.abcr   Útypingr   r   rª   Úhuggingface_hub.dataclassesr   r   Útorch.nnr   r	   r
   Ú r   r  Úactivationsr   Úconfiguration_utilsr   Úintegrationsr   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r    Úutils.genericr!   r"   Úutils.output_capturingr#   Úalign.modeling_alignr%   Úgemma3.modeling_gemma3r&   r'   Ú
get_loggerr‚   Úloggerr*   r  r“   r®   r½   rÔ   rÖ   rü   r  r8  r   r"  r$  r&  r'  r%  Ú__all__r]   rv   rg   ú<module>r¥     s7  ðó  Ý $ß $ã Ý .Ý ß AÑ Aå &Ý !Ý 3ß Iß `Ý 9÷÷ õ 7ß FÝ &ß @Ñ @ß IÝ 5Ý :ß Gð 
ˆ×	Ñ	˜HÓ	%€ñ Ð8Ô9ØôG)Ð'ó G)ó ó :ðG)ôT˜2Ÿ9™9ô ô,:�B—I‘Iô :ô(\Ð 5ô \ñ Ð*Ó+òBó ,ðBñ4 Ð)Ó*ôN)˜"Ÿ)™)ó N)ó +ðN)ôbÐ7ô ð@ ôG^ ó G^ó ðG^ðT ôB@Ð/ó B@ó ðB@ôJ	>˜rŸy™yô 	>ñ ðôô
?
Ð5ó ?
óð
?
ñD ðôô
S
Ð*Có S
óð
S
ñl ðôô
3
Ð'@ó 3
óð
3
ðl ô1
Ð%>ó 1
ó ð1
ñh ðôô
R
Ð";ó R
óð
R
òj	�rv   