Ë
    ÜÓ^jóŽ  ã                   óì  — d dl Z d dlmZ d dlmZ d dlZd dlmZ d dlmZm	Z	m
Z
 ddlmZ ddlmZ dd	lmZmZ dd
lmZmZ ddlmZ ddlmZmZmZmZmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z) ddl*m+Z+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1  G d„ dejd                  «      Z3 G d„ dejd                  «      Z4 G d„ dejd                  «      Z5	 d@dejd                  dejl                  dejl                  dejl                  dejl                  dz  d e7d!e7fd"„Z8d#„ Z9 ed$«      dAd%„«       Z: ee:«       G d&„ d'ejd                  «      «       Z; G d(„ d)e«      Z<e) G d*„ d+e$«      «       Z=e) G d,„ d-e=«      «       Z> G d.„ d/ejd                  «      Z? e)d0¬1«       G d2„ d3e=«      «       Z@ e)d4¬1«       G d5„ d6e=«      «       ZA e)d7¬1«       G d8„ d9e=«      «       ZBe) G d:„ d;e=«      «       ZC e)d<¬1«       G d=„ d>e=«      «       ZDg d?¢ZEy)Bé    N)ÚCallable)ÚOptional)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)Úuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚ(create_bidirectional_sliding_window_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstring)Úcan_return_tupleÚmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚModernBertConfigc                   óˆ   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej                  dz  dej                  dz  dej                  fd„Z	ˆ xZ
S )
ÚModernBertEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    Úconfigc                 ód  •— t         ‰| �  «        || _        t        j                  |j
                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _        y )N)Úpadding_idx©ÚepsÚbias)ÚsuperÚ__init__r&   r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚtok_embeddingsÚ	LayerNormÚnorm_epsÚ	norm_biasÚnormÚDropoutÚembedding_dropoutÚdrop©Úselfr&   Ú	__class__s     €ú}/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/modernbert/modeling_modernbert.pyr-   zModernBertEmbeddings.__init__9   sw   ø€ Ü‰ÑÔØˆŒÜ Ÿl™l¨6×+<Ñ+<¸f×>PÑ>PÐ^d×^qÑ^qÔrˆÔÜ—L‘L ×!3Ñ!3¸¿¹Èv×O_ÑO_Ô`ˆŒ	Ü—J‘J˜v×7Ñ7Ó8ˆ�	ó    NÚ	input_idsÚinputs_embedsÚreturnc                 ó¬   — |�"| j                  | j                  |«      «      }|S | j                  | j                  | j                  |«      «      «      }|S ©N)r9   r6   r2   )r;   r?   r@   Úhidden_statess       r=   ÚforwardzModernBertEmbeddings.forward@   sS   € ð Ð$Ø ŸI™I d§i¡i°Ó&>Ó?ˆMð Ðð !ŸI™I d§i¡i°×0CÑ0CÀIÓ0NÓ&OÓPˆMØÐr>   ©NN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r#   r-   ÚtorchÚ
LongTensorÚTensorrE   Ú__classcell__©r<   s   @r=   r%   r%   4   sR   ø„ ñð9Ð/õ 9ð _cñØ×)Ñ)¨DÑ0ðØHMÏÉÐW[ÑH[ðà	�‰÷r>   r%   c                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚModernBertMLPa6  Applies the GLU at the end of each ModernBERT layer.

    Compared to the default BERT architecture, this block replaces :class:`~transformers.model.bert.modeling_bert.BertIntermediate`
    and :class:`~transformers.model.bert.modeling_bert.SelfOutput` with a single module that has similar functionality.
    r&   c                 ó¬  •— t         ‰| �  «        || _        t        j                  |j
                  t        |j                  «      dz  |j                  ¬«      | _	        t        |j                     | _        t        j                  |j                  «      | _        t        j                  |j                  |j
                  |j                  ¬«      | _        y )Né   ©r+   )r,   r-   r&   r   ÚLinearr0   ÚintÚintermediate_sizeÚmlp_biasÚWir   Úhidden_activationÚactr7   Úmlp_dropoutr9   ÚWor:   s     €r=   r-   zModernBertMLP.__init__Q   s“   ø€ Ü‰ÑÔØˆŒÜ—)‘)˜F×.Ñ.´°F×4LÑ4LÓ0MÐPQÑ0QÐX^×XgÑXgÔhˆŒÜ˜&×2Ñ2Ñ3ˆŒÜ—J‘J˜v×1Ñ1Ó2ˆŒ	Ü—)‘)˜F×4Ñ4°f×6HÑ6HÈvÏÉÔ_ˆ�r>   rD   rA   c                 ó°   — | j                  |«      j                  dd¬«      \  }}| j                  | j                  | j	                  |«      |z  «      «      S )NrS   éÿÿÿÿ©Údim)rY   Úchunkr]   r9   r[   )r;   rD   ÚinputÚgates       r=   rE   zModernBertMLP.forwardY   sI   € Ø—g‘g˜mÓ,×2Ñ2°1¸"Ð2Ó=‰ˆˆtØ�w‰w�t—y‘y §¡¨%£°4Ñ!7Ó8Ó9Ð9r>   )
rG   rH   rI   rJ   r#   r-   rK   rM   rE   rN   rO   s   @r=   rQ   rQ   J   s2   ø„ ñð`Ð/õ `ð: U§\¡\ð :°e·l±l÷ :r>   rQ   c                   óÒ   ‡ — e Zd ZU ej                  ed<   ddefˆ fd„Ze	 	 	 	 ddedz  de	d   de
dz  dedz  d	ed
ef   f
d„«       Z ej                  «       edd„«       «       Zˆ xZS )ÚModernBertRotaryEmbeddingÚinv_freqNr&   c                 ót  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        t        t        |j                  «      «      | _        i | _	        | j                  D ]Ç  }| j
                  j                  |   }|€Œ|d   | j                  |<   | j                  }| j                  |   dk7  rt        | j                  |      } || j
                  |¬«      \  }}| j                  |› d�|d¬«       | j                  |› d�|j                  «       d¬«       t        | |› d�|«       ŒÉ y )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqF)Ú
persistentÚ_original_inv_freqÚ_attention_scaling)r,   r-   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr&   ÚlistÚsetÚlayer_typesri   Úrope_parametersÚcompute_default_rope_parametersr   Úregister_bufferÚcloneÚsetattr)	r;   r&   Údevicerl   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingr<   s	           €r=   r-   z"ModernBertRotaryEmbedding.__init__a   s6  ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØˆŒÜ¤ F×$6Ñ$6Ó 7Ó8ˆÔØˆŒØ×*Ñ*ò 	UˆJØŸ+™+×5Ñ5°jÑAˆKØÐ"Øà)4°[Ñ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÑ)¨YÒ6Ü2°4·>±>À*Ñ3MÑN�Ù4@ÀÇÁÐYcÔ4dÑ1ˆMÐ1Ø× Ñ  J <¨yÐ!9¸=ÐUZÐ Ô[Ø× Ñ  J <Ð/AÐ!BÀM×DWÑDWÓDYÐfkÐ ÔlÜ�D˜Z˜LÐ(:Ð;Ð=SÕTñ	Ur>   r|   ztorch.deviceÚseq_lenrl   rA   ztorch.Tensorc                 ó   — | j                   |   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a|  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`

        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNg      ð?r   rS   ©Údtype©r|   r†   )	rw   Úgetattrr0   Únum_attention_headsrK   ÚarangeÚint64ÚtoÚfloat)r&   r|   r�   rl   Úbasera   Úattention_factorrg   s           r=   rx   z9ModernBertRotaryEmbedding.compute_default_rope_parametersv   s•   € ð2 ×%Ñ% jÑ1°,Ñ?ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r>   c                 óN  — t        | |› d�«      }t        | |› d�«      }|d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬	«      5  |j                  «       |j                  «       z  j                  dd
«      }	t        j                  |	|	fd¬«      }
|
j                  «       |z  }|
j                  «       |z  }d d d «       j	                  |j                  ¬«      j	                  |j                  ¬«      fS # 1 sw Y   ŒAxY w)Nrm   rp   r   r_   r"   ÚmpsÚcpuF)Údevice_typeÚenabledrS   r`   r…   )rˆ   r�   ÚexpandÚshaperŒ   r|   Ú
isinstanceÚtypeÚstrr   Ú	transposerK   ÚcatÚcosÚsinr†   )r;   ÚxÚposition_idsrl   rg   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedr“   ÚfreqsÚembrœ   r�   s                r=   rE   z!ModernBertRotaryEmbedding.forwardš   sl  € ô ˜4 J <¨yÐ!9Ó:ˆÜ# D¨Z¨LÐ8JÐ*KÓLÐà$ Tª1¨d ]Ñ3×9Ñ9Ó;×BÑBÀ<×CUÑCUÐVWÑCXÐZ\Ð^_Ó`×cÑcÐde×dlÑdlÓmÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	0Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)Ð/Ñ/ˆCØ—'‘'“)Ð/Ñ/ˆC÷		0ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	0ð 	0ús   Ã*A1FÆF$rC   ©NNNN)rG   rH   rI   rK   rM   Ú__annotations__r#   r-   Ústaticmethodr   rV   r™   Útupler�   rx   Úno_gradr   rE   rN   rO   s   @r=   rf   rf   ^   s«   ø… Ø�l‰lÓñUÐ/õ Uð* à*.Ø+/Ø"Ø!%ñ	!*Ø  4Ñ'ð!*à˜Ñ(ð!*ð �t‘ð!*ð ˜$‘Jð	!*ð
 
ˆ~˜uÐ$Ñ	%ò!*ó ð!*ðF €U‡]�]ƒ_Øò<ó ó ô<r>   rf   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 óÀ  — t        j                  ||j                  dd«      «      |z  }|�||z   }t        j                  j                  |dt         j                  ¬«      j                  |j                  «      }t        j                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )NrS   r	   r_   )ra   r†   )ÚpÚtrainingr"   )rK   Úmatmulrš   r   Ú
functionalÚsoftmaxÚfloat32rŒ   r†   r°   r³   Ú
contiguous)
rª   r«   r¬   r­   r®   r¯   r°   ÚkwargsÚattn_weightsÚattn_outputs
             r=   Úeager_attention_forwardr¼   ­   sº   € ô —<‘<  s§}¡}°Q¸Ó':Ó;¸gÑE€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØ˜Ð$Ð$r>   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Nr_   rS   r`   )r–   rK   r›   )rž   Úx1Úx2s      r=   Úrotate_halfrÀ   Ã   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r>   Úrotary_pos_embc                 ób  — | j                   }|j                  |«      }|j                  |«      }| j                  «       |z  t        | j                  «       «      |z  z   }|j                  «       |z  t        |j                  «       «      |z  z   }|j	                  |«      |j	                  |«      fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )r†   Ú	unsqueezer�   rÀ   rŒ   )ÚqÚkrœ   r�   Úunsqueeze_dimÚoriginal_dtypeÚq_embedÚk_embeds           r=   Úapply_rotary_pos_embrÊ   Ê   sŽ   € ð& —W‘W€NØ
�-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�w‰w‹y˜3‰¤;¨q¯w©w«yÓ#9¸CÑ#?Ñ@€GØ�w‰w‹y˜3‰¤;¨q¯w©w«yÓ#9¸CÑ#?Ñ@€GØ�:‰:�nÓ% w§z¡z°.Ó'AÐAÐAr>   c                   óú   ‡ — e Zd ZdZddededz  fˆ fd„Z	 	 ddej                  de	ej                  ej                  f   dz  dej                  dz  d	e
e   d
e	ej                  ej                  dz  f   f
d„Zˆ xZS )ÚModernBertAttentiona‚  Performs multi-headed self attention on a batch of unpadded sequences.

    If Flash Attention 2 is installed, this module uses Flash Attention to improve throughput.
    If Flash Attention 2 is not installed, the implementation will use PyTorch's SDPA kernel,
    which requires padding and unpadding inputs, adding some overhead.

    See `forward` method for additional details.
    Nr&   Ú	layer_idxc                 óR  •— t         ‰| �  «        || _        || _        |j                  |j
                  z  dk7  r&t        d|j                  › d|j
                  › d�«      ‚|j                  | _        |j                  | _        |j                  |j
                  z  | _	        t        j                  |j                  d| j                  z  |j
                  z  |j                  ¬«      | _        |j                  |   dk(  r|j                  dz   | _        nd | _        d	| _        t        j                  |j                  |j                  |j                  ¬«      | _        |j                  d
kD  r%t        j$                  |j                  «      | _        y t        j&                  «       | _        y )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (ú)r	   rT   Úsliding_attentionr"   Fç        )r,   r-   r&   rÍ   r0   r‰   Ú
ValueErrorÚattention_dropoutÚdeterministic_flash_attnr„   r   rU   Úattention_biasÚWqkvrv   Úsliding_windowÚ	is_causalr]   r7   ÚIdentityÚout_drop©r;   r&   rÍ   r<   s      €r=   r-   zModernBertAttention.__init__ð   sz  ø€ Ü‰ÑÔØˆŒØ"ˆŒà×Ñ × :Ñ :Ñ:¸aÒ?ÜØ# F×$6Ñ$6Ð#7Ð7mÐnt÷  oIñ  oIð  nJð  JKð  Lóð ð "(×!9Ñ!9ˆÔØ(.×(GÑ(GˆÔ%Ø×*Ñ*¨f×.HÑ.HÑHˆŒÜ—I‘IØ×Ñ  D§M¡MÑ 1°F×4NÑ4NÑ NÐU[×UjÑUjô
ˆŒ	ð ×Ñ˜iÑ(Ð,?Ò?ð #)×"7Ñ"7¸!Ñ";ˆDÕà"&ˆDÔàˆŒä—)‘)˜F×.Ñ.°×0BÑ0BÈ×I^ÑI^Ô_ˆŒØ@F×@XÑ@XÐ[^Ò@^œŸ
™
 6×#;Ñ#;Ó<ˆ�Ôdf×doÑdoÓdqˆ�r>   rD   Úposition_embeddingsr®   r¹   rA   c                 óÒ  — |j                   d d }| j                  |«      } |j                  g |¢d‘d‘| j                  ‘­Ž }|j	                  d¬«      \  }}}	|j                  dd«      }|j                  dd«      }|	j                  dd«      }	|\  }
}t        |||
|d¬«      \  }}t        j                  | j                  j                  t        «      } || |||	|f| j                  r| j                  nd| j                  d	z  | j                  | j                  d
œ|¤Ž\  }} |j                   g |¢d‘­Ž j#                  «       }| j%                  | j'                  |«      «      }||fS )Nr_   r	   éýÿÿÿr`   r"   rS   )rÆ   rÑ   ç      à¿)r°   r¯   r×   Údeterministic)r–   rÖ   Úviewr„   Úunbindrš   rÊ   r   Úget_interfacer&   Ú_attn_implementationr¼   r³   rÓ   r×   rÔ   Úreshaper¸   rÚ   r]   )r;   rD   rÜ   r®   r¹   Úinput_shapeÚqkvÚquery_statesÚ
key_statesÚvalue_statesrœ   r�   Úattention_interfacer»   rº   s                  r=   rE   zModernBertAttention.forward  s‚  € ð $×)Ñ)¨#¨2Ð.ˆà�i‰i˜Ó&ˆØˆc�h‰hÐ:˜Ð: QÐ:¨Ð:¨D¯M©MÒ:ˆØ14·±À°Ó1CÑ.ˆ�j ,à#×-Ñ-¨a°Ó3ˆØ×)Ñ)¨!¨QÓ/ˆ
Ø#×-Ñ-¨a°Ó3ˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÐjkÔ#lÑ ˆ�jä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð%
ð /3¯mªm�D×*Ò*ÀØ—M‘M 4Ñ'Ø×.Ñ.Ø×7Ñ7ñ%
ð ñ%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—m‘m D§G¡G¨KÓ$8Ó9ˆØ˜LÐ(Ð(r>   rC   rF   )rG   rH   rI   rJ   r#   rV   r-   rK   rM   r¨   r   r   rE   rN   rO   s   @r=   rÌ   rÌ   å   s¤   ø„ ññrÐ/ð r¸CÀ$¹Jõ rð@ IMØ.2ñ	')à—|‘|ð')ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð')ð Ÿ™ tÑ+ð	')ð
 Ð+Ñ,ð')ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷')r>   rÌ   c                   ó´   ‡ — e Zd Zddededz  fˆ fd„Z	 	 ddej                  dej                  dz  dej                  dz  dee	   d	ej                  f
d
„Z
ˆ xZS )ÚModernBertEncoderLayerNr&   rÍ   c                 óÔ  •— t         ‰| �  «        || _        || _        |dk(  rt	        j
                  «       | _        n;t	        j                  |j                  |j                  |j                  ¬«      | _        t        ||¬«      | _        t	        j                  |j                  |j                  |j                  ¬«      | _        t        |«      | _        |j                   |   | _        y )Nr   r)   )r&   rÍ   )r,   r-   r&   rÍ   r   rÙ   Ú	attn_normr3   r0   r4   r5   rÌ   ÚattnÚmlp_normrQ   Úmlprv   Úattention_typerÛ   s      €r=   r-   zModernBertEncoderLayer.__init__8  s§   ø€ Ü‰ÑÔØˆŒØ"ˆŒØ˜Š>ÜŸ[™[›]ˆD�NäŸ\™\¨&×*<Ñ*<À&Ç/Á/ÐX^×XhÑXhÔiˆDŒNÜ'¨vÀÔKˆŒ	ÜŸ™ V×%7Ñ%7¸V¿_¹_ÐSY×ScÑScÔdˆŒÜ  Ó(ˆŒØ$×0Ñ0°Ñ;ˆÕr>   rD   r®   rÜ   r¹   rA   c                 ó¦   —  | j                   | j                  |«      f||dœ|¤Ž\  }}||z   }|| j                  | j                  |«      «      z   }|S )N)rÜ   r®   )rð   rï   rò   rñ   )r;   rD   r®   rÜ   r¹   r»   Ú_s          r=   rE   zModernBertEncoderLayer.forwardE  sg   € ð #˜Ÿ™Ø�N‰N˜=Ó)ð
à 3Ø)ñ
ð ñ	
‰ˆ�Qð &¨Ñ3ˆØ%¨¯©°·±¸}Ó1MÓ(NÑNˆØÐr>   rC   rF   )rG   rH   rI   r#   rV   r-   rK   rM   r   r   rE   rN   rO   s   @r=   rí   rí   7  sx   ø„ ñ<Ð/ð <¸CÀ$¹Jõ <ð  /3Ø37ñ	à—|‘|ðð Ÿ™ tÑ+ðð #Ÿ\™\¨DÑ0ð	ð
 Ð+Ñ,ðð 
�‰÷r>   rí   c                   ó–   ‡ — e Zd ZU eed<   dZdZddgZdZdZ	dZ
dZeedœZ ej                   «       dej$                  fˆ fd„«       Zˆ xZS )	ÚModernBertPreTrainedModelr&   ÚmodelTr%   rí   )rD   Ú
attentionsrª   c                 óR  •‡— t         ‰	| �  |«       | j                  j                  Š‰€dŠdt        j
                  dt        fˆfd„}| j                  j                  | j                  j                  t        j                  d| j                  j                  z  «      z  | j                  j                  | j                  j                  dz  dœ}t        |t        «      r ||j                  |d   «       y t        |t        «      r- ||j                   |d	   «        ||j"                  |d
   «       y t        |t$        «      r- ||j&                  |d	   «        ||j"                  |d
   «       y t        |t(        «      r ||j*                  |d
   «       y t        |t,        «      r ||j.                  |d
   «       y t        |t0        t2        t4        t6        f«      r ||j8                  |d   «       y t        |t:        «      r£|j<                  D ]“  }|j>                  }|j@                  |   dk7  rtB        |j@                  |      } ||j                  |¬«      \  }}tE        jF                  tI        ||› d�«      |«       tE        jF                  tI        ||› d�«      |«       Œ• y y )Nr	   rª   Ústdc                 óê   •— t        j                  | j                  d|‰ |z  ‰|z  ¬«       t        | t        j
                  «      r-| j                  � t        j                  | j                  «       y y y )NrÑ   )Úmeanrû   ÚaÚb)ÚinitÚtrunc_normal_Úweightr—   r   rU   r+   Úzeros_)rª   rû   Úcutoff_factors     €r=   Úinit_weightz<ModernBertPreTrainedModel._init_weights.<locals>.init_weightn  sd   ø€ Ü×ÑØ—‘ØØØ �. 3Ñ&Ø #Ñ%õô ˜&¤"§)¡)Ô,Ø—;‘;Ð*Ü—K‘K §¡Õ,ð +ð -r>   g       @rß   )ÚinÚoutÚ	embeddingÚ	final_outr  r  r  r	  rj   rk   rm   ro   )%r,   Ú_init_weightsr&   Úinitializer_cutoff_factorr   ÚModuler�   Úinitializer_rangeÚmathÚsqrtÚnum_hidden_layersr0   r—   r%   r2   rQ   rY   r]   rÌ   rÖ   ÚModernBertPredictionHeadÚdenseÚModernBertForMaskedLMÚdecoderÚ#ModernBertForSequenceClassificationÚModernBertForMultipleChoiceÚ ModernBertForTokenClassificationÚModernBertForQuestionAnsweringÚ
classifierrf   rv   rx   ri   r   r   Úcopy_rˆ   )
r;   rª   r  Ústdsrl   r~   r   rõ   r  r<   s
           @€r=   r
  z'ModernBertPreTrainedModel._init_weightsg  s4  ù€ ä‰Ñ˜fÔ%ØŸ™×=Ñ=ˆØÐ ØˆMð	-¤§	¡	ð 	-´õ 	-ð —+‘+×/Ñ/Ø—;‘;×0Ñ0´4·9±9¸SÀ4Ç;Á;×C`ÑC`Ñ=`Ó3aÑaØŸ™×6Ñ6ØŸ™×0Ñ0°$Ñ6ñ	
ˆô �fÔ2Ô3Ù˜×-Ñ-¨t°KÑ/@ÕAÜ˜¤Ô.Ù˜Ÿ	™	 4¨¡:Ô.Ù˜Ÿ	™	 4¨¡;Õ/Ü˜Ô 3Ô4Ù˜Ÿ™ T¨$¡ZÔ0Ù˜Ÿ	™	 4¨¡;Õ/Ü˜Ô 8Ô9Ù˜Ÿ™ d¨5¡kÕ2Ü˜Ô 5Ô6Ù˜Ÿ™¨¨U©Õ4ÜØä3Ü+Ü0Ü.ð	ô
ñ ˜×)Ñ)¨4°Ñ+<Õ=Ü˜Ô 9Ô:Ø$×0Ñ0ò ^�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ò<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ô#UÑ �˜qÜ—
‘
œ7 6¨j¨\¸Ð+CÓDÀmÔTÜ—
‘
œ7 6¨j¨\Ð9KÐ+LÓMÈ}Õ]ñ^ð ;r>   )rG   rH   rI   r#   r¦   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendrí   rÌ   Ú_can_record_outputsrK   r©   r   r  r
  rN   rO   s   @r=   r÷   r÷   W  sr   ø… àÓØÐØ&*Ð#Ø/Ð1IÐJÐØÐØ€NØÐØ"&Ðð 0Ø)ñÐð
 €U‡]�]ƒ_ð7^ B§I¡Iô 7^ó ô7^r>   r÷   c                   óæ   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zeee		 	 	 	 dde
j                  dz  de
j                  dz  de
j                  dz  d	e
j                  dz  d
ee   defd„«       «       «       Zˆ xZS )ÚModernBertModelr&   c           	      ó¬  •— t         ‰| �  |«       || _        t        |«      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _
        t        j                  |j                  |j                  |j                  ¬«      | _        t!        |¬«      | _        d| _        | j'                  «        y c c}w )Nr)   )r&   F)r,   r-   r&   r%   Ú
embeddingsr   Ú
ModuleListÚranger  rí   Úlayersr3   r0   r4   r5   Ú
final_normrf   Ú
rotary_embÚgradient_checkpointingÚ	post_initrÛ   s      €r=   r-   zModernBertModel.__init__¤  sŸ   ø€ Ü‰Ñ˜Ô ØˆŒÜ.¨vÓ6ˆŒÜ—m‘mÜHMÈf×NfÑNfÓHgÖh¸9Ô# F¨IÕ6Òhó
ˆŒô Ÿ,™, v×'9Ñ'9¸v¿¹ÐU[×UeÑUeÔfˆŒÜ3¸6ÔBˆŒØ&+ˆÔ#Ø�‰Õùò is   ÁCc                 ó.   — | j                   j                  S rC   ©r'  r2   ©r;   s    r=   Úget_input_embeddingsz$ModernBertModel.get_input_embeddings°  s   € Ø�‰×-Ñ-Ð-r>   c                 ó&   — || j                   _        y rC   r0  )r;   r­   s     r=   Úset_input_embeddingsz$ModernBertModel.set_input_embeddings³  s   € Ø).ˆ�‰Õ&r>   Nr?   r®   rŸ   r@   r¹   rA   c                 ó¦  — |d u |d uz  rt        d«      ‚|�|j                  d   n|j                  d   }|�|j                  n|j                  }|€&t        j                  ||¬«      j                  d«      }| j                  ||¬«      }t        |x}	t        «      s'| j                  ||dœ}
t        d
i |
¤Žt        d
i |
¤Ždœ}	i }t        | j                  j                  «      D ]  }| j                  |||«      ||<   Œ | j                  D ](  } ||f|	|j                      ||j                      dœ|¤Ž}Œ* | j#                  |«      }t%        |¬	«      S )Nz:You must specify exactly one of input_ids or inputs_embedsr"   ©r|   r   )r?   r@   )r&   r@   r®   )Úfull_attentionrÐ   )r®   rÜ   )Úlast_hidden_state© )rÒ   r–   r|   rK   rŠ   rÃ   r'  r—   Údictr&   r   r   ru   rv   r,  r*  ró   r+  r   )r;   r?   r®   rŸ   r@   r¹   r�   r|   rD   Úattention_mask_mappingÚmask_kwargsrÜ   rl   Úencoder_layers                 r=   rE   zModernBertModel.forward¶  s|  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZà,9Ð,E�-×%Ñ% aÒ(È9Ï?É?Ð[\ÑK]ˆØ%.Ð%:�×!Ò!À×@TÑ@TˆàÐÜ Ÿ<™<¨¸Ô?×IÑIÈ!ÓLˆLàŸ™°)È=˜ÓYˆä°NÐBÐ0ÄDÔIàŸ+™+Ø!.Ø"0ñˆKô #<Ñ"J¸kÑ"JÜ%MÑ%\ÐP[Ñ%\ñ&Ð"ð
 !ÐÜ˜dŸk™k×5Ñ5Ó6ò 	gˆJØ.2¯o©o¸mÈ\Ð[eÓ.fÐ 
Ò+ð	gð "Ÿ[™[ò 	ˆMÙ)Øðà5°m×6RÑ6RÑSØ$7¸×8TÑ8TÑ$Uñð ñ	‰Mð	ð Ÿ™¨Ó6ˆä°Ô?Ð?r>   r¥   )rG   rH   rI   r#   r-   r2  r4  r    r!   r   rK   rL   rM   r   r   r   rE   rN   rO   s   @r=   r%  r%  ¢  s¼   ø„ ð
Ð/õ 
ò.ò/ð  ØØð .2Ø.2Ø04Ø-1ñ,@à×#Ñ# dÑ*ð,@ð Ÿ™ tÑ+ð,@ð ×&Ñ&¨Ñ-ð	,@ð
 —|‘| dÑ*ð,@ð Ð+Ñ,ð,@ð 
ò,@ó ó ó  ô,@r>   r%  c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )r  r&   c                 óJ  •— t         ‰| �  «        || _        t        j                  |j
                  |j
                  |j                  «      | _        t        |j                     | _
        t        j                  |j
                  |j                  |j                  ¬«      | _        y )Nr)   )r,   r-   r&   r   rU   r0   Úclassifier_biasr  r   Úclassifier_activationr[   r3   r4   r5   r6   r:   s     €r=   r-   z!ModernBertPredictionHead.__init__é  sq   ø€ Ü‰ÑÔØˆŒÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÀv×G]ÑG]Ó^ˆŒ
Ü˜&×6Ñ6Ñ7ˆŒÜ—L‘L ×!3Ñ!3¸¿¹Èv×O_ÑO_Ô`ˆ�	r>   rD   rA   c                 ó`   — | j                  | j                  | j                  |«      «      «      S rC   )r6   r[   r  )r;   rD   s     r=   rE   z ModernBertPredictionHead.forwardð  s#   € Ø�y‰y˜Ÿ™ $§*¡*¨]Ó";Ó<Ó=Ð=r>   )	rG   rH   rI   r#   r-   rK   rM   rE   rN   rO   s   @r=   r  r  è  s-   ø„ ðaÐ/õ að> U§\¡\ð >°e·l±l÷ >r>   r  zd
    The ModernBert Model with a decoder head on top that is used for masked language modeling.
    )Úcustom_introc                   ó>  ‡ — e Zd ZddiZdefˆ fd„Zd„ Zdej                  fd„Z	e
e	 	 	 	 	 dd	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dee   deej                     ez  fd„«       «       Zˆ xZS )r  zdecoder.weightz&model.embeddings.tok_embeddings.weightr&   c                 ót  •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        t        j                  |j                  |j                  |j                  ¬«      | _        | j                  j                  | _        | j                  j                  | _        | j                  «        y )NrT   )r,   r-   r&   r%  rø   r  Úheadr   rU   r0   r/   Údecoder_biasr  Úsparse_predictionÚsparse_pred_ignore_indexr.  r:   s     €r=   r-   zModernBertForMaskedLM.__init__ü  s…   ø€ Ü‰Ñ˜Ô ØˆŒÜ$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—y‘y ×!3Ñ!3°V×5FÑ5FÈV×M`ÑM`ÔaˆŒà!%§¡×!>Ñ!>ˆÔØ(,¯©×(LÑ(LˆÔ%ð 	�‰Õr>   c                 ó   — | j                   S rC   ©r  r1  s    r=   Úget_output_embeddingsz+ModernBertForMaskedLM.get_output_embeddings	  s   € Ø�|‰|Ðr>   Únew_embeddingsc                 ó   — || _         y rC   rK  )r;   rM  s     r=   Úset_output_embeddingsz+ModernBertForMaskedLM.set_output_embeddings  s	   € Ø%ˆ�r>   Nr?   r®   rŸ   r@   Úlabelsr¹   rA   c                 óÊ  —  | j                   d||||dœ|¤Ž}|d   }| j                  rK|�I|j                  d«      }|j                  |j                  d   d«      }|| j                  k7  }	||	   }||	   }| j                  | j                  |«      «      }
d }|�* | j                  |
|fd| j                  j                  i|¤Ž}t        ||
|j                  |j                  ¬«      S )N©r?   r®   rŸ   r@   r   r_   r/   ©ÚlossÚlogitsrD   rù   r9  )rø   rH  rá   r–   rI  r  rF  Úloss_functionr&   r/   r   rD   rù   )r;   r?   r®   rŸ   r@   rP  r¹   Úoutputsr8  Úmask_tokensrU  rT  s               r=   rE   zModernBertForMaskedLM.forward  s	  € ð �$—*‘*ð 
ØØ)Ø%Ø'ñ	
ð
 ñ
ˆð $ A™JÐà×!Ò! fÐ&8à—[‘[ “_ˆFØ 1× 6Ñ 6°v·|±|ÀA±ÈÓ KÐð ! D×$AÑ$AÑAˆKØ 1°+Ñ >ÐØ˜KÑ(ˆFà—‘˜dŸi™iÐ(9Ó:Ó;ˆàˆØÐØ%�4×%Ñ% f¨fÑbÀÇÁ×AWÑAWÐbÐ[aÑbˆDäØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r>   ©NNNNN)rG   rH   rI   Ú_tied_weights_keysr#   r-   rL  r   rU   rO  r   r   rK   rL   rM   r   r   r¨   r   rE   rN   rO   s   @r=   r  r  ô  sç   ø„ ð +Ð,TÐUÐðÐ/õ òð&°B·I±Ió &ð Øð .2Ø.2Ø,0Ø-1Ø&*ñ'
à×#Ñ# dÑ*ð'
ð Ÿ™ tÑ+ð'
ð —l‘l TÑ)ð	'
ð
 —|‘| dÑ*ð'
ð —‘˜tÑ#ð'
ð Ð+Ñ,ð'
ð 
ˆu�|‰|Ñ	˜~Ñ	-ò'
ó ó ô'
r>   r  z`
    The ModernBert Model with a sequence classification head on top that performs pooling.
    c                   ó  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e
e   d
eej                     ez  fd„«       «       Zˆ xZS )r  r&   c                 ón  •— t         ‰| �  |«       |j                  | _        || _        t	        |«      | _        t        |«      | _        t        j                  j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j!                  «        y rC   )r,   r-   Ú
num_labelsr&   r%  rø   r  rF  rK   r   r7   Úclassifier_dropoutr9   rU   r0   r  r.  r:   s     €r=   r-   z,ModernBertForSequenceClassification.__init__A  s‚   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØˆŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr>   Nr?   r®   rŸ   r@   rP  r¹   rA   c                 ód  —  | j                   d||||dœ|¤Ž}|d   }| j                  j                  dk(  r
|dd…df   }n�| j                  j                  dk(  rw|€=t        j                  |j
                  dd |j                  t        j                  ¬«      }||j                  d«      z  j                  d	¬
«      |j                  d	d¬«      z  }| j                  |«      }	| j                  |	«      }	| j                  |	«      }
d}|��‡| j                  j                  €�| j                  d	k(  rd| j                  _        nl| j                  d	kD  rL|j                  t        j                   k(  s|j                  t        j"                  k(  rd| j                  _        nd| j                  _        | j                  j                  dk(  rIt%        «       }| j                  d	k(  r& ||
j'                  «       |j'                  «       «      }nŒ ||
|«      }n‚| j                  j                  dk(  r=t)        «       } ||
j+                  d| j                  «      |j+                  d«      «      }n,| j                  j                  dk(  rt-        «       } ||
|«      }t/        ||
|j0                  |j2                  ¬«      S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        rR  r   ÚclsNrý   rS   r‡   r_   r"   r`   T©ra   ÚkeepdimÚ
regressionÚsingle_label_classificationÚmulti_label_classificationrS  r9  )rø   r&   Úclassifier_poolingrK   Úonesr–   r|   ÚboolrÃ   ÚsumrF  r9   r  Úproblem_typer]  r†   ÚlongrV   r   Úsqueezer   rá   r   r   rD   rù   )r;   r?   r®   rŸ   r@   rP  r¹   rW  r8  Úpooled_outputrU  rT  Úloss_fcts                r=   rE   z+ModernBertForSequenceClassification.forwardN  se  € ð" �$—*‘*ð 
ØØ)Ø%Ø'ñ	
ð
 ñ
ˆð $ A™JÐà�;‰;×)Ñ)¨UÒ2Ø 1²!°Q°$Ñ 7ÑØ�[‰[×+Ñ+¨vÒ5ØÐ%Ü!&§¡Ø%×+Ñ+¨B¨QÐ/Ð8I×8PÑ8PÔX]×XbÑXbô"�ð "3°^×5MÑ5MÈbÓ5QÑ!Q× VÑ VÐ[\Ð VÓ ]Ð`n×`rÑ`rØ˜tð asó añ !Ðð Ÿ	™	Ð"3Ó4ˆØŸ	™	 -Ó0ˆØ—‘ Ó/ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r>   rY  )rG   rH   rI   r#   r-   r   r   rK   rL   rM   r   r   r¨   r   rE   rN   rO   s   @r=   r  r  ;  sÎ   ø„ ðÐ/õ ð Øð .2Ø.2Ø,0Ø-1Ø&*ñC
à×#Ñ# dÑ*ðC
ð Ÿ™ tÑ+ðC
ð —l‘l TÑ)ð	C
ð
 —|‘| dÑ*ðC
ð —‘˜tÑ#ðC
ð Ð+Ñ,ðC
ð 
ˆu�|‰|Ñ	Ð7Ñ	7òC
ó ó ôC
r>   r  zv
    The ModernBert Model with a token classification head on top, e.g. for Named Entity Recognition (NER) tasks.
    c                   ó  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e
e   d
eej                     ez  fd„«       «       Zˆ xZS )r  r&   c                 ó`  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        |«      | _        t        j                  j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j                  «        y rC   ©r,   r-   r]  r%  rø   r  rF  rK   r   r7   r^  r9   rU   r0   r  r.  r:   s     €r=   r-   z)ModernBertForTokenClassification.__init__œ  s{   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr>   Nr?   r®   rŸ   r@   rP  r¹   rA   c                 óf  —  | j                   d||||dœ|¤Ž}|d   }| j                  |«      }| j                  |«      }| j                  |«      }	d}
|�<t	        «       } ||	j                  d| j                  «      |j                  d«      «      }
t        |
|	|j                  |j                  ¬«      S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        rR  r   Nr_   rS  r9  )
rø   rF  r9   r  r   rá   r]  r   rD   rù   )r;   r?   r®   rŸ   r@   rP  r¹   rW  r8  rU  rT  rn  s               r=   rE   z(ModernBertForTokenClassification.forward¨  sÅ   € ð �$—*‘*ð 
ØØ)Ø%Ø'ñ	
ð
 ñ
ˆð $ A™JÐà ŸI™IÐ&7Ó8ÐØ ŸI™IÐ&7Ó8ÐØ—‘Ð!2Ó3ˆàˆØÐÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r>   rY  )rG   rH   rI   r#   r-   r   r   rK   rL   rM   r   r   r¨   r   rE   rN   rO   s   @r=   r  r  –  sÅ   ø„ ð
Ð/õ 
ð Øð .2Ø.2Ø,0Ø-1Ø&*ñ$
à×#Ñ# dÑ*ð$
ð Ÿ™ tÑ+ð$
ð —l‘l TÑ)ð	$
ð
 —|‘| dÑ*ð$
ð —‘˜tÑ#ð$
ð Ð+Ñ,ð$
ð 
ˆu�|‰|Ñ	Ð4Ñ	4ò$
ó ó ô$
r>   r  c                   ó  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
   d
eej                     ez  fd„«       «       Zˆ xZS )r  r&   c                 ó`  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        |«      | _        t        j                  j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j                  «        y rC   rq  r:   s     €r=   r-   z'ModernBertForQuestionAnswering.__init__Ó  sy   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒà�‰Õr>   Nr?   r®   rŸ   Ústart_positionsÚend_positionsr¹   rA   c                 óÆ  —  | j                   |f||dœ|¤Ž}|d   }| j                  |«      }| j                  |«      }| j                  |«      }	|	j	                  dd¬«      \  }
}|
j                  d«      j                  «       }
|j                  d«      j                  «       }d }|�|� | j                  |
|||fi |¤Ž}t        ||
||j                  |j                  ¬«      S )N)r®   rŸ   r   r"   r_   r`   )rT  Ústart_logitsÚ
end_logitsrD   rù   )rø   rF  r9   r  Úsplitrl  r¸   rV  r   rD   rù   )r;   r?   r®   rŸ   ru  rv  r¹   rW  r8  rU  rx  ry  rT  s                r=   rE   z&ModernBertForQuestionAnswering.forwardÞ  s  € ð �$—*‘*Øð
à)Ø%ñ
ð ñ	
ˆð $ A™JÐà ŸI™IÐ&7Ó8ÐØ ŸI™IÐ&7Ó8ÐØ—‘Ð!2Ó3ˆà#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆØÐ&¨=Ð+DØ%�4×%Ñ% l°JÀÐQ^ÑiÐbhÑiˆDä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r>   rY  )rG   rH   rI   r#   r-   r   r   rK   rM   r   r   r¨   r   rE   rN   rO   s   @r=   r  r  Ñ  sÃ   ø„ ð	Ð/õ 	ð Øð *.Ø.2Ø,0Ø/3Ø-1ñ#
à—<‘< $Ñ&ð#
ð Ÿ™ tÑ+ð#
ð —l‘l TÑ)ð	#
ð
 Ÿ™¨Ñ,ð#
ð —|‘| dÑ*ð#
ð Ð+Ñ,ð#
ð 
ˆu�|‰|Ñ	Ð;Ñ	;ò#
ó ó ô#
r>   r  z«
    The ModernBert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks.
    c                   ó  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e
e   d
eej                     ez  fd„«       «       Zˆ xZS )r  r&   c                 ó8  •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        t        j                  j                  |j                  «      | _        t        j                  |j                  d«      | _        | j                  «        y )Nr"   )r,   r-   r&   r%  rø   r  rF  rK   r   r7   r^  r9   rU   r0   r  r.  r:   s     €r=   r-   z$ModernBertForMultipleChoice.__init__  so   ø€ Ü‰Ñ˜Ô ØˆŒä$ VÓ,ˆŒ
Ü,¨VÓ4ˆŒ	Ü—H‘H×$Ñ$ V×%>Ñ%>Ó?ˆŒ	ÜŸ)™) F×$6Ñ$6¸Ó:ˆŒð 	�‰Õr>   Nr?   r®   rŸ   r@   rP  r¹   rA   c                 óÔ  — |�|j                   d   n|j                   d   }|�!|j                  d|j                  d«      «      nd}|�!|j                  d|j                  d«      «      nd}|�!|j                  d|j                  d«      «      nd}|�1|j                  d|j                  d«      |j                  d«      «      nd} | j                  d||||dœ|¤Ž}|d   }	| j                  j
                  dk(  r”t        j                  |	j                   d   |	j                  ¬«      }
|�,|j                  d¬	«      j                  |	j                  «      }n0t        j                  dt        j                  |	j                  ¬
«      }|	|
|f   }	nS| j                  j
                  dk(  r:|j                  dd¬«      }|	|j                  d«      z  j                  d¬	«      |z  }	| j                  |	«      }| j!                  |«      }| j#                  |«      }|j                  d|«      }d}|�t%        j&                  «       } |||«      }t)        |||j*                  |j,                  ¬«      S )a&  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors.
        Nr"   r_   éþÿÿÿrR  r   r`  r6  r`   )r†   r|   rý   Tra  rS  r9  )r–   rá   Úsizerø   r&   rf  rK   rŠ   r|   ÚargmaxrŒ   Útensorrk  ri  rÃ   rF  r9   r  r   r   r   rD   rù   )r;   r?   r®   rŸ   r@   rP  r¹   Únum_choicesrW  r8  Ú	indices_0Úcls_maskÚnum_non_pad_tokensrm  rU  Úreshaped_logitsrT  rn  s                     r=   rE   z#ModernBertForMultipleChoice.forward  sn  € ð  -6Ð,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆà>GÐ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÐG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ð(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð �$—*‘*ð 
ØØ)Ø%Ø'ñ	
ð
 ñ
ˆð $ A™JÐð �;‰;×)Ñ)¨UÒ2ÜŸ™Ð%6×%<Ñ%<¸QÑ%?ÐHY×H`ÑH`ÔaˆIàÐ)Ø)×0Ñ0°RÐ0Ó8×;Ñ;Ð<M×<TÑ<TÓU‘ô !Ÿ<™<¨´·±ÐDU×D\ÑD\Ô]�à 1°)¸XÐ2EÑ FÑð �[‰[×+Ñ+¨vÒ5Ø!/×!3Ñ!3¸À4Ð!3Ó!HÐØ!2°^×5MÑ5MÈbÓ5QÑ!Q× VÑ VÐ[\Ð VÓ ]Ð`rÑ rÐàŸ	™	Ð"3Ó4ˆØŸ	™	 -Ó0ˆØ—‘ Ó/ˆà Ÿ+™+ b¨+Ó6ˆàˆØÐÜ×*Ñ*Ó,ˆHÙ˜O¨VÓ4ˆDä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r>   rY  )rG   rH   rI   r#   r-   r   r   rK   rL   rM   r   r   r¨   r   rE   rN   rO   s   @r=   r  r    sÎ   ø„ ð
Ð/õ 
ð Øð .2Ø.2Ø,0Ø-1Ø&*ñC
à×#Ñ# dÑ*ðC
ð Ÿ™ tÑ+ðC
ð —l‘l TÑ)ð	C
ð
 —|‘| dÑ*ðC
ð —‘˜tÑ#ðC
ð Ð+Ñ,ðC
ð 
ˆu�|‰|Ñ	Ð8Ñ	8òC
ó ó ôC
r>   r  )r%  r÷   r  r  r  r  r  )rÑ   )r"   )Fr  Úcollections.abcr   Útypingr   rK   r   Útorch.nnr   r   r   Ú r
   r   Úactivationsr   Úintegrationsr   r   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.genericr   r   r    Úutils.output_capturingr!   Úconfiguration_modernbertr#   r  r%   rQ   rf   rM   r�   r¼   rÀ   rÊ   rÌ   rí   r÷   r%  r  r  r  r  r  r  Ú__all__r9  r>   r=   ú<module>r˜     s[  ðó, Ý $Ý ã Ý ß AÑ Aå &Ý !ß Iß `Ý 9÷÷ ÷ Lß FÝ &ß 7ß YÑ YÝ 5Ý 6ô˜2Ÿ9™9ô ô,:�B—I‘Iô :ô(L< §	¡	ô L<ðl ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ó%ò,(ñ Ð*Ó+òBó ,ðBñ4 Ð)Ó*ôN)˜"Ÿ)™)ó N)ó +ðN)ôbÐ7ô ð@ ôG^ ó G^ó ðG^ðT ôB@Ð/ó B@ó ðB@ôJ	>˜rŸy™yô 	>ñ ðôô
?
Ð5ó ?
óð
?
ñD ðôô
S
Ð*Có S
óð
S
ñl ðôô
3
Ð'@ó 3
óð
3
ðl ô1
Ð%>ó 1
ó ð1
ñh ðôô
R
Ð";ó R
óð
R
òj�r>   