
    ^j                        d dl Z d dlmZ d dlmZmZ d dlZd dlmZ d dlm	Z	 d dl
mZmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZmZ ddlmZ ddlmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z,m-Z-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5 ddl6m7Z7m8Z8  e.jr                  e:      Z; e-d      e G d de                    Z< G d de	jz                        Z> G d de	jz                        Z? G d  d!e7      Z@ ed"      d>d#       ZA eeA       G d$ d%e	jz                               ZB G d& d'e      ZCe- G d( d)e(             ZDe- G d* d+eD             ZE G d, d-e	jz                        ZF e-d./       G d0 d1eD             ZG e-d2/       G d3 d4eD             ZH e-d5/       G d6 d7eD             ZIe- G d8 d9eD             ZJ e-d:/       G d; d<eD             ZKg d=ZLy)?    N)Callable)LiteralOptional)strict)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )initialization)ACT2FN)PreTrainedConfig)use_kernel_func_from_hubuse_kernelized_func)create_bidirectional_mask(create_bidirectional_sliding_window_mask)GradientCheckpointingLayer)BaseModelOutputMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)ROPE_INIT_FUNCTIONS)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringlogging)can_return_tuplemerge_with_config_defaults)capture_outputs   )eager_attention_forward)Gemma3RotaryEmbeddingrotate_halfzanswerdotai/ModernBERT-base)
checkpointc                       e Zd ZU dZdZdgZdddZdZee	d<   d	Z
ee	d
<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZedz  e	d<   d Zeee   z  dz  e	d!<   d"Zedz  e	d#<   d"Zedz  e	d$<   d Zedz  e	d%<   dZee	d&<   d'Zeez  e	d(<   dZee   dz  e	d)<   dZ e!e"d*   e!f   dz  e	d+<   d,Z#ee	d-<   d'Z$eez  e	d.<   dZ%ee	d/<   d'Z&eez  e	d0<   d1Z'ee	d2<   d3Z(e"d4   e	d5<   d'Z)eez  e	d6<   dZ*ee	d7<   dZ+ee	d8<   dZ,ee	d9<   dZ-ee	d:<   d;Z.ee	d<<   d1Z/ee	d=<    fd>Z0d? Z1 fd@Z2e3dA        Z4e4jj                  dB        Z4 xZ6S )CModernBertConfiga+  
    initializer_cutoff_factor (`float`, *optional*, defaults to 2.0):
        The cutoff factor for the truncated_normal_initializer for initializing all weight matrices.
    norm_eps (`float`, *optional*, defaults to 1e-05):
        The epsilon used by the rms normalization layers.
    norm_bias (`bool`, *optional*, defaults to `False`):
        Whether to use bias in the normalization layers.
    local_attention (`int`, *optional*, defaults to 128):
        The window size for local attention.
    mlp_dropout (`float`, *optional*, defaults to 0.0):
        The dropout ratio for the MLP layers.
    decoder_bias (`bool`, *optional*, defaults to `True`):
        Whether to use bias in the decoder layers.
    classifier_pooling (`str`, *optional*, defaults to `"cls"`):
        The pooling method for the classifier. Should be either `"cls"` or `"mean"`. In local attention layers, the
        CLS token doesn't attend to all tokens on long sequences.
    classifier_bias (`bool`, *optional*, defaults to `False`):
        Whether to use bias in the classifier.
    classifier_activation (`str`, *optional*, defaults to `"gelu"`):
        The activation function for the classifier.
    deterministic_flash_attn (`bool`, *optional*, defaults to `False`):
        Whether to use deterministic flash attention. If `False`, inference will be faster but not deterministic.
    sparse_prediction (`bool`, *optional*, defaults to `False`):
        Whether to use sparse prediction for the masked language model instead of returning the full dense logits.
    sparse_pred_ignore_index (`int`, *optional*, defaults to -100):
        The index to ignore for the sparse prediction.

    Examples:

    ```python
    >>> from transformers import ModernBertModel, ModernBertConfig

    >>> # Initializing a ModernBert style configuration
    >>> configuration = ModernBertConfig()

    >>> # Initializing a model from the modernbert-base style configuration
    >>> model = ModernBertModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
modernbertpast_key_valuesg     Ag     @)globallocali  
vocab_sizei   hidden_sizei  intermediate_size   num_hidden_layers   num_attention_headsgeluhidden_activationi    max_position_embeddingsg{Gz?initializer_range       @initializer_cutoff_factorgh㈵>norm_epsF	norm_biasik  Npad_token_idij  eos_token_idii  bos_token_idcls_token_idsep_token_idattention_bias        attention_dropoutlayer_typesfull_attentionsliding_attentionrope_parameters   local_attentionembedding_dropoutmlp_biasmlp_dropoutTdecoder_biascls)rQ   meanclassifier_poolingclassifier_dropoutclassifier_biasclassifier_activationdeterministic_flash_attnsparse_predictionisparse_pred_ignore_indextie_word_embeddingsc                     |j                  dd      }| j                  8t        | j                        D cg c]  }t	        ||z        rdnd c}| _        t        |   di | y c c}w )Nglobal_attn_every_n_layersr   rI   rH    )getrF   ranger3   boolsuper__post_init__)selfkwargsr\   i	__class__s       |/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/modernbert/modular_modernbert.pyrb   zModernBertConfig.__post_init__   st    %+ZZ0La%P"# t556  (,A0J,J'K#Qaa D
 	'' s   A(c                    |j                  dd       }ddiddid}| j                  | j                  n|| _        |<| j                  d   j                  |       | j                  d   j                  |       | j                  j                  d      ddi| j                  d<   | j                  d   j	                  d|j                  d| j
                  d	                | j                  j                  d      ddi| j                  d<   | j                  d   j	                  d|j                  d
| j
                  d                | j                          |S )Nrope_scaling	rope_typedefault)rI   rH   rH   rI   
rope_thetaglobal_rope_thetar-   local_rope_thetar.   )poprJ   updater^   
setdefaultdefault_thetastandardize_rope_params)rc   rd   ri   default_rope_paramss       rg   convert_rope_params_to_dictz,ModernBertConfig.convert_rope_params_to_dict   sc   zz.$7
 #.y!9*I6
 8<7K7K7Wt33]p#  !1299,G  !45<<\J ##$45=6A95MD  !12-.99&**%8$:L:LX:VW	
 ##$78@9Di8PD  !4501<<&**%79K9KG9TU	

 	$$&    c                 H    t         |          }|j                  dd        |S )Nreference_compile)ra   to_dictro   )rc   outputrf   s     rg   ry   zModernBertConfig.to_dict   s#    "

&-rv   c                      | j                   dz  S )zKHalf-window size: `local_attention` is the total window, so we divide by 2.r$   rL   rc   s    rg   sliding_windowzModernBertConfig.sliding_window   s     ##q((rv   c                     |dz  | _         y)z<Set sliding_window by updating local_attention to 2 * value.r$   Nr|   rc   values     rg   r~   zModernBertConfig.sliding_window   s      %qyrv   )7__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencerr   r/   int__annotations__r0   r1   r3   r5   r7   strr8   r9   floatr;   r<   r=   r`   r>   r?   listr@   rA   rB   rC   rE   rF   rJ   dictr   rL   rM   rN   rO   rP   rS   rT   rU   rV   rW   rX   rY   rZ   rb   ru   ry   propertyr~   setter__classcell__rf   s   @rg   r*   r*   4   s   (T J#4"5(8<MJK!s!s!!#s##'S'#u#'*u*HeIt$L#*$+0L#S	/D(0$L#*$$L#*$$L#*$ ND %(us{($(KcT!(Y]OT'"GH$NORVV]OS%(us{(Hd"K"L$16.6&))!OT!!'3'%*d*#t#$(c( $$	(<
 ) ) ) )rv   r*   c                        e Zd ZdZdef fdZ	 d	dej                  dz  dej                  dz  dej                  fdZ	 xZ
S )
ModernBertEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    configc                 d   t         |           || _        t        j                  |j
                  |j                  |j                        | _        t        j                  |j                  |j                  |j                        | _        t        j                  |j                        | _        y )N)padding_idxepsbias)ra   __init__r   r   	Embeddingr/   r0   r>   tok_embeddings	LayerNormr<   r=   normDropoutrM   droprc   r   rf   s     rg   r   zModernBertEmbeddings.__init__   sw     ll6+<+<f>P>P^d^q^qrLL!3!3vO_O_`	JJv778	rv   N	input_idsinputs_embedsreturnc                     |"| j                  | j                  |            }|S | j                  | j                  | j                  |                  }|S N)r   r   r   )rc   r   r   hidden_statess       rg   forwardzModernBertEmbeddings.forward   sS     $ IIdii&>?M  !IIdii0C0CI0N&OPMrv   NN)r   r   r   r   r*   r   torch
LongTensorTensorr   r   r   s   @rg   r   r      sR    9/ 9 _c))D0HMW[H[	rv   r   c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ xZ	S )ModernBertMLPa6  Applies the GLU at the end of each ModernBERT layer.

    Compared to the default BERT architecture, this block replaces :class:`~transformers.model.bert.modeling_bert.BertIntermediate`
    and :class:`~transformers.model.bert.modeling_bert.SelfOutput` with a single module that has similar functionality.
    r   c                    t         |           || _        t        j                  |j
                  t        |j                        dz  |j                        | _	        t        |j                     | _        t        j                  |j                        | _        t        j                  |j                  |j
                  |j                        | _        y )Nr$   r   )ra   r   r   r   Linearr0   r   r1   rN   Wir   r7   actr   rO   r   Wor   s     rg   r   zModernBertMLP.__init__   s    ))F..F4L4L0MPQ0QX^XgXgh&223JJv112	))F44f6H6Hv_rv   r   r   c                     | j                  |      j                  dd      \  }}| j                  | j                  | j	                  |      |z              S )Nr$   dim)r   chunkr   r   r   )rc   r   inputgates       rg   r   zModernBertMLP.forward   sI    ggm,221"2=twwtyy%4!7899rv   )
r   r   r   r   r*   r   r   r   r   r   r   s   @rg   r   r      s2    `/ `:U\\ :ell :rv   r   c                   |     e Zd Zddef fdZe	 	 	 	 ddedz  ded   dedz  dedz  de	d	e
f   f
 fd
       Z xZS )ModernBertRotaryEmbeddingNr   c                 &    t         |   ||       y r   )ra   r   )rc   r   devicerf   s      rg   r   z"ModernBertRotaryEmbedding.__init__   s    (rv   r   ztorch.deviceseq_len
layer_typer   ztorch.Tensorc                 (    t         |   | |||      S r   )ra   compute_default_rope_parameters)r   r   r   r   rf   s       rg   r   z9ModernBertRotaryEmbedding.compute_default_rope_parameters   s     w6vvwPZ[[rv   r   NNNN)r   r   r   r*   r   staticmethodr   r   r   tupler   r   r   r   s   @rg   r   r      s    )/ ) *.+/"!%	\ 4'\(\ t\ $J	\
 
~u$	%\ \rv   r   rotary_pos_embc                 b   | j                   }|j                  |      }|j                  |      }| j                         |z  t        | j                               |z  z   }|j                         |z  t        |j                               |z  z   }|j	                  |      |j	                  |      fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )dtype	unsqueezer   r'   to)qkcossinunsqueeze_dimoriginal_dtypeq_embedk_embeds           rg   apply_rotary_pos_embr      s    & WWN
--
&C
--
&Cwwy3;qwwy#9C#?@Gwwy3;qwwy#9C#?@G::n%wzz.'AAArv   c                        e Zd ZdZddededz  f fdZ	 	 ddej                  de	ej                  ej                  f   dz  dej                  dz  d	e
e   d
e	ej                  ej                  dz  f   f
dZ xZS )ModernBertAttentiona  Performs multi-headed self attention on a batch of unpadded sequences.

    If Flash Attention 2 is installed, this module uses Flash Attention to improve throughput.
    If Flash Attention 2 is not installed, the implementation will use PyTorch's SDPA kernel,
    which requires padding and unpadding inputs, adding some overhead.

    See `forward` method for additional details.
    Nr   	layer_idxc                 R   t         |           || _        || _        |j                  |j
                  z  dk7  r&t        d|j                   d|j
                   d      |j                  | _        |j                  | _        |j                  |j
                  z  | _	        t        j                  |j                  d| j                  z  |j
                  z  |j                        | _        |j                  |   dk(  r|j                  dz   | _        nd | _        d	| _        t        j                  |j                  |j                  |j                        | _        |j                  d
kD  r%t        j$                  |j                        | _        y t        j&                         | _        y )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads ()r   r   rI      FrD   )ra   r   r   r   r0   r5   
ValueErrorrE   rW   head_dimr   r   rC   WqkvrF   r~   	is_causalr   r   Identityout_droprc   r   r   rf   s      rg   r   zModernBertAttention.__init__  sz   " : ::a?#F$6$6#77mnt  oI  oI  nJ  JK  L  "(!9!9(.(G(G%**f.H.HHIIDMM 1F4N4N NU[UjUj
	 i(,?? #)"7"7!";D"&D))F..0B0BI^I^_@F@X@X[^@^

6#;#;<dfdododqrv   r   position_embeddingsattention_maskrd   r   c                    |j                   d d }| j                  |      } |j                  g |dd| j                   }|j	                  d      \  }}}	|j                  dd      }|j                  dd      }|	j                  dd      }	|\  }
}t        |||
|d      \  }}t        j                  | j                  j                  t              } || |||	|f| j                  r| j                  nd| j                  d	z  | j                  | j                  d
|\  }} |j                   g |d j#                         }| j%                  | j'                  |            }||fS )Nr   r   r   r   r$   )r   rD         )dropoutscalingr~   deterministic)shaper   viewr   unbind	transposer   r   get_interfacer   _attn_implementationr%   trainingrE   r~   rW   reshape
contiguousr   r   )rc   r   r   r   rd   input_shapeqkvquery_states
key_statesvalue_statesr   r   attention_interfaceattn_outputattn_weightss                  rg   r   zModernBertAttention.forward;  s    $))#2.ii&chh::Q::DMM:141C.j,#--a3))!Q/
#--a3&S#7jRUWZjk#l j(?(M(MKK,,.E)
 %8%
 /3mmD**MM4'..77%
 %
!\ *k));;;;FFHmmDGGK$89L((rv   r   r   )r   r   r   r   r*   r   r   r   r   r   r   r   r   r   r   s   @rg   r   r     s    r/ rC$J r@ IM.2	')||') #5<<#=>E') t+	')
 +,') 
u||U\\D00	1')rv   r   c                        e Zd Zddededz  f fdZ	 	 ddej                  dej                  dz  dej                  dz  dee	   d	ej                  f
d
Z
 xZS )ModernBertEncoderLayerNr   r   c                    t         |           || _        || _        |dk(  rt	        j
                         | _        n;t	        j                  |j                  |j                  |j                        | _        t        ||      | _        t	        j                  |j                  |j                  |j                        | _        t        |      | _        |j                   |   | _        y )Nr   r   )r   r   )ra   r   r   r   r   r   	attn_normr   r0   r<   r=   r   attnmlp_normr   mlprF   attention_typer   s      rg   r   zModernBertEncoderLayer.__init__f  s    ">[[]DN\\&*<*<&//X^XhXhiDN'vK	V%7%7V__SYScScd ($00;rv   r   r   r   rd   r   c                      | j                   | j                  |      f||d|\  }}||z   }|| j                  | j                  |            z   }|S )N)r   r   )r   r   r  r   )rc   r   r   r   rd   r   _s          rg   r   zModernBertEncoderLayer.forwards  sg     #NN=)
 3)
 	
Q &3%}1M(NNrv   r   r   )r   r   r   r*   r   r   r   r   r   r   r   r   r   s   @rg   r   r   e  sx    </ <C$J <  /337	|| t+ #\\D0	
 +, 
rv   r   c                        e Zd ZU eed<   dZdZddgZdZdZ	dZ
dZeedZ ej                          dej$                  f fd       Z xZS )	ModernBertPreTrainedModelr   modelTr   r   )r   
attentionsmodulec                 R   t         	|   |       | j                  j                  ddt        j
                  dt        ffd}| j                  j                  | j                  j                  t        j                  d| j                  j                  z        z  | j                  j                  | j                  j                  dz  d}t        |t              r ||j                  |d          y t        |t              r- ||j                   |d	           ||j"                  |d
          y t        |t$              r- ||j&                  |d	           ||j"                  |d
          y t        |t(              r ||j*                  |d
          y t        |t,              r ||j.                  |d
          y t        |t0        t2        t4        t6        f      r ||j8                  |d          y t        |t:              r|j<                  D ]  }|j>                  }|j@                  |   dk7  rtB        |j@                  |      } ||j                  |      \  }}tE        jF                  tI        || d      |       tE        jF                  tI        || d      |        y y )Nr   r	  stdc                     t        j                  | j                  d| |z  |z         t        | t        j
                        r-| j                   t        j                  | j                         y y y )NrD   )rR   r  ab)inittrunc_normal_weight
isinstancer   r   r   zeros_)r	  r  cutoff_factors     rg   init_weightz<ModernBertPreTrainedModel._init_weights.<locals>.init_weight  sd     .3&#% &")),;;*KK, + -rv   r:   r   )inout	embedding	final_outr  r  r  r  rk   )r   	_inv_freq_original_inv_freq)%ra   _init_weightsr   r;   r   Moduler   r9   mathsqrtr3   r0   r  r   r   r   r   r   r   r   ModernBertPredictionHeaddenseModernBertForMaskedLMdecoder#ModernBertForSequenceClassificationModernBertForMultipleChoice ModernBertForTokenClassificationModernBertForQuestionAnswering
classifierr   rF   r   rj   r   r  copy_getattr)
rc   r	  r  stdsr   rope_init_fncurr_inv_freqr  r  rf   s
           @rg   r  z'ModernBertPreTrainedModel._init_weights  s4   f%== M	-		 	- 	- ++//;;00499S4;;C`C`=`3aa6600$6	
 f23--tK/@A.		4:.		4;/ 34T$Z0		4;/ 89d5k2 56U43+0.	
 ))4+<= 9:$00 ^
%EE##J/9<#6v7G7G
7S#TL#/*#U q

76j\+CDmT

76j\9K+LM}]^ ;rv   )r   r   r   r*   r   base_model_prefixsupports_gradient_checkpointing_no_split_modules_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendr   r   _can_record_outputsr   no_gradr   r  r  r   r   s   @rg   r  r    sr    &*#/1IJN"& 0)
 U]]_7^BII 7^ 7^rv   r  c                        e Zd Zdef fdZd Zd Zeee		 	 	 	 dde
j                  dz  de
j                  dz  de
j                  dz  d	e
j                  dz  d
ee   defd                     Z xZS )ModernBertModelr   c           	         t         |   |       || _        t        |      | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _
        t        j                  |j                  |j                  |j                        | _        t!        |      | _        d| _        | j'                          y c c}w )Nr   )r   F)ra   r   r   r   
embeddingsr   
ModuleListr_   r3   r   layersr   r0   r<   r=   
final_normr   
rotary_embgradient_checkpointing	post_initr   s      rg   r   zModernBertModel.__init__  s     .v6mmHMfNfNfHgh9#FI6h
 ,,v'9'9vU[UeUef36B&+# is   Cc                 .    | j                   j                  S r   r:  r   r}   s    rg   get_input_embeddingsz$ModernBertModel.get_input_embeddings  s    ---rv   c                 &    || j                   _        y r   rB  r   s     rg   set_input_embeddingsz$ModernBertModel.set_input_embeddings  s    ).&rv   Nr   r   position_idsr   rd   r   c                    |d u |d uz  rt        d      ||j                  d   n|j                  d   }||j                  n|j                  }|&t        j                  ||      j                  d      }| j                  ||      }t        |x}	t              s'| j                  ||d}
t        d
i |
t        d
i |
d}	i }t        | j                  j                        D ]  }| j                  |||      ||<    | j                  D ](  } ||f|	|j                      ||j                      d|}* | j#                  |      }t%        |	      S )Nz:You must specify exactly one of input_ids or inputs_embedsr   r   r   )r   r   )r   r   r   rG   )r   r   )last_hidden_stater]   )r   r   r   r   aranger   r:  r  r   r   r   r   setrF   r>  r<  r  r=  r   )rc   r   r   rF  r   rd   r   r   r   attention_mask_mappingmask_kwargsr   r   encoder_layers                 rg   r   zModernBertModel.forward  s|    -t";<YZZ,9,E-%%a(9??[\K]%.%:!!@T@T <<?II!LL)=YNB0DI++!."0K #<"Jk"J%M%\P[%\&"
 !dkk556 	gJ.2oom\[e.f
+	g "[[ 	M)5m6R6RS$78T8T$U 	M	 6??rv   r   )r   r   r   r*   r   rC  rE  r"   r#   r   r   r   r   r   r   r   r   r   r   s   @rg   r8  r8    s    
/ 
./   .2.204-1,@##d*,@ t+,@ &&-	,@
 ||d*,@ +,,@ 
,@    ,@rv   r8  c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )r   r   c                 J   t         |           || _        t        j                  |j
                  |j
                  |j                        | _        t        |j                     | _
        t        j                  |j
                  |j                  |j                        | _        y )Nr   )ra   r   r   r   r   r0   rU   r!  r   rV   r   r   r<   r=   r   r   s     rg   r   z!ModernBertPredictionHead.__init__  sq    YYv1163E3EvG]G]^
&667LL!3!3vO_O_`	rv   r   r   c                 `    | j                  | j                  | j                  |                  S r   )r   r   r!  )rc   r   s     rg   r   z ModernBertPredictionHead.forward  s#    yy$**]";<==rv   )	r   r   r   r*   r   r   r   r   r   r   s   @rg   r   r     s-    a/ a>U\\ >ell >rv   r   zd
    The ModernBert Model with a decoder head on top that is used for masked language modeling.
    )custom_introc                   >    e Zd ZddiZdef fdZd Zdej                  fdZ	e
e	 	 	 	 	 dd	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dee   deej                     ez  fd              Z xZS )r"  zdecoder.weightz&model.embeddings.tok_embeddings.weightr   c                 t   t         |   |       || _        t        |      | _        t        |      | _        t        j                  |j                  |j                  |j                        | _        | j                  j                  | _        | j                  j                  | _        | j                          y )Nr   )ra   r   r   r8  r  r   headr   r   r0   r/   rP   r#  rX   rY   r@  r   s     rg   r   zModernBertForMaskedLM.__init__*  s     $V,
,V4	yy!3!3V5F5FVM`M`a!%!>!>(,(L(L% 	rv   c                     | j                   S r   r#  r}   s    rg   get_output_embeddingsz+ModernBertForMaskedLM.get_output_embeddings7  s    ||rv   new_embeddingsc                     || _         y r   rW  )rc   rY  s     rg   set_output_embeddingsz+ModernBertForMaskedLM.set_output_embeddings:  s	    %rv   Nr   r   rF  r   labelsrd   r   c                     | j                   d||||d|}|d   }| j                  rK|I|j                  d      }|j                  |j                  d   d      }|| j                  k7  }	||	   }||	   }| j                  | j                  |            }
d }|* | j                  |
|fd| j                  j                  i|}t        ||
|j                  |j                        S )Nr   r   rF  r   r   r   r/   losslogitsr   r  r]   )r  rX   r   r   rY   r#  rU  loss_functionr   r/   r   r   r  )rc   r   r   rF  r   r\  rd   outputsrI  mask_tokensra  r`  s               rg   r   zModernBertForMaskedLM.forward=  s	    $** 
)%'	

 
 $AJ!!f&8[[_F 1 6 6v||A K !D$A$AAK 1+ >K(Fdii(9:;%4%%ffbAWAWb[abD!//))	
 	
rv   NNNNN)r   r   r   _tied_weights_keysr*   r   rX  r   r   r[  r!   r   r   r   r   r   r   r   r   r   r   r   s   @rg   r"  r"  "  s     +,TU/ &BII &  .2.2,0-1&*'
##d*'
 t+'
 llT)	'

 ||d*'
 t#'
 +,'
 
u||	~	-'
  '
rv   r"  z`
    The ModernBert Model with a sequence classification head on top that performs pooling.
    c                       e Zd Zdef fdZee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e
e   d
eej                     ez  fd              Z xZS )r$  r   c                 n   t         |   |       |j                  | _        || _        t	        |      | _        t        |      | _        t        j                  j                  |j                        | _        t        j                  |j                  |j                        | _        | j!                          y r   )ra   r   
num_labelsr   r8  r  r   rU  r   r   r   rT   r   r   r0   r(  r@  r   s     rg   r   z,ModernBertForSequenceClassification.__init__o  s      ++$V,
,V4	HH$$V%>%>?	))F$6$68I8IJ 	rv   Nr   r   rF  r   r\  rd   r   c                 d    | j                   d||||d|}|d   }| j                  j                  dk(  r
|dddf   }n| j                  j                  dk(  rw|=t        j                  |j
                  dd |j                  t        j                        }||j                  d      z  j                  d	
      |j                  d	d      z  }| j                  |      }	| j                  |	      }	| j                  |	      }
d}|| j                  j                  | j                  d	k(  rd| j                  _        nl| j                  d	kD  rL|j                  t        j                   k(  s|j                  t        j"                  k(  rd| j                  _        nd| j                  _        | j                  j                  dk(  rIt%               }| j                  d	k(  r& ||
j'                         |j'                               }n ||
|      }n| j                  j                  dk(  r=t)               } ||
j+                  d| j                        |j+                  d            }n,| j                  j                  dk(  rt-               } ||
|      }t/        ||
|j0                  |j2                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        r^  r   rQ   NrR   r$   )r   r   r   r   r   Tr   keepdim
regressionsingle_label_classificationmulti_label_classificationr_  r]   )r  r   rS   r   onesr   r   r`   r   sumrU  r   r(  problem_typeri  r   longr   r
   squeezer	   r   r   r   r   r  )rc   r   r   rF  r   r\  rd   rc  rI  pooled_outputra  r`  loss_fcts                rg   r   z+ModernBertForSequenceClassification.forward|  se   " $** 
)%'	

 
 $AJ;;))U2 1!Q$ 7[[++v5%!&%++BQ/8I8P8PX]XbXb" "3^5M5Mb5Q!Q V V[\ V ]`n`r`rt as a ! 		"34		-0/{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#FF3D))-JJ+-B @&++b/R))-II,./'!//))	
 	
rv   re  )r   r   r   r*   r   r!   r   r   r   r   r   r   r   r   r   r   r   s   @rg   r$  r$  i  s    /   .2.2,0-1&*C
##d*C
 t+C
 llT)	C

 ||d*C
 t#C
 +,C
 
u||	7	7C
  C
rv   r$  zv
    The ModernBert Model with a token classification head on top, e.g. for Named Entity Recognition (NER) tasks.
    c                       e Zd Zdef fdZee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e
e   d
eej                     ez  fd              Z xZS )r&  r   c                 `   t         |   |       |j                  | _        t        |      | _        t        |      | _        t        j                  j                  |j                        | _        t        j                  |j                  |j                        | _        | j                          y r   ra   r   ri  r8  r  r   rU  r   r   r   rT   r   r   r0   r(  r@  r   s     rg   r   z)ModernBertForTokenClassification.__init__  s{      ++$V,
,V4	HH$$V%>%>?	))F$6$68I8IJ 	rv   Nr   r   rF  r   r\  rd   r   c                 f    | j                   d||||d|}|d   }| j                  |      }| j                  |      }| j                  |      }	d}
|<t	               } ||	j                  d| j                        |j                  d            }
t        |
|	|j                  |j                        S )z
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        r^  r   Nr   r_  r]   )
r  rU  r   r(  r	   r   ri  r   r   r  )rc   r   r   rF  r   r\  rd   rc  rI  ra  r`  rv  s               rg   r   z(ModernBertForTokenClassification.forward  s     $** 
)%'	

 
 $AJ II&78 II&78!23')HFKKDOO<fkk"oND$!//))	
 	
rv   re  )r   r   r   r*   r   r!   r   r   r   r   r   r   r   r   r   r   r   s   @rg   r&  r&    s    
/ 
  .2.2,0-1&*$
##d*$
 t+$
 llT)	$

 ||d*$
 t#$
 +,$
 
u||	4	4$
  $
rv   r&  c                       e Zd Zdef fdZee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
   d
eej                     ez  fd              Z xZS )r'  r   c                 `   t         |   |       |j                  | _        t        |      | _        t        |      | _        t        j                  j                  |j                        | _        t        j                  |j                  |j                        | _        | j                          y r   ry  r   s     rg   r   z'ModernBertForQuestionAnswering.__init__  sy      ++$V,
,V4	HH$$V%>%>?	))F$6$68I8IJrv   Nr   r   rF  start_positionsend_positionsrd   r   c                     | j                   |f||d|}|d   }| j                  |      }| j                  |      }| j                  |      }	|	j	                  dd      \  }
}|
j                  d      j                         }
|j                  d      j                         }d }|| | j                  |
|||fi |}t        ||
||j                  |j                        S )N)r   rF  r   r   r   r   )r`  start_logits
end_logitsr   r  )r  rU  r   r(  splitrt  r   rb  r   r   r  )rc   r   r   rF  r}  r~  rd   rc  rI  ra  r  r  r`  s                rg   r   z&ModernBertForQuestionAnswering.forward  s    $**
)%
 	
 $AJ II&78 II&78!23#)<<r<#: j#++B/::<''+668
&=+D%4%%lJQ^ibhiD+%!!//))
 	
rv   re  )r   r   r   r*   r   r!   r   r   r   r   r   r   r   r   r   r   s   @rg   r'  r'    s    	/ 	  *..2,0/3-1#
<<$&#
 t+#
 llT)	#

 ,#
 ||d*#
 +,#
 
u||	;	;#
  #
rv   r'  z
    The ModernBert Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks.
    c                       e Zd Zdef fdZee	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e
e   d
eej                     ez  fd              Z xZS )r%  r   c                 8   t         |   |       || _        t        |      | _        t        |      | _        t        j                  j                  |j                        | _        t        j                  |j                  d      | _        | j                          y )Nr   )ra   r   r   r8  r  r   rU  r   r   r   rT   r   r   r0   r(  r@  r   s     rg   r   z$ModernBertForMultipleChoice.__init__:  so     $V,
,V4	HH$$V%>%>?	))F$6$6: 	rv   Nr   r   rF  r   r\  rd   r   c                    ||j                   d   n|j                   d   }|!|j                  d|j                  d            nd}|!|j                  d|j                  d            nd}|!|j                  d|j                  d            nd}|1|j                  d|j                  d      |j                  d            nd} | j                  d||||d|}|d   }	| j                  j
                  dk(  rt        j                  |	j                   d   |	j                        }
|,|j                  d	      j                  |	j                        }n0t        j                  dt        j                  |	j                  
      }|	|
|f   }	nS| j                  j
                  dk(  r:|j                  dd      }|	|j                  d      z  j                  d	      |z  }	| j                  |	      }| j!                  |      }| j#                  |      }|j                  d|      }d}|t%        j&                         } |||      }t)        |||j*                  |j,                        S )a&  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors.
        Nr   r   r^  r   rQ   rH  r   )r   r   rR   Trk  r_  r]   )r   r   sizer  r   rS   r   rJ  r   argmaxr   tensorrs  rq  r   rU  r   r(  r   r	   r   r   r  )rc   r   r   rF  r   r\  rd   num_choicesrc  rI  	indices_0cls_masknum_non_pad_tokensru  ra  reshaped_logitsr`  rv  s                     rg   r   z#ModernBertForMultipleChoice.forwardF  sn     -6,Aiooa(}GZGZ[\G]>G>SINN2y~~b'9:Y]	M[Mg,,R1D1DR1HImqGSG_|((\->->r-BCei ( r=#5#5b#9=;M;Mb;QR 	 $** 
)%'	

 
 $AJ ;;))U2%6%<%<Q%?HYH`H`aI))00R08;;<M<T<TU !<<DUD\D\] 1)X2E F [[++v5!/!3!34!3!H!2^5M5Mb5Q!Q V V[\ V ]`r r		"34		-0/ ++b+6**,HOV4D("!//))	
 	
rv   re  )r   r   r   r*   r   r!   r   r   r   r   r   r   r   r   r   r   r   s   @rg   r%  r%  4  s    
/ 
  .2.2,0-1&*C
##d*C
 t+C
 llT)	C

 ||d*C
 t#C
 +,C
 
u||	8	8C
  C
rv   r%  )r*   r8  r  r"  r$  r&  r'  r%  )r   )Mr  collections.abcr   typingr   r   r   huggingface_hub.dataclassesr   r   torch.nnr   r	   r
    r   r  activationsr   configuration_utilsr   integrationsr   r   masking_utilsr   r   modeling_layersr   modeling_outputsr   r   r   r   r   r   modeling_rope_utilsr   modeling_utilsr   r   processing_utilsr   utilsr   r   r    utils.genericr!   r"   utils.output_capturingr#   align.modeling_alignr%   gemma3.modeling_gemma3r&   r'   
get_loggerr   loggerr*   r  r   r   r   r   r   r   r  r8  r   r"  r$  r&  r'  r%  __all__r]   rv   rg   <module>r     s7     $ $  .  A A & ! 3 I ` 9  7 F & @ @ I 5 : G 
		H	% 89G)' G)  :G)T299 ,:BII :(\ 5 \ *+B ,B4 )*N)")) N) +N)b7 @ G^ G^ G^T B@/ B@ B@J	>ryy 	> 
?
5 ?

?
D 
S
*C S

S
l 
3
'@ 3

3
l 1
%> 1
 1
h 
R
"; R

R
j	rv   