Ë
    ÜÓ^jÆ  ã                   ór   — d dl mZ d dlmZ ddlmZ ddlmZ  ed¬«      e G d„ d	e«      «       «       Zd	gZ	y
)é    )ÚLiteral)Ústricté   )ÚPreTrainedConfig)Úauto_docstringzanswerdotai/ModernBERT-base)Ú
checkpointc                   ó´  ‡ — e Zd ZU dZdZdgZdddœZdZee	d<   d	Z
ee	d
<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZedz  e	d<   d Zeee   z  dz  e	d!<   d"Zedz  e	d#<   d"Zedz  e	d$<   d Zedz  e	d%<   dZee	d&<   d'Zeez  e	d(<   dZee   dz  e	d)<   dZ e!e"d*   e!f   dz  e	d+<   d,Z#ee	d-<   d'Z$eez  e	d.<   dZ%ee	d/<   d'Z&eez  e	d0<   d1Z'ee	d2<   d3Z(e"d4   e	d5<   d'Z)eez  e	d6<   dZ*ee	d7<   dZ+ee	d8<   dZ,ee	d9<   dZ-ee	d:<   d;Z.ee	d<<   d1Z/ee	d=<   ˆ fd>„Z0d?„ Z1ˆ fd@„Z2e3dA„ «       Z4e4jj                  dB„ «       Z4ˆ xZ6S )CÚModernBertConfiga+  
    initializer_cutoff_factor (`float`, *optional*, defaults to 2.0):
        The cutoff factor for the truncated_normal_initializer for initializing all weight matrices.
    norm_eps (`float`, *optional*, defaults to 1e-05):
        The epsilon used by the rms normalization layers.
    norm_bias (`bool`, *optional*, defaults to `False`):
        Whether to use bias in the normalization layers.
    local_attention (`int`, *optional*, defaults to 128):
        The window size for local attention.
    mlp_dropout (`float`, *optional*, defaults to 0.0):
        The dropout ratio for the MLP layers.
    decoder_bias (`bool`, *optional*, defaults to `True`):
        Whether to use bias in the decoder layers.
    classifier_pooling (`str`, *optional*, defaults to `"cls"`):
        The pooling method for the classifier. Should be either `"cls"` or `"mean"`. In local attention layers, the
        CLS token doesn't attend to all tokens on long sequences.
    classifier_bias (`bool`, *optional*, defaults to `False`):
        Whether to use bias in the classifier.
    classifier_activation (`str`, *optional*, defaults to `"gelu"`):
        The activation function for the classifier.
    deterministic_flash_attn (`bool`, *optional*, defaults to `False`):
        Whether to use deterministic flash attention. If `False`, inference will be faster but not deterministic.
    sparse_prediction (`bool`, *optional*, defaults to `False`):
        Whether to use sparse prediction for the masked language model instead of returning the full dense logits.
    sparse_pred_ignore_index (`int`, *optional*, defaults to -100):
        The index to ignore for the sparse prediction.

    Examples:

    ```python
    >>> from transformers import ModernBertModel, ModernBertConfig

    >>> # Initializing a ModernBert style configuration
    >>> configuration = ModernBertConfig()

    >>> # Initializing a model from the modernbert-base style configuration
    >>> model = ModernBertModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Ú
modernbertÚpast_key_valuesg     ˆAg     ˆÃ@)ÚglobalÚlocaliÀÄ  Ú
vocab_sizei   Úhidden_sizei€  Úintermediate_sizeé   Únum_hidden_layersé   Únum_attention_headsÚgeluÚhidden_activationi    Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeg       @Úinitializer_cutoff_factorgñhãˆµøä>Únorm_epsFÚ	norm_biasikÄ  NÚpad_token_idijÄ  Úeos_token_idiiÄ  Úbos_token_idÚcls_token_idÚsep_token_idÚattention_biasg        Úattention_dropoutÚlayer_types)Úfull_attentionÚsliding_attentionÚrope_parametersé€   Úlocal_attentionÚembedding_dropoutÚmlp_biasÚmlp_dropoutTÚdecoder_biasÚcls)r.   ÚmeanÚclassifier_poolingÚclassifier_dropoutÚclassifier_biasÚclassifier_activationÚdeterministic_flash_attnÚsparse_predictioniœÿÿÿÚsparse_pred_ignore_indexÚtie_word_embeddingsc                 óÚ   •— |j                  dd«      }| j                  €8t        | j                  «      D �cg c]  }t	        ||z  «      rdnd‘Œ c}| _        t        ‰| �  di |¤Ž y c c}w )NÚglobal_attn_every_n_layersr   r&   r%   © )Úgetr$   Úranger   ÚboolÚsuperÚ__post_init__)ÚselfÚkwargsr9   ÚiÚ	__class__s       €ú‚/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/modernbert/configuration_modernbert.pyr?   zModernBertConfig.__post_init__q   st   ø€ à%+§Z¡ZÐ0LÈaÓ%PÐ"Ø×ÑÐ#ô ˜t×5Ñ5Ó6ö àô (,¨AÐ0JÑ,JÔ'KÑ#ÐQaÑaò ˆDÔô
 	‰ÑÑ' Ó'ùò s   ·A(c                 ó¶  — |j                  dd «      }ddiddidœ}| j                  �| j                  n|| _        |�<| j                  d   j                  |«       | j                  d   j                  |«       | j                  j                  d«      €ddi| j                  d<   | j                  d   j	                  d|j                  d| j
                  d	   «      «       | j                  j                  d«      €ddi| j                  d<   | j                  d   j	                  d|j                  d
| j
                  d   «      «       | j                  «        |S )NÚrope_scalingÚ	rope_typeÚdefault)r&   r%   r%   r&   Ú
rope_thetaÚglobal_rope_thetar   Úlocal_rope_thetar   )Úpopr'   Úupdater;   Ú
setdefaultÚdefault_thetaÚstandardize_rope_params)r@   rA   rF   Údefault_rope_paramss       rD   Úconvert_rope_params_to_dictz,ModernBertConfig.convert_rope_params_to_dict|   sc  € Ø—z‘z .°$Ó7ˆð
 #.¨yÐ!9Ø*¨IÐ6ñ
Ðð 8<×7KÑ7KÐ7W˜t×3Ò3Ð]pˆÔØÐ#Ø× Ñ Ð!1Ñ2×9Ñ9¸,ÔGØ× Ñ Ð!4Ñ5×<Ñ<¸\ÔJð ×Ñ×#Ñ#Ð$4Ó5Ð=Ø6AÀ9Ð5MˆD× Ñ Ð!1Ñ2Ø×ÑÐ-Ñ.×9Ñ9Ø˜&Ÿ*™*Ð%8¸$×:LÑ:LÈXÑ:VÓWô	
ð ×Ñ×#Ñ#Ð$7Ó8Ð@Ø9DÀiÐ8PˆD× Ñ Ð!4Ñ5Ø×ÑÐ0Ñ1×<Ñ<Ø˜&Ÿ*™*Ð%7¸×9KÑ9KÈGÑ9TÓUô	
ð
 	×$Ñ$Ô&Øˆó    c                 óH   •— t         ‰| �  «       }|j                  dd «       |S )NÚreference_compile)r>   Úto_dictrL   )r@   ÚoutputrC   s     €rD   rV   zModernBertConfig.to_dictš   s#   ø€ Ü‘‘Ó"ˆØ�
‰
Ð&¨Ô-ØˆrS   c                 ó    — | j                   dz  S )zKHalf-window size: `local_attention` is the total window, so we divide by 2.é   ©r)   )r@   s    rD   Úsliding_windowzModernBertConfig.sliding_windowŸ   s   € ð ×#Ñ# qÑ(Ð(rS   c                 ó   — |dz  | _         y)z<Set sliding_window by updating local_attention to 2 * value.rY   NrZ   )r@   Úvalues     rD   r[   zModernBertConfig.sliding_window¤   s   € ð  % q™yˆÕrS   )7Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferencerO   r   ÚintÚ__annotations__r   r   r   r   r   Ústrr   r   Úfloatr   r   r   r=   r   r   Úlistr   r    r!   r"   r#   r$   r'   Údictr   r)   r*   r+   r,   r-   r0   r1   r2   r3   r4   r5   r6   r7   r?   rR   rV   Úpropertyr[   ÚsetterÚ__classcell__)rC   s   @rD   r
   r
      s  ø… ñ(ðT €JØ#4Ð"5ÐØ(°8Ñ<€Mà€J�ÓØ€K�ÓØ!Ð�sÓ!ØÐ�sÓØ!Ð˜Ó!Ø#Ð�sÓ#Ø#'Ð˜SÓ'Ø#Ð�uÓ#Ø'*Ð˜uÓ*Ø€HˆeÓØ€IˆtÓØ$€L�#˜‘*Ó$Ø+0€L�#˜˜S™	‘/ DÑ(Ó0Ø$€L�#˜‘*Ó$Ø$€L�#˜‘*Ó$Ø$€L�#˜‘*Ó$Ø €N�DÓ Ø%(Ð�u˜s‘{Ó(Ø$(€K��c‘˜TÑ!Ó(ØY]€O�T˜'Ð"GÑHÈ$ÐNÑOÐRVÑVÓ]Ø€O�SÓØ%(Ð�u˜s‘{Ó(Ø€HˆdÓØ"€K�˜‘Ó"Ø€L�$ÓØ16Ð˜ Ñ.Ó6Ø&)Ð˜ ™Ó)Ø!€O�TÓ!Ø!'Ð˜3Ó'Ø%*Ð˜dÓ*Ø#Ð�tÓ#Ø$(Ð˜cÓ(Ø $Ð˜Ó$ô	(òô<ð
 ñ)ó ð)ð ×Ññ)ó ô)rS   r
   N)
Útypingr   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úutilsr   r
   Ú__all__r:   rS   rD   ú<module>rr      sK   ðõ, å .å 3Ý #ñ Ð8Ô9ØôG)Ð'ó G)ó ó :ðG)ðT Ð
�rS   