Ë
    ÛÓ^j“%  ã                   óò  — d dl mZ d dlZd dlmZ d dlmZ ddlmZ ddlm	Z	 ddl
mZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZ ddlmZmZmZmZmZmZmZmZmZm Z  ddl!m"Z"  ejF                  e$«      Z% ed¬«      e G d„ de«      «       «       Z& G d„ de"«      Z' G d„ de«      Z( G d„ de«      Z) G d„ de«      Z* G d„ de«      Z+ G d„ de«      Z, G d „ d!e«      Z- G d"„ d#e«      Z. G d$„ d%e«      Z/g d&¢Z0y)'é    )ÚCallableN)Ústrict)Únné   )ÚACT2CLS)ÚCache)ÚPreTrainedConfig)ÚRopeParameters)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚloggingé   )
ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaForCausalLMÚLlamaForTokenClassificationÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRMSNormÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)ÚNemotronMLPz!swiss-ai/Apertus-8B-Instruct-2509)Ú
checkpointc            	       ó   ‡ — e Zd ZU dZdZdgZdZdddddddddœZd	gd
gfddgdgfdgdgfdœZdZ	e
ed<   dZe
ed<   dZe
ed<   dZe
ed<   dZe
ed<   dZe
dz  ed<   dZeed<   dZe
ed<   dZeed<   dZeed <   d!Zeed"<   d#Ze
dz  ed$<   d%Ze
dz  ed&<   d'Ze
ee
   z  dz  ed(<   d)Zeed*<   dZee z  dz  ed+<   d)Z!eed,<   d-Z"ee
z  ed.<   ˆ fd/„Z#ˆ xZ$S )0ÚApertusConfigaz  
    ```python
    >>> from transformers import ApertusModel, ApertusConfig

    >>> # Initializing a Apertus-8B style configuration
    >>> configuration = ApertusConfig()

    >>> # Initializing a model from the Apertus-8B style configuration
    >>> model = ApertusModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```ÚapertusÚpast_key_valuesç    `ãfAÚcolwiseÚreplicated_with_grad_allreduceÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.q_normzlayers.*.self_attn.k_normzlayers.*.self_attn.o_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormi   Ú
vocab_sizei   Úhidden_sizei 8  Úintermediate_sizeé    Únum_hidden_layersÚnum_attention_headsNÚnum_key_value_headsÚxieluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsTÚ	use_cacher   Úpad_token_idé   Úbos_token_idr   Úeos_token_idFÚtie_word_embeddingsÚrope_parametersÚattention_biasç        Úattention_dropoutc                 ó’   •— | j                   €| j                  | _         | j                  €dddddddœ| _        t        ‰| �  di |¤Ž y )	NÚllama3r!   g       @i    g      ð?g      @)Ú	rope_typeÚ
rope_thetaÚfactorÚ original_max_position_embeddingsÚlow_freq_factorÚhigh_freq_factor© )r2   r1   r>   ÚsuperÚ__post_init__)ÚselfÚkwargsÚ	__class__s     €úv/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/apertus/modular_apertus.pyrL   zApertusConfig.__post_init__e   sW   ø€ Ø×#Ñ#Ð+Ø'+×'?Ñ'?ˆDÔ$à×ÑÐ'à%Ø(ØØ48Ø#&Ø$'ñ$ˆDÔ ô 	‰ÑÑ' Ó'ó    )%Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚdefault_thetaÚbase_model_tp_planÚbase_model_pp_planr,   ÚintÚ__annotations__r-   r.   r0   r1   r2   r4   Ústrr5   r6   Úfloatr7   r8   Úboolr9   r;   r<   Úlistr=   r>   r
   Údictr?   rA   rL   Ú__classcell__©rO   s   @rP   r   r   .   sb  ø… ñð €JØ#4Ð"5ÐØ€Mà%.Ø%.Ø%.Ø%EØ%EØ%.Ø )Ø"+ñ	Ðð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð €J�ÓØ€K�ÓØ"Ð�sÓ"ØÐ�sÓØ!Ð˜Ó!Ø&*Ð˜˜t™Ó*Ø€J�ÓØ#(Ð˜SÓ(Ø#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ €L�#˜‘*Ó Ø €L�#˜‘*Ó Ø+,€L�#˜˜S™	‘/ DÑ(Ó,Ø %Ð˜Ó%Ø48€O�^ dÑ*¨TÑ1Ó8Ø €N�DÓ Ø%(Ð�u˜s‘{Ó(÷(ð (rQ   r   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )Ú
ApertusMLPc                 óD  •— t         ‰| �  |«       t        j                  | j                  | j
                  d¬«      | _        t        j                  | j
                  | j                  d¬«      | _        |j                  dk(  rt        d   |j                  ¬«      | _        y y )NF)Úbiasr3   )Údtype)rK   Ú__init__r   ÚLinearr-   r.   Úup_projÚ	down_projr4   r   rh   Úact_fn)rM   ÚconfigrO   s     €rP   ri   zApertusMLP.__init__v   sz   ø€ Ü‰Ñ˜Ô Ü—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒØ×Ñ Ò'Ü! 'Ñ*°·±Ô>ˆD�Kð (rQ   )rR   rS   rT   ri   rb   rc   s   @rP   re   re   u   s   ø„ ÷?ð ?rQ   re   c                   ó   — e Zd Zy)ÚApertusRMSNormN©rR   rS   rT   rJ   rQ   rP   rp   rp   ~   ó   „ ØrQ   rp   c                   ó   — e Zd Zy)ÚApertusRotaryEmbeddingNrq   rJ   rQ   rP   rt   rt   ‚   rr   rQ   rt   c                   óò   ‡ — e Zd Zddededz  fˆ fd„Z	 ddej                  deej                  ej                  f   dej                  dz  de	dz  d	e
e   d
eej                  ej                  f   fd„Zˆ xZS )ÚApertusAttentionNrn   Ú	layer_idxc                 óº   •— t         ‰| �  ||«       t        | j                  |j                  «      | _        t        | j                  |j                  «      | _        y ©N)rK   ri   rp   Úhead_dimr7   Úq_normÚk_norm©rM   rn   rw   rO   s      €rP   ri   zApertusAttention.__init__‡   sB   ø€ Ü‰Ñ˜ Ô+Ü$ T§]¡]°F×4GÑ4GÓHˆŒÜ$ T§]¡]°F×4GÑ4GÓHˆ�rQ   r'   Úposition_embeddingsr(   r    rN   Úreturnc                 óN  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
| j                  |«      }| j                  |	«      }	|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t         «      } || ||	|
|f| j"                  sdn| j$                  | j&                  dœ|¤Ž\  }} |j(                  g |¢d‘­Ž j+                  «       }| j-                  |«      }||fS )Néÿÿÿÿr:   r   r@   )ÚdropoutÚscaling)Úshaperz   Úq_projÚviewÚ	transposeÚk_projÚv_projr{   r|   r   Úupdaterw   r   Úget_interfacern   Ú_attn_implementationr   ÚtrainingrA   rƒ   ÚreshapeÚ
contiguousÚo_proj)rM   r'   r~   r(   r    rN   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                   rP   ÚforwardzApertusAttention.forwardŒ   s­  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—{‘{ <Ó0ˆØ—[‘[ Ó,ˆ
à&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(rQ   ry   )rR   rS   rT   r   r[   ri   ÚtorchÚTensorÚtupler   r   r   r›   rb   rc   s   @rP   rv   rv   †   sœ   ø„ ñI˜}ð I¸¸t¹õ Ið )-ñ()à—|‘|ð()ð # 5§<¡<°·±Ð#=Ñ>ð()ð Ÿ™ tÑ+ð	()ð
  ™ð()ð Ð+Ñ,ð()ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷()rQ   rv   c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
eej                  ej                  f   dz  dee   dej                  fd„Zˆ xZS )ÚApertusDecoderLayerrn   rw   c                 óÆ   •— t         ‰| �  ||«       t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        | `| `y )N)Úeps)	rK   ri   rp   r-   r7   Úattention_layernormÚfeedforward_layernormÚinput_layernormÚpost_attention_layernormr}   s      €rP   ri   zApertusDecoderLayer.__init__¸   sT   ø€ Ü‰Ñ˜ Ô+Ü#1°&×2DÑ2DÈ&×J]ÑJ]Ô#^ˆÔ Ü%3°F×4FÑ4FÈF×L_ÑL_Ô%`ˆÔ"àÐ ØÑ)rQ   Nr'   r(   Úposition_idsr    r8   r~   rN   r   c           
      óÀ   — |}| j                  |«      } | j                  d||||||dœ|¤Ž\  }}	||z   }|}| j                  |«      }| j                  |«      }||z   }|S )N)r'   r(   r§   r    r8   r~   rJ   )r£   Ú	self_attnr¤   Úmlp)
rM   r'   r(   r§   r    r8   r~   rN   ÚresidualÚ_s
             rP   r›   zApertusDecoderLayer.forwardÀ   s�   € ð !ˆØ×0Ñ0°Ó?ˆØ)˜4Ÿ>™>ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
Ñˆ�qð ! =Ñ0ˆð !ˆØ×2Ñ2°=ÓAˆØŸ™ Ó/ˆØ  =Ñ0ˆØÐrQ   )NNNFN)rR   rS   rT   r   r[   ri   rœ   r�   Ú
LongTensorr   r_   rž   r   r   r›   rb   rc   s   @rP   r    r    ·   s»   ø„ ð*˜}ð *¸õ *ð /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð+Ñ,ðð 
�‰÷rQ   r    c                   ó   — e Zd Zy)ÚApertusPreTrainedModelNrq   rJ   rQ   rP   r¯   r¯   ß   rr   rQ   r¯   c                   ó   — e Zd Zy)ÚApertusModelNrq   rJ   rQ   rP   r±   r±   ã   rr   rQ   r±   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚApertusForCausalLMc                 ó"   •— t        ‰| �  di |¤ŽS )aŠ  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, ApertusForCausalLM

        >>> model = ApertusForCausalLM.from_pretrained("swiss-ai/Apertus-8B-Instruct-2509")
        >>> tokenizer = AutoTokenizer.from_pretrained("swiss-ai/Apertus-8B-Instruct-2509")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```rJ   )rK   r›   )rM   Úsuper_kwargsrO   s     €rP   r›   zApertusForCausalLM.forwardè   s   ø€ ô. ‰w‰Ñ. Ñ.Ð.rQ   )rR   rS   rT   r›   rb   rc   s   @rP   r³   r³   ç   s   ø„ ÷/ð /rQ   r³   c                   ó   — e Zd Zy)ÚApertusForTokenClassificationNrq   rJ   rQ   rP   r·   r·     rr   rQ   r·   )r   r±   r³   r·   r¯   )1Úcollections.abcr   rœ   Úhuggingface_hub.dataclassesr   r   Úactivationsr   Úcache_utilsr   Úconfiguration_utilsr	   Úmodeling_rope_utilsr
   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   r   Únemotron.modeling_nemotronr   Ú
get_loggerrR   Úloggerr   re   rp   rt   rv   r    r¯   r±   r³   r·   Ú__all__rJ   rQ   rP   ú<module>rÆ      sü   ðõ %ã Ý .Ý å "Ý  Ý 3Ý 1Ý 5Ý &ß @Ñ @÷÷ ÷ õ 5ð 
ˆ×	Ñ	˜HÓ	%€ñ Ð>Ô?ØôB(Ð$ó B(ó ó @ðB(ôJ?�ô ?ô	�\ô 	ô	Ð1ô 	ô.)�~ô .)ôb%Ð+ô %ôP	Ð1ô 	ô	�:ô 	ô/Ð)ô /ô6	Ð$?ô 	ò�rQ   