Ë
    ÛÓ^j�  ã                   ó
  — d dl mZ d dlmZ ddlmZmZ ddlmZ ddl	m
Z
mZmZmZ ddlmZ  ed	¬
«      e G d„ de«      «       «       Z G d„ de«      Z G d„ de
«      Z G d„ de«      Z G d„ de«      Z G d„ de«      Zg d¢Zy)é    N)Ústricté   )Úauto_docstringÚcan_return_tupleé   )ÚLlamaConfig)ÚLlamaDecoderLayerÚLlamaForCausalLMÚ
LlamaModelÚLlamaPreTrainedModel)ÚNemotronMLPzinceptionai/Jais-2-8B-Chat)Ú
checkpointc                   óî   — e Zd ZU dddddddœZdZeed<   dZeed<   dZeed	<   d
Z	eed<   dZ
eed<   dZeed<   dZeed<   dZedz  ed<   dZeee   z  dz  ed<   dZeed<   dZeed<    e«       Z e«       Zy)ÚJais2ConfigÚcolwiseÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proji K Ú
vocab_sizei   Úhidden_sizei h  Úintermediate_sizeé   Únum_attention_headsÚrelu2Ú
hidden_acti    Úmax_position_embeddingsgñhãˆµøä>Úlayer_norm_epsr   NÚbos_token_idiJ Úeos_token_idTÚattention_biasÚmlp_bias)Ú__name__Ú
__module__Ú__qualname__Úbase_model_tp_planr   ÚintÚ__annotations__r   r   r   r   Ústrr   r   Úfloatr   r   Úlistr   Úboolr   ÚAttributeErrorÚrms_norm_epsÚpretraining_tp© ó    úr/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/jais2/modular_jais2.pyr   r      s²   … ð &/Ø%.Ø%.Ø%.Ø )Ø"+ñÐð €J�ÓØ€K�ÓØ"Ð�sÓ"Ø!Ð˜Ó!Ø€J�ÓØ#'Ð˜SÓ'Ø €N�EÓ Ø €L�#˜‘*Ó Ø+1€L�#˜˜S™	‘/ DÑ(Ó1Ø€N�DÓØ€HˆdÓÙ!Ó#€LÙ#Ó%�Nr.   r   c                   ó   — e Zd Zy)ÚJais2MLPN©r    r!   r"   r-   r.   r/   r1   r1   9   ó   „ Ør.   r1   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚJais2DecoderLayerÚconfigÚ	layer_idxc                 óæ   •— t         ‰| �  ||«       t        j                  |j                  |j
                  ¬«      | _        t        j                  |j                  |j
                  ¬«      | _        y ©N)Úeps)ÚsuperÚ__init__ÚnnÚ	LayerNormr   r   Úinput_layernormÚpost_attention_layernorm)Úselfr6   r7   Ú	__class__s      €r/   r<   zJais2DecoderLayer.__init__>   sP   ø€ Ü‰Ñ˜ Ô+Ü!Ÿ|™|¨F×,>Ñ,>ÀF×DYÑDYÔZˆÔÜ(*¯©°V×5GÑ5GÈV×MbÑMbÔ(cˆÕ%r.   )r    r!   r"   r   r$   r<   Ú__classcell__©rB   s   @r/   r5   r5   =   s   ø„ ðd˜{ð d°s÷ dñ dr.   r5   c                   ó   — e Zd Zy)ÚJais2PreTrainedModelNr2   r-   r.   r/   rF   rF   D   r3   r.   rF   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )Ú
Jais2Modelr6   c                 ó„   •— t         ‰| �  |«       t        j                  |j                  |j
                  ¬«      | _        y r9   )r;   r<   r=   r>   r   r   Únorm)rA   r6   rB   s     €r/   r<   zJais2Model.__init__I   s.   ø€ Ü‰Ñ˜Ô Ü—L‘L ×!3Ñ!3¸×9NÑ9NÔOˆ�	r.   )r    r!   r"   r   r<   rC   rD   s   @r/   rH   rH   H   s   ø„ ðP˜{÷ Pñ Pr.   rH   c                   ó2   ‡ — e Zd Zeeˆ fd„«       «       Zˆ xZS )ÚJais2ForCausalLMc                 ó"   •— t        ‰| �  di |¤ŽS )aÓ  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, Jais2ForCausalLM

        >>> model = Jais2ForCausalLM.from_pretrained("inceptionai/Jais-2-8B-Chat")
        >>> tokenizer = AutoTokenizer.from_pretrained("inceptionai/Jais-2-8B-Chat")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```r-   )r;   Úforward)rA   Úsuper_kwargsrB   s     €r/   rN   zJais2ForCausalLM.forwardO   s   ø€ ô( ‰w‰Ñ. Ñ.Ð.r.   )r    r!   r"   r   r   rN   rC   rD   s   @r/   rL   rL   N   s   ø„ ØØó/ó ó ô/r.   rL   )r   rH   rL   rF   )Útorch.nnr=   Úhuggingface_hub.dataclassesr   Úutilsr   r   Úllama.configuration_llamar   Úllama.modeling_llamar	   r
   r   r   Únemotron.modeling_nemotronr   r   r1   r5   rF   rH   rL   Ú__all__r-   r.   r/   ú<module>rW      s•   ðõ  Ý .ç 5Ý 3÷ó õ 5ñ Ð7Ô8Øô&�+ó &ó ó 9ð&ô2	ˆ{ô 	ôdÐ)ô dô	Ð/ô 	ôP�ô Pô/Ð'ô /ò0�r.   