Ë
    ÛÓ^j³!  ã                   óÊ  — d dl Z d dlmZ d dlZd dlmZ ddlmZ ddlm	Z	 ddl
mZmZ ddlmZ d	d
lmZ d	dlmZmZmZmZmZmZmZmZmZmZ d	dlmZ ddlmZ  ej@                  e!«      Z"dZ#dZ$ G d„ de«      Z%d„ Z& G d„ de«      Z' G d„ de«      Z( G d„ de«      Z) G d„ de«      Z* G d„ de«      Z+ G d„ de«      Z, G d „ d!e«      Z- G d"„ d#e«      Z. G d$„ d%e«      Z/g d&¢Z0y)'é    N)ÚCallable)Únné   )Úinitialization)ÚCache)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)Úloggingé   )ÚGemmaForCausalLM)
ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaForQuestionAnsweringÚLlamaForSequenceClassificationÚLlamaForTokenClassificationÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)Ú
MistralMLPé   )ÚDiffLlamaConfigzkajuma/DiffLlama-0.3B-handcutr   c                   ó   — e Zd Zy)ÚDiffLlamaMLPN©Ú__name__Ú
__module__Ú__qualname__© ó    úz/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/diffllama/modular_diffllama.pyr   r   2   ó   „ Ør!   r   c                 ó>   — ddt        j                  d| z  «      z  z
  S )Ngš™™™™™é?g333333ã?g333333Ó¿)ÚmathÚexp)Ú	layer_idxs    r"   Úlambda_init_fnr(   6   s    € Ø�”t—x‘x  yÑ 0Ó1Ñ1Ñ1Ð1r!   c                   ó   — e Zd Zy)ÚDiffLlamaRotaryEmbeddingNr   r    r!   r"   r*   r*   :   r#   r!   r*   c                   óî   ‡ — e Zd ZdZddededz  fˆ fd„Z	 	 ddej                  de	ej                  ej                  f   dej                  dz  d	e
dz  d
e	ej                  ej                  f   f
d„Zˆ xZS )ÚDiffLlamaAttentionuÿ  Multi-headed differential attention (https://huggingface.co/papers/2410.05258).

    Computes ``(softmax(Q1 K1áµ€) - Î» Â· softmax(Q2 K2áµ€)) Â· V`` as two standard attention calls
    sharing Q and K over the two halves of V. The two-call structure is ~30% faster than the
    V-doubling shortcut at production shapes, since asymmetric V (``head_dim_v != head_dim_q``)
    forces SDPA off Flash/cuDNN onto the memory-efficient/math kernel; Flash Attention 2 also
    requires ``head_dim_v == head_dim_q``.
    NÚconfigr'   c                 óx  •— t         ‰| �  ||«       |j                  dkD  rt        d«      ‚|j                  �|j                  dz  dk7  rt        d|j                  › d�«      ‚t        |«      | _        t        j                  t        j                  d|j                  | j                  f¬«      «      | _        t        j                  t        j                  d|j                  | j                  f¬«      «      | _        t        j                  t        j                  d|j                  | j                  f¬«      «      | _        t        j                  t        j                  d|j                  | j                  f¬«      «      | _        t        j"                  d| j                  z  |j$                  d¬	«      | _        y )
Nç        z€DiffLlama does not support `attention_dropout > 0`: the differential attention mechanism has no paper-defined dropout semantics.r   r   z–DiffLlama requires `num_key_value_heads` to be even (and at least 2): the two-call differential attention splits the value tensor along KV heads, got ú.)ÚsizeF)ÚepsÚelementwise_affine)ÚsuperÚ__init__Úattention_dropoutÚ
ValueErrorÚnum_key_value_headsr(   Úlambda_initr   Ú	ParameterÚtorchÚnormalÚlambda_std_devÚhead_dimÚ	lambda_q1Ú	lambda_k1Ú	lambda_q2Ú	lambda_k2ÚRMSNormÚrms_norm_epsÚ	groupnorm©Úselfr-   r'   Ú	__class__s      €r"   r5   zDiffLlamaAttention.__init__H   se  ø€ Ü‰Ñ˜ Ô+ð
 ×#Ñ# cÒ)ÜðDóð ð ×%Ñ%Ð-°×1KÑ1KÈaÑ1OÐSTÒ1TÜðVØV\×VpÑVpÐUqÐqrðtóð ô *¨)Ó4ˆÔÜŸ™¤e§l¡l°1°f×6KÑ6KÐSW×S`ÑS`ÐRbÔ&cÓdˆŒÜŸ™¤e§l¡l°1°f×6KÑ6KÐSW×S`ÑS`ÐRbÔ&cÓdˆŒÜŸ™¤e§l¡l°1°f×6KÑ6KÐSW×S`ÑS`ÐRbÔ&cÓdˆŒÜŸ™¤e§l¡l°1°f×6KÑ6KÐSW×S`ÑS`ÐRbÔ&cÓdˆŒÜŸ™ A¨¯©Ñ$5¸6×;NÑ;NÐchÔiˆ�r!   Úhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valuesÚreturnc                 ó¾  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
d„ t        j                  |
dd¬«      D «       \  }}t        j                  | j                  j                  t         «      } || ||	||fd| j"                  dœ|¤Ž\  }} || ||	||fd| j"                  dœ|¤Ž\  }}t        j$                  ||gd¬«      }t        j                  |dd¬«      \  }}t        j&                  t        j(                  | j*                  | j,                  z  dt        j.                  ¬«      «      j1                  |j2                  «      }t        j&                  t        j(                  | j4                  | j6                  z  dt        j.                  ¬«      «      j1                  |j2                  «      }||z
  | j8                  z   }|||z  z
  }d| j8                  z
  | j;                  |«      z  } |j<                  g |¢d‘­Ž }| j?                  |«      }||fS )	Néÿÿÿÿr   r   c              3   óD   K  — | ]  }|j                  d dd d «      –— Œ y­w)r   r   N)Úrepeat)Ú.0Úvs     r"   ú	<genexpr>z-DiffLlamaAttention.forward.<locals>.<genexpr>x   s   è ø€ Ò'jÀ¨¯©°°A°q¸!×(<Ñ'jùs   ‚ )Údimr/   )ÚdropoutÚscaling)rU   Údtype) Úshaper>   Úq_projÚviewÚ	transposeÚk_projÚv_projr   Úupdater'   r;   Úchunkr   Úget_interfacer-   Ú_attn_implementationr   rW   Úcatr&   Úsumr?   r@   Úfloat32ÚtorX   rA   rB   r9   rE   ÚreshapeÚo_proj)rG   rI   rJ   rK   rL   ÚkwargsÚinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚvalue_states1Úvalue_states2Úattention_interfaceÚattn_output1Úattn_weightsÚattn_output2Ú_Úattn_outputÚlambda_1Úlambda_2Úlambda_fulls                           r"   ÚforwardzDiffLlamaAttention.forwarda   sÐ  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ñ (kÄeÇkÁkÐR^Ð`aÐghÔFiÔ'jÑ$ˆ�}ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ &9ØØØØØð	&
ð Ø—L‘Lñ	&
ð ñ	&
Ñ"ˆ�lñ .ØØØØØð	
ð Ø—L‘Lñ	
ð ñ	
‰ˆ�aô —i‘i ¨|Ð <À"ÔEˆô &+§[¡[°¸aÀQÔ%GÑ"ˆ�lÜ—9‘9œUŸY™Y t§~¡~¸¿¹Ñ'FÈBÔV[×VcÑVcÔdÓe×hÑhØ×Ñó
ˆô —9‘9œUŸY™Y t§~¡~¸¿¹Ñ'FÈBÔV[×VcÑVcÔdÓe×hÑhØ×Ñó
ˆð  Ñ)¨D×,<Ñ,<Ñ<ˆØ" [°<Ñ%?Ñ?ˆØ˜4×+Ñ+Ñ+¨t¯~©~¸kÓ/JÑJˆØ)�k×)Ñ)Ð;¨;Ð;¸Ò;ˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r!   )N)NN)r   r   r   Ú__doc__r   Úintr5   r;   ÚTensorÚtupler   r|   Ú__classcell__©rH   s   @r"   r,   r,   >   sš   ø„ ññj˜ð j¸3À¹:õ jð: /3Ø(,ñC)à—|‘|ðC)ð # 5§<¡<°·±Ð#=Ñ>ðC)ð Ÿ™ tÑ+ð	C)ð
  ™ðC)ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷C)r!   r,   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚDiffLlamaDecoderLayerr-   r'   c                 óJ   •— t         ‰| �  ||«       t        ||¬«      | _        y )N)r-   r'   )r4   r5   r,   Ú	self_attnrF   s      €r"   r5   zDiffLlamaDecoderLayer.__init__¨   s    ø€ Ü‰Ñ˜ Ô+ä+°6ÀYÔOˆ�r!   )r   r   r   r   r~   r5   r�   r‚   s   @r"   r„   r„   §   s   ø„ ðP˜ð P¸3÷ Pñ Pr!   r„   c                   ó:   — e Zd Z ej                  «       d„ «       Zy)ÚDiffLlamaPreTrainedModelc                 óú  — t        j                  | |«       t        |t        «      rÕt	        j
                  |j                  d| j                  j                  «       t	        j
                  |j                  d| j                  j                  «       t	        j
                  |j                  d| j                  j                  «       t	        j
                  |j                  d| j                  j                  «       y y )Nr   )r	   Ú_init_weightsÚ
isinstancer,   ÚinitÚnormal_r?   r-   r=   r@   rA   rB   )rG   Úmodules     r"   rŠ   z&DiffLlamaPreTrainedModel._init_weights¯   s¢   € ä×%Ñ% d¨FÔ3Ü�fÔ0Ô1Ü�L‰L˜×)Ñ)¨1¨d¯k©k×.HÑ.HÔIÜ�L‰L˜×)Ñ)¨1¨d¯k©k×.HÑ.HÔIÜ�L‰L˜×)Ñ)¨1¨d¯k©k×.HÑ.HÔIÜ�L‰L˜×)Ñ)¨1¨d¯k©k×.HÑ.HÕIð	 2r!   N)r   r   r   r;   Úno_gradrŠ   r    r!   r"   rˆ   rˆ   ®   s   „ Ø€U‡]�]ƒ_ñJó ñJr!   rˆ   c                   ó   — e Zd Zy)ÚDiffLlamaModelNr   r    r!   r"   r‘   r‘   ¹   r#   r!   r‘   c                   ó   — e Zd Zy)ÚDiffLlamaForCausalLMNr   r    r!   r"   r“   r“   ½   r#   r!   r“   c                   ó   — e Zd Zy)Ú"DiffLlamaForSequenceClassificationNr   r    r!   r"   r•   r•   Á   r#   r!   r•   c                   ó   — e Zd Zy)ÚDiffLlamaForQuestionAnsweringNr   r    r!   r"   r—   r—   Å   r#   r!   r—   c                   ó   — e Zd Zy)ÚDiffLlamaForTokenClassificationNr   r    r!   r"   r™   r™   É   r#   r!   r™   )rˆ   r‘   r“   r•   r—   r™   )1r%   Úcollections.abcr   r;   r   Ú r   rŒ   Úcache_utilsr   Úmodeling_utilsr   r	   Úutilsr
   Úgemma.modeling_gemmar   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   r   Úmistral.modeling_mistralr   Úconfiguration_diffllamar   Ú
get_loggerr   ÚloggerÚ_CHECKPOINT_FOR_DOCÚ_CONFIG_FOR_DOCr   r(   r*   r,   r„   rˆ   r‘   r“   r•   r—   r™   Ú__all__r    r!   r"   ú<module>r¨      sê   ðó" Ý $ã Ý å &Ý  ß FÝ Ý 3÷÷ ÷ õ 2Ý 4ð 
ˆ×	Ñ	˜HÓ	%€à5Ð Ø#€ô	�:ô 	ò2ô	Ð3ô 	ôf)˜ô f)ôRPÐ-ô PôJÐ3ô Jô	�Zô 	ô	Ð+ô 	ô	Ð)Gô 	ô	Ð$=ô 	ô	Ð&Aô 	ò�r!   