
    ^j!                        d dl Z d dlmZ d dlZd dlmZ ddlmZ ddlm	Z	 ddl
mZmZ ddlmZ d	d
lmZ d	dlmZmZmZmZmZmZmZmZmZmZ d	dlmZ ddlmZ  ej@                  e!      Z"dZ#dZ$ G d de      Z%d Z& G d de      Z' G d de      Z( G d de      Z) G d de      Z* G d de      Z+ G d de      Z, G d  d!e      Z- G d" d#e      Z. G d$ d%e      Z/g d&Z0y)'    N)Callable)nn   )initialization)Cache)ALL_ATTENTION_FUNCTIONSPreTrainedModel)logging   )GemmaForCausalLM)
LlamaAttentionLlamaDecoderLayerLlamaForQuestionAnsweringLlamaForSequenceClassificationLlamaForTokenClassification
LlamaModelLlamaPreTrainedModelLlamaRotaryEmbeddingapply_rotary_pos_embeager_attention_forward)
MistralMLP   )DiffLlamaConfigzkajuma/DiffLlama-0.3B-handcutr   c                       e Zd Zy)DiffLlamaMLPN__name__
__module____qualname__     z/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/diffllama/modular_diffllama.pyr   r   2       r!   r   c                 >    ddt        j                  d| z        z  z
  S )Ng?g333333?g333333ӿ)mathexp)	layer_idxs    r"   lambda_init_fnr(   6   s     txxy 01111r!   c                       e Zd Zy)DiffLlamaRotaryEmbeddingNr   r    r!   r"   r*   r*   :   r#   r!   r*   c                        e Zd ZdZddededz  f fdZ	 	 ddej                  de	ej                  ej                  f   dej                  dz  d	e
dz  d
e	ej                  ej                  f   f
dZ xZS )DiffLlamaAttentionu  Multi-headed differential attention (https://huggingface.co/papers/2410.05258).

    Computes ``(softmax(Q1 K1ᵀ) - λ · softmax(Q2 K2ᵀ)) · V`` as two standard attention calls
    sharing Q and K over the two halves of V. The two-call structure is ~30% faster than the
    V-doubling shortcut at production shapes, since asymmetric V (``head_dim_v != head_dim_q``)
    forces SDPA off Flash/cuDNN onto the memory-efficient/math kernel; Flash Attention 2 also
    requires ``head_dim_v == head_dim_q``.
    Nconfigr'   c                 x   t         |   ||       |j                  dkD  rt        d      |j                  |j                  dz  dk7  rt        d|j                   d      t        |      | _        t        j                  t        j                  d|j                  | j                  f            | _        t        j                  t        j                  d|j                  | j                  f            | _        t        j                  t        j                  d|j                  | j                  f            | _        t        j                  t        j                  d|j                  | j                  f            | _        t        j"                  d| j                  z  |j$                  d	      | _        y )
N        zDiffLlama does not support `attention_dropout > 0`: the differential attention mechanism has no paper-defined dropout semantics.r   r   zDiffLlama requires `num_key_value_heads` to be even (and at least 2): the two-call differential attention splits the value tensor along KV heads, got .)sizeF)epselementwise_affine)super__init__attention_dropout
ValueErrornum_key_value_headsr(   lambda_initr   	Parametertorchnormallambda_std_devhead_dim	lambda_q1	lambda_k1	lambda_q2	lambda_k2RMSNormrms_norm_eps	groupnormselfr-   r'   	__class__s      r"   r5   zDiffLlamaAttention.__init__H   se   +
 ##c)D  %%-1K1Ka1OST1TVV\VpVpUqqrt  *)4ell1f6K6KSWS`S`Rb&cdell1f6K6KSWS`S`Rb&cdell1f6K6KSWS`S`Rb&cdell1f6K6KSWS`S`Rb&cdA$56;N;Nchir!   hidden_statesposition_embeddingsattention_maskpast_key_valuesreturnc                    |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }	| j                  |      j                  |      j	                  dd      }
|\  }}t        ||	||      \  }}	| |j                  |	|
| j                        \  }	}
d t        j                  |
dd      D        \  }}t        j                  | j                  j                  t               } || ||	||fd| j"                  d|\  }} || ||	||fd| j"                  d|\  }}t        j$                  ||gd      }t        j                  |dd      \  }}t        j&                  t        j(                  | j*                  | j,                  z  dt        j.                              j1                  |j2                        }t        j&                  t        j(                  | j4                  | j6                  z  dt        j.                              j1                  |j2                        }||z
  | j8                  z   }|||z  z
  }d| j8                  z
  | j;                  |      z  } |j<                  g |d }| j?                  |      }||fS )	Nr   r   c              3   D   K   | ]  }|j                  d dd d         yw)r   r   N)repeat).0vs     r"   	<genexpr>z-DiffLlamaAttention.forward.<locals>.<genexpr>x   s     'jAq!(<'js    )dimr/   )dropoutscaling)rU   dtype) shaper>   q_projview	transposek_projv_projr   updater'   r;   chunkr   get_interfacer-   _attn_implementationr   rW   catr&   sumr?   r@   float32torX   rA   rB   r9   rE   reshapeo_proj)rG   rI   rJ   rK   rL   kwargsinput_shapehidden_shapequery_states
key_statesvalue_statescossinvalue_states1value_states2attention_interfaceattn_output1attn_weightsattn_output2_attn_outputlambda_1lambda_2lambda_fulls                           r"   forwardzDiffLlamaAttention.forwarda   s    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST&S#7jRUWZ#[ j&'6'='=j,X\XfXf'g$J (kekkR^`aghFi'j$}(?(M(MKK,,.E)
 &9	&
 LL	&
 	&
"l .	
 LL	
 	
a ii| <"E &+[[aQ%G"l99UYYt~~'FBV[VcVcdehh
 99UYYt~~'FBV[VcVcdehh
 )D,<,<<"[<%??4+++t~~k/JJ)k));;;;kk+.L((r!   )N)NN)r   r   r   __doc__r   intr5   r;   Tensortupler   r|   __classcell__rH   s   @r"   r,   r,   >   s    j j3: j: /3(,C)||C) #5<<#=>C) t+	C)
 C) 
u||U\\)	*C)r!   r,   c                   (     e Zd Zdedef fdZ xZS )DiffLlamaDecoderLayerr-   r'   c                 J    t         |   ||       t        ||      | _        y )N)r-   r'   )r4   r5   r,   	self_attnrF   s      r"   r5   zDiffLlamaDecoderLayer.__init__   s     ++6YOr!   )r   r   r   r   r~   r5   r   r   s   @r"   r   r      s    P P3 P Pr!   r   c                   :    e Zd Z ej                         d        Zy)DiffLlamaPreTrainedModelc                    t        j                  | |       t        |t              rt	        j
                  |j                  d| j                  j                         t	        j
                  |j                  d| j                  j                         t	        j
                  |j                  d| j                  j                         t	        j
                  |j                  d| j                  j                         y y )Nr   )r	   _init_weights
isinstancer,   initnormal_r?   r-   r=   r@   rA   rB   )rG   modules     r"   r   z&DiffLlamaPreTrainedModel._init_weights   s    %%dF3f01LL))1dkk.H.HILL))1dkk.H.HILL))1dkk.H.HILL))1dkk.H.HI	 2r!   N)r   r   r   r;   no_gradr   r    r!   r"   r   r      s    U]]_J Jr!   r   c                       e Zd Zy)DiffLlamaModelNr   r    r!   r"   r   r      r#   r!   r   c                       e Zd Zy)DiffLlamaForCausalLMNr   r    r!   r"   r   r      r#   r!   r   c                       e Zd Zy)"DiffLlamaForSequenceClassificationNr   r    r!   r"   r   r      r#   r!   r   c                       e Zd Zy)DiffLlamaForQuestionAnsweringNr   r    r!   r"   r   r      r#   r!   r   c                       e Zd Zy)DiffLlamaForTokenClassificationNr   r    r!   r"   r   r      r#   r!   r   )r   r   r   r   r   r   )1r%   collections.abcr   r;   r    r   r   cache_utilsr   modeling_utilsr   r	   utilsr
   gemma.modeling_gemmar   llama.modeling_llamar   r   r   r   r   r   r   r   r   r   mistral.modeling_mistralr   configuration_diffllamar   
get_loggerr   logger_CHECKPOINT_FOR_DOC_CONFIG_FOR_DOCr   r(   r*   r,   r   r   r   r   r   r   r   __all__r    r!   r"   <module>r      s   "  $   &   F  3   2 4 
		H	%5 #	: 	2	3 	f) f)RP- PJ3 J	Z 		+ 		)G 		$= 		&A 	r!   