
    ^j                     ^   d dl mZ d dlmZmZ d dlZd dlmc mZ	 d dlmZ d dl
mZ ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZmZmZ ddlmZ ddlmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z* ddl+m,Z,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2  G d dejf                        Z4 ed       G d dejf                               Z5dejl                  de7dejl                  fdZ8 G d d ejf                        Z9 G d! d"ejf                        Z:d# Z;	 dEd$ejf                  d%ejl                  d&ejl                  d'ejl                  d(ejl                  dz  d)e<d*e<d+e&e(   fd,Z=dFd-Z> G d. d/ejf                        Z? G d0 d1e      Z@ G d2 d3ejf                        ZA G d4 d5ejf                        ZB G d6 d7ejf                        ZCe G d8 d9ejf                               ZD G d: d;ejf                        ZEe) G d< d=e$             ZFe) G d> d?eF             ZG e)d@A       G dB dCeFe             ZHg dDZIy)G    )Callable)AnyOptionalN)nn)init   )ACT2FN)CacheDynamicCache)GenerationMixin)use_experts_implementationuse_kernel_forward_from_hub)create_causal_maskcreate_recurrent_attention_mask!create_sliding_window_causal_mask)GradientCheckpointingLayer)MoeCausalLMOutputWithPastMoeModelOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuple)maybe_autocastmerge_with_config_defaults)OutputRecordercapture_outputs   )
ZayaConfigc                        e Zd ZU ej                  ed<   def fdZe	 	 	 	 ddedz  de	d   de
dz  dedz  d	ed
ef   f
d       Z ej                         edd              Z xZS )ZayaRotaryEmbeddinginv_freqconfigc                 t   t         |           |j                  | _        |j                  | _        || _        t        t        |j                              | _        i | _	        | j                  D ]  }| j
                  j                  |   }||d   | j                  |<   | j                  }| j                  |   dk7  rt        | j                  |      } || j
                  |      \  }}| j                  | d|d       | j                  | d|j                         d       t        | | d|        y )	N	rope_typedefault
layer_type	_inv_freqF)
persistent_original_inv_freq_attention_scaling)super__init__max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr&   listsetlayer_typesr(   rope_parameterscompute_default_rope_parametersr   register_bufferclonesetattr)selfr&   r+   rope_paramsrope_init_fncurr_inv_freqcurr_attention_scaling	__class__s          q/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/zaya/modeling_zaya.pyr1   zZayaRotaryEmbedding.__init__1   s6   "("@"@$*$B$B!F$6$6 78** 	UJ++55jAK")4[)ADNN:&%)%I%IL~~j)Y624>>*3MN4@Yc4d1M1  J<y!9=UZ [  J</A!BMDWDWDYfk lDZL(:;=ST	U    Ndeviceztorch.deviceseq_lenr+   returnztorch.Tensorc                 z   | j                   |   d   }| j                   |   j                  dd      }t        | dd      xs | j                  | j                  z  }t        ||z        }d}d|t        j                  d|dt        j                        j                  |t        j                  	      |z  z  z  }	|	|fS )
a{  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        
rope_thetapartial_rotary_factorg      ?head_dimNr      dtype)rE   rN   )r8   getgetattrhidden_sizenum_attention_headsinttorcharangeint64tofloat)
r&   rE   rF   r+   baserJ   rK   dimattention_factorr%   s
             rC   r9   z3ZayaRotaryEmbedding.compute_default_rope_parametersF   s    . %%j1,? & 6 6z B F FG^`c d6:t4h8J8JfNhNh8h(223 U\\!S!5;;?BB&X]XcXcBdgjjk
 )))rD   c                 N   t        | | d      }t        | | d      }|d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d	      5  |j                         |j                         z  j                  dd
      }	t        j                  |	|	fd      }
|
j                         |z  }|
j                         |z  }d d d        j	                  |j                        j	                  |j                        fS # 1 sw Y   AxY w)Nr,   r/   r   r!   mpscpuF)device_typeenabledrL   rZ   rM   )rP   rX   expandshaperW   rE   
isinstancetypestrr   	transposerT   catcossinrN   )r=   xposition_idsr+   r%   attention_scalinginv_freq_expandedposition_ids_expandedr`   freqsembrj   rk   s                rC   forwardzZayaRotaryEmbedding.forwardk   sl    4J<y!9:#DZL8J*KL$T1d]399;BB<CUCUVWCXZ\^_`ccdedldlm ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfkUC 	0&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')//C'')//C		0 vvAGGv$cff177f&;;;	0 	0s   *A1FF$NNNNN)__name__
__module____qualname__rT   Tensor__annotations__r"   r1   staticmethodr   rS   rg   tuplerX   r9   no_gradr   rs   __classcell__rB   s   @rC   r$   r$   .   s    llUz U* $(+/"!%	"*T!"*("* t"* $J	"*
 
~u$	%"* "*H U]]_<  <rD   r$   RMSNormc                   h     e Zd Zddeddf fdZdej                  dej                  fdZd Z xZ	S )	ZayaRMSNormepsrG   Nc                     t         |           t        j                  t	        j
                  |            | _        || _        y)z:
        ZayaRMSNorm is equivalent to T5LayerNorm
        N)r0   r1   r   	ParameterrT   onesweightvariance_epsilon)r=   rQ   r   rB   s      rC   r1   zZayaRMSNorm.__init__   s1     	ll5::k#:; #rD   hidden_statesc                 "   |j                   }|j                  t        j                        }|j	                  d      j                  dd      }|t        j                  || j                  z         z  }| j                  |j                  |      z  S )NrL   r]   T)keepdim)	rN   rW   rT   float32powmeanrsqrtr   r   )r=   r   input_dtypevariances       rC   rs   zZayaRMSNorm.forward   sy    #))%((7 $$Q',,R,>%Ht?T?T4T(UU{{]--k:::rD   c                 ^    t        | j                  j                         d| j                   S )Nz, eps=)r|   r   rd   r   )r=   s    rC   
extra_reprzZayaRMSNorm.extra_repr   s*    ))*+6$2G2G1HIIrD   )gư>)
rv   rw   rx   rX   r1   rT   ry   rs   r   r~   r   s   @rC   r   r   ~   s7    $ $$ $;U\\ ;ell ;JrD   r   r   n_reprG   c                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r!   N)rd   rc   reshape)r   r   batchnum_key_value_headsslenrK   s         rC   	repeat_kvr      so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TTrD   c                   x     e Zd ZdZdedef fdZ	 d
dej                  de	dz  dej                  dz  fd	Z
 xZS )ZayaCCAProjectionav  
    Projects hidden states into attention q/k/v states with ZAYA's Compressed Convolutional Attention (CCA) path.
    See https://huggingface.co/papers/2510.04476.

    This follows the usual q/k/v projection flow, with three ZAYA-specific changes: q/k are mixed by a causal 1D
    convolution, q/k keep residual projection paths, and v uses a delayed recurrent state.
    r&   	layer_idxc                    t         |           || _        || _        |j                  | _        |j
                  | _        |j                  | _        | j                  dz
  | j                  dz
  z   | _	        |j                  | _
        |j                  | _        |j                  | _        | j                  | j                  z  | _        | j                  | j                  z  }| j                  | j                  z  }t        j                  | j                  || j                  j                         | _        t        j                  | j                  || j                  j                         | _        t        j                  | j                  |dz  | j                  j                         | _        t        j                  | j                  |dz  | j                  j                         | _        ||z   }t        j*                  ||| j                  |dd      | _        t        j*                  ||| j                  | j                  | j                  z   dd      | _        y )Nr!   biasrL   r   )in_channelsout_channelskernel_sizegroupspaddingstride)r0   r1   r&   r   rQ   	cca_time0depthwise_kernel_size	cca_time1grouped_kernel_sizeconv_kernel_sizer   rR   rK   num_key_value_groupsr   Linearattention_biasq_projk_projv_proj_currentv_proj_delayedConv1dconv_qk_depthwiseconv_qk_grouped)r=   r&   r   query_hidden_sizekey_value_hidden_sizeconv_channelsrB   s         rC   r1   zZayaCCAProjection.__init__   s   "!--%+%5%5"#)#3#3 !%!;!;a!?DD\D\_`D` a#)#=#= #)#=#= $($<$<@X@X$X! 44t}}D $ 8 84== Hii 0 02C$++JdJdeii 0 02GdkkNhNhi ii(8(8:OST:T[_[f[f[u[uv ii(8(8:OST:T[_[f[f[u[uv-0AA!#%&22 "
  "yy%&00,,t/G/GG 
rD   Nr   past_key_values	conv_maskc                 8   |(||d d d d d f   j                  |j                        z  }|j                  d d }g |d| j                  }| j	                  |      }| j                  |      }t        j                  ||gd      } |j                  | }	 |j                  | j                  dd      }
t        |
| j                        j                  dd      }
|	|
z   dz  }	 |	j                  g |d| j                  | j                   j                  d      }
|j                  dd      }|d uxr |j                  | j                        }|r@|j                  | j                     j                   d   }t        j                  ||gd      }n"t#        j$                  || j&                  df      }|b|d| j&                   d f   }t#        j$                  || j&                  |j                  d   z
  df      }|j)                  || j                         | j+                  |      }| j-                  |      j                  dd      }|	j                  d   |	j                  d   z  } |dd |f   j                  | |	z   } |d|d f   j                  | |
z   }| j/                  |      }| j1                  |      }|r6|j                  | j                     j2                  d   j5                  d      }n/| j1                  |j7                  |d   d| j8                              }t        j                  ||d d d df   gd      }|&|j;                  |d d dd d f   | j                          t        j                  ||gd      j                  | }|||fS )	Nr]   rb   r!   rL         ?r   .)rW   rN   rd   rK   r   r   rT   ri   viewrh   r   r   r   has_previous_stater   layersconv_statesFpadr   update_conv_stater   r   r   r   recurrent_states	unsqueeze	new_zerosrQ   update_recurrent_state)r=   r   r   r   input_shapehidden_shapeprojected_queriesprojected_keys	qk_statesquery_residualkey_residualuse_precomputed_statescached_qk_statesnew_conv_stater   querykeyvalue_currentdelayed_v_staterecurrent_v_statevalue_delayedvalues                         rC   rs   zZayaCCAProjection.forward   s     )IaDj,A,D,D]EXEX,YYM#))#2.88b8$--8 KK6]3II0.ArJ	/*//>*~**L9CCAqI t/H/HISSTUWXY(<73>*~**fKffT=V=VfX\XeXefkkprks''1-	!0!<!sAcAcdhdrdrAs!.55dnnEQQRST		#3Y"?RHIi$*?*?)CDI&&sT-B-B,B,D'DENUU>D4I4INL`L`acLd4dfg3hiN--ndnnM**95	((3==aC	*004~7K7KB7OO7	#1 111277FW5i.//055|D|S ++M:--m<! / 6 6t~~ F W WXY Z d def g $ 3 3M4K4KKXYN\]_c_o_o4p q		#4oa"f6M"NTUV&22?1b!83Ldnn]F		=-8bAFFUc5  rD   ru   )rv   rw   rx   __doc__r"   rS   r1   rT   ry   r
   rs   r~   r   s   @rC   r   r      sS    (
z (
c (
\ *.	9!||9! 9! <<$&	9!rD   r   c                        e Zd ZdZdef fdZdej                  dej                  deej                  ej                  f   fdZ	 xZ
S )
ZayaQKNormzf
    L2-normalizes q/k states to sqrt(head_dim) and applies ZAYA's learned per-KV-head key scale.
    r&   c                     t         |           |j                  dz  | _        t	        j
                  t        j                  |j                              | _	        y )Nr   )
r0   r1   rK   head_dim_scaler   r   rT   zerosr   tempr=   r&   rB   s     rC   r1   zZayaQKNorm.__init__  s>    $oos2LLV-G-G!HI	rD   query_states
key_statesrG   c                 X   t        j                  |j                        j                  }|| j                  |j                  ddd      j                  |      z  z  }|| j                  |j                  ddd      j                  |      z  z  }|| j                  d d d d d f   z  }||fS )NrL   r]   T)prZ   r   )rT   finforN   r   r   norm	clamp_minr   )r=   r   r   norm_epss       rC   rs   zZayaQKNorm.forward  s    ;;|11266#,"3"3aR"3"N"X"XYa"bb
  *//A2t/"L"V"VW_"``

  $))D$4,?"@@
Z''rD   )rv   rw   rx   r   r"   r1   rT   ry   r|   rs   r~   r   s   @rC   r   r     sP    Jz J
	(ELL 	(ell 	(uUZUaUachcocoUoOp 	(rD   r   c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..Nr]   rL   rb   )rd   rT   ri   )rl   x1x2s      rC   rotate_halfr   $  sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''rD   moduler   r   r   attention_maskscalingdropoutkwargsc                    t        || j                        }t        || j                        }	t        j                  ||j	                  dd            |z  }
||
|z   }
t
        j                  j                  |
dt        j                        j                  |j                        }
t
        j                  j                  |
|| j                        }
t        j                  |
|	      }|j	                  dd      j                         }||
fS )NrL   r   r]   )rZ   rN   )r   trainingr!   )r   r   rT   matmulrh   r   
functionalsoftmaxr   rW   rN   r   r   
contiguous)r   r   r   r   r   r   r   r   r   value_statesattn_weightsattn_outputs               rC   eager_attention_forwardr   +  s     3 ; ;<JUF$?$?@L<<z';';Aq'ABWLL!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|\:K''1-88:K$$rD   c                 h   |j                  |      }|j                  |      }|j                  d   }| dd|f   | d|df   }}|dd|f   |d|df   }	}||z  t        |      |z  z   }
||z  t        |      |z  z   }t        j                  |
|gd      }
t        j                  ||	gd      }|
|fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Removes the interleaving of cos and sin from GLM

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    r]   .Nrb   )r   rd   r   rT   ri   )qkrj   rk   unsqueeze_dim
rotary_dimq_rotq_passk_rotk_passq_embedk_embeds               rC   apply_rotary_pos_embr  E  s    ( --
&C
--
&C 2Jc;J;&'3
+;)<6Ec;J;&'3
+;)<6E s{{51C78Gs{{51C78G ii&)r2Gii&)r2GGrD   c                        e Zd ZdZdedef fdZ	 	 	 ddej                  de	e
ef   dz  dedz  d	eej                  ej                  f   dz  d
ee   deej                  ej                  dz  f   fdZ xZS )ZayaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr&   r   c                    t         |           || _        || _        t	        |d|j
                  |j                  z        | _        |j                  |j                  z  | _	        |j                  | _        | j                  dz  | _
        |j                  | _        d| _        t        j                  |j                  | j                  z  |j
                  |j                        | _        t#        | j                  |      | _        |j&                  |   | _        | j(                  dk(  r|j*                  nd | _        |j
                  | _        |j                  | _        t-        |      | _        y )NrK   g      Tr   r&   r   hybrid_sliding)r0   r1   r&   r   rP   rQ   rR   rK   r   r   r   attention_dropout	is_causalr   r   r   o_projr   qkv_projr7   r+   sliding_windowr   qk_normr=   r&   r   rB   s      rC   r1   zZayaAttention.__init__n  s-   "
F4F4F&JdJd4de$*$>$>&B\B\$\!#)#=#= }}d*!'!9!9ii&&68J8JQWQfQf
 *;;
 !,,Y77;JZ7Zf33`d!--#)#=#= !&)rD   Nr   r   r   position_embeddingsr   rG   c                    |j                   d d }|xs i }|j                  d      }|j                  d      }	| j                  |||	      \  }
}}| j                  |
|      \  }
}|
j	                  dd      }
|j	                  dd      }|j	                  dd      }|\  }}t        |
|||      \  }
}| |j                  ||| j                        \  }}t        j                  | j                  j                  t              } || |
|||f| j                  sdn| j                  | j                  | j                   d|\  }} |j"                  g |d }| j%                  |      }||fS )Nr]   causalconvr!   rL           )r   r   r  )rd   rO   r  r  rh   r  updater   r   get_interfacer&   _attn_implementationr   r   r  r   r  r   r  )r=   r   r   r   r  r   r   mask_mappingcausal_maskr   r   r   r   rj   rk   attention_interfacer   r   s                     rC   rs   zZayaAttention.forward  s    $))#2.%+"&&x0 $$V,	 26}o_h1i.j,#'<<j#I j#--a3))!Q/
#--a3&S#7jRUWZ#[ j&'6'='=j,X\XfXf'g$J(?(M(MKK,,.E)
 %8
%
  $}}C$2H2HLL..
%
 
%
!\ *k));;;;kk+.L((rD   )NNN)rv   rw   rx   r   r"   rS   r1   rT   ry   dictrg   r   r
   r|   r   r   rs   r~   r   s   @rC   r
  r
  k  s    G*z *c *6 15(,HL.)||.) S#X-.) 	.)
 #5<<#=>E.) +,.) 
u||U\\D00	1.)rD   r
  c                       e Zd Zdedef fdZ	 	 	 	 ddej                  dej                  dz  dee	e
f   dz  dedz  d	eej                  ej                  f   dz  d
ee   deej                  ej                  dz  f   fdZ xZS )ZayaDecoderLayerr&   r   c                    t         |           |j                  | _        t        ||      | _        t        ||      | _        t        |j                  |j                        | _	        t        |j                  |j                        | _
        t        |j                        | _        t        |j                        | _        y )Nr  r   )r0   r1   rQ   r
  	self_attnZayaSparseMoeBlockmlpr   rms_norm_epsinput_layernormpost_attention_layernormZayaResidualScalingpost_attention_residual_scalepost_mlp_residual_scaler  s      rC   r1   zZayaDecoderLayer.__init__  s    !--&f	J%fi8*6+=+=6CVCVW(3F4F4FFL_L_(`%-@ASAS-T*':6;M;M'N$rD   Nr   prev_router_hidden_statesr   r   r  r   rG   c                    |}| j                  |j                  | j                   j                  j                              } | j                  d||||d|\  }}| j                  ||      }| j                  |j                  | j                  j                  j                              }| j                  ||      \  }}| j                  ||      }||fS )NrM   )r   r   r   r   )	r)  rW   r   rN   r%  r,  r*  r'  r-  )	r=   r   r.  r   r   r  r   residual_s	            rC   rs   zZayaDecoderLayer.forward  s     ! ,,X[[t?S?S?Z?Z?`?`[-ab)4>> 
')+ 3	

 
q 55mXN55hkkHeHeHlHlHrHrk6st3788%4
00
 44]HM777rD   rt   )rv   rw   rx   r"   rS   r1   rT   ry   r   rg   r   r
   r|   r   r   rs   r~   r   s   @rC   r"  r"    s    	Oz 	Oc 	O :>04(,HL 8|| 8 $)<<$#6 8 S#X-	 8
  8 #5<<#=>E 8 +, 8 
u||U\\D00	1 8rD   r"  c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )r+  rQ   c                    t         |           t        j                  t	        j
                  |            | _        t        j                  t	        j                  |            | _        t        j                  t	        j
                  |            | _	        t        j                  t	        j                  |            | _
        y ru   )r0   r1   r   r   rT   r   hidden_states_scaler   hidden_states_biasresidual_scaleresidual_bias)r=   rQ   rB   s     rC   r1   zZayaResidualScaling.__init__  sv    #%<<

;0G#H "$,,u{{;/G"H ll5::k+BC\\%++k*BCrD   r   r1  c                 |    || j                   z   | j                  z  }|| j                  z   | j                  z  }||z   S ru   )r6  r5  r8  r7  )r=   r   r1  s      rC   rs   zZayaResidualScaling.forward  sC    &)@)@@DD\D\\t111T5H5HHx''rD   )	rv   rw   rx   rS   r1   rT   ry   rs   r~   r   s   @rC   r+  r+    s,    DC D(U\\ (U\\ (rD   r+  c                   d     e Zd Zdededef fdZdej                  dej                  fdZ xZ	S )ZayaRouterMLPrQ   num_expertsr(  c                 &   t         |           t        ||      | _        t	        j
                  ||d      | _        t	        j
                  ||d      | _        t	        j
                  ||d      | _        t	        j                         | _
        y )Nr$  Tr   F)r0   r1   r   r   r   r   fc1fc2out_projGELUact_fn)r=   rQ   r<  r(  rB   s       rC   r1   zZayaRouterMLP.__init__  se    >	99[+DA99[+DA		+{GggirD   r   rG   c                     | j                  |      }| j                  | j                  |            }| j                  | j                  |            }| j	                  |      S ru   )r   rB  r>  r?  r@  )r=   r   s     rC   rs   zZayaRouterMLP.forward  sM    		-0DHH]$;<DHH]$;<}}]++rD   )
rv   rw   rx   rS   rX   r1   rT   ry   rs   r~   r   s   @rC   r;  r;    s8     C  c    ,U\\ ,ell ,rD   r;  c                        e Zd Zdeddf fdZ	 ddej                  dej                  dz  deej                  ej                  ej                  ej                  f   fdZ xZ	S )	
ZayaRouterr   rG   Nc                    t         |           || _        |j                  | _        || _        |j
                  | _        |j
                  dz   | _        |j                  | _        |j                  | _	        t        j                  | j                  | j                  d      | _        | j                  dk7  | _        | j                  r7t        j                  t        j                   | j                              | _        t%        | j                  | j                  |j&                        | _        | j+                  dt        j,                  | j                  t        j.                               d| j0                  d<   y )	Nr!   Tr   r   balancing_biasesrM         r]   )r0   r1   r&   rQ   r   r<  num_router_classesnum_experts_per_toktop_krouter_hidden_sizer   r   	down_projuse_edar   rT   r   router_states_scaler;  r(  
router_mlpr:   r   r   rG  r  s      rC   r1   zZayaRouter.__init__  s   
 	!--"!--"("4"4q"8//
"(";";4#3#3T5L5LSWX~~*<<')||EJJt?V?V4W'XD$'(?(?AXAXZ`ZmZmn/T=T=T\a\i\i1jk$(b!rD   r   router_statesc                    d| j                   f}|j                  d   }| j                  |      }| j                  r|||| j                  z  z   }|d d | d f   j                         }| j                  |      }t        j                  |d      }|j                         j                  t        j                        | j                  z   }	t        j                  |	| j                   d      \  }
}t        j                  |d|      }|| j                  j                   k(  }|j#                  |d      }|j#                  |d      }|j%                  d| j&                        |j%                  |      |j%                  |      |fS )Nr]   r!   rb   rL   )rZ   indexr   )rK  rd   rM  rN  rO  r;   rP  rT   r   detachrW   r   rG  topkgatherr&   r<  masked_fillr   rI  )r=   r   rQ  final_shape
seq_lengthrouter_hidden_statesrouter_hidden_states_nextrouter_logitsrouter_probsbiased_router_probsr2  router_indicesskip_experts                rC   rs   zZayaRouter.forward"  s`   
 4::&"((+
#~~m<<<M5#7-$JbJb:b#b $8ZKL$I$O$O$Q!(<=}}];*11366u}}EH]H]]!JJ':DJJBO>||La~N %(?(??#//Q?'33KC !!"d&=&=>  -"";/%	
 	
rD   ru   
rv   rw   rx   rS   r1   rT   ry   r|   rs   r~   r   s   @rC   rE  rE    sm    ) ) 
	)> .2
||
 ||d*
 
u||U\\5<<E	F	
rD   rE  c                        e Zd ZdZ fdZdej                  dej                  dej                  dej                  fdZ xZS )ZayaExpertsz2Collection of expert weights stored as 3D tensors.c                    t         |           |j                  | _        |j                  | _        |j
                  | _        t        j                  t        j                  | j                  d| j                  z  | j                              | _        t        j                  t        j                  | j                  | j                  | j                              | _        t        |j                     | _        y )NrL   )r0   r1   r<  rQ   
hidden_dimmoe_intermediate_sizeintermediate_dimr   r   rT   emptygate_up_projrM  r	   
hidden_actrB  r   s     rC   r1   zZayaExperts.__init__H  s    !-- ,, & < <LLT5E5Eq4K`K`G`bfbqbq)rsekk$2B2BDOOUYUjUj&klV../rD   r   top_k_indextop_k_weightsrG   c                 f   t        j                  |      }t        j                         5  t         j                  j                  j                  || j                        }|j                  ddd      }t        j                  |j                  d      d      j                         }d d d        D ]  }|d   }|| j                  k(  rt        j                  |         \  }}	||	   }
t        j                  j                  |
| j                  |         j                  dd      \  }}| j                  |      |z  }t        j                  j                  || j                   |         }|||	|d f   z  }|j#                  d|	|j%                  |j&                                |S # 1 sw Y   xY w)N)num_classesrL   r!   r   )r]   r   rb   r]   )rT   
zeros_liker}   r   r   one_hotr<  permutegreatersumnonzerowherelinearri  chunkrB  rM  
index_add_rW   rN   )r=   r   rk  rl  final_hidden_statesexpert_mask
expert_hit
expert_idx	top_k_pos	token_idxcurrent_stategateupcurrent_hidden_statess                 rC   rs   zZayaExperts.forwardQ  s    $..}=]]_ 	S((--55ktO_O_5`K%--aA6K{8'DaHPPRJ	S
 % 
	nJ#AJT---#(;;{:/F#G Iy))4M}}++M4;L;LZ;XY__`agi_jHD"$(KK$5$:!$&MM$8$89NPTP^P^_iPj$k!$9M)U^`dJd<e$e!**1i9N9Q9QReRkRk9lm
	n #"#	S 	Ss   A=F&&F0)	rv   rw   rx   r   r1   rT   ry   rs   r~   r   s   @rC   rc  rc  D  sF    <0#||# \\# ||	#
 
#rD   rc  c            
            e Zd Zdef fdZ	 ddej                  dej                  dz  deej                  ej                  dz  f   fdZ xZ	S )	r&  r   c                 d    t         |           t        ||      | _        t	        |      | _        y ru   )r0   r1   rE  r  rc  expertsr  s      rC   r1   zZayaSparseMoeBlock.__init__m  s(    vy1	"6*rD   Nr   r.  rG   c                     | j                  ||      \  }}}}|j                  \  }}}|j                  ||z  |      }	| j                  |	||      }
|
j                  |||      }
|
|fS )N)rQ  )r  rd   r   r  )r=   r   r.  r2  r]  r_  
batch_sizerY  emb_dimhidden_states_flatexpert_outputs              rC   rs   zZayaSparseMoeBlock.forwardr  s     FJYY)B FO F
B<)B +8*=*='
J*//
Z0GQ%7V%**:z7K777rD   ru   ra  r   s   @rC   r&  r&  l  sY    +# + :>8||8 $)<<$#68 
u||U\\D00	1	8rD   r&  c                        e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZ eed      eed	Z ej(                          fd
       Z xZS )ZayaPreTrainedModelr&   modelTr"  r   Fr   )rS  )r\  r   
attentionsc                 8   t         |   |       t        |t              r}t	        j
                  |j                         t	        j                  |j                         t	        j
                  |j                         t	        j                  |j                         y t        |t              r?t	        j
                  |j                         t	        j                  |j                         y t        |t              r t	        j                  |j                         y t        |t               rZ|j"                  rt	        j
                  |j$                         t	        j                  |j&                         d|j&                  d<   y t        |t(              r[| j*                  j,                  }t	        j.                  |j0                  d|       t	        j.                  |j2                  d|       y t        |t4              r|j6                  D ]  }|j8                  }|j:                  |   dk7  rt<        |j:                  |      } ||j*                  |      \  }}t?        || d      jA                  |       t?        || d      jA                  |        y y )	NrH  r]   r  )r   stdr)   r*   r,   r.   )!r0   _init_weightsre   r+  r   ones_r5  zeros_r6  r7  r8  	ZayaModelinput_hidden_states_scaleinput_hidden_states_biasr   r   rE  rN  rO  rG  rc  r&   initializer_rangenormal_ri  rM  r$   r7   r9   r(   r   rP   copy_)r=   r   r  r+   r?   r@   r2  rB   s          rC   r  z!ZayaPreTrainedModel._init_weights  s   f%f12JJv112KK112JJv,,-KK,,-	*JJv778KK778
+KK$
+~~

6556KK//0*.F##B',++//CLL,,3C@LL))= 34$00 X
%EE##J/9<#6v7G7G
7S#TL#/*#U q:,i 89??N:,.@ ABHHWX 5rD   )rv   rw   rx   r"   rz   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraph_supports_attention_backendr   rE  r"  r
  _can_record_outputsrT   r}   r  r~   r   s   @rC   r  r    sz    &*#+,#4"5N""&'
!<)# U]]_X XrD   r  c                        e Zd Zdef fdZeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   defd                     Z xZS )r  r&   c           	         t         |   |       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                        | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        |j                  |j                        | _        t#        |      | _        d| _        t        j(                  t+        j,                  |j                              | _        t        j(                  t+        j0                  |j                              | _        | j5                          y c c}w )Nr$  r&   F)r0   r1   pad_token_idpadding_idx
vocab_sizer   	EmbeddingrQ   embed_tokens
ModuleListrangenum_hidden_layersr"  r   r   r(  r   r$   
rotary_embgradient_checkpointingr   rT   r   r  r   r  	post_initr  s      rC   r1   zZayaModel.__init__  s    !.. ++LL):):F<N<NPTP`P`ammBGH`H`BabYfi0b
   2 28K8KL	-V<&+#)+ejjASAS6T)U&(*U[[ASAS5T(U% 	 cs   E/N	input_idsr   rm   r   inputs_embeds	use_cacher   rG   c           	         |d u |d uz  rt        d      || j                  |      }|r|t        | j                        }|V||j	                         nd}t        j                  |j                  d   |j                        |z   }|j                  d      }t        |x}	t              s`| j                  ||||dfdfdd	}
t        | j                  j                        D ci c]  }| |
|           }	}t        di |	d
<   |}t        | j                  j                        D ci c]  }|| j                  |||       }}|| j                   z   | j"                  z  j%                  t
        j&                        }d }t)        | j*                        D ]G  \  }}| j                  j                  |   } |||f|	|   |	j-                  d
      d|||   d|\  }}I | j/                  |j%                  | j.                  j0                  j2                              }t5        ||r|      S d       S c c}w c c}w )Nz:You must specify exactly one of input_ids or inputs_embedsr  r   r!   )rE   r&   r  r   r   rm   c                      t        di  S Nr0  r   mask_kwargss   rC   <lambda>z#ZayaModel.forward.<locals>.<lambda>  s    "4"C{"C rD   c                      t        di  S r  r   r  s   rC   r  z#ZayaModel.forward.<locals>.<lambda>  s    *K*Zk*Z rD   hybridr  r  )r  r  )r   r   r  rM   )last_hidden_stater   r0  )
ValueErrorr  r   r&   get_seq_lengthrT   rU   rd   rE   r   re   r   r6   r7   r   r  r  r  rW   r   	enumerater   rO   r   r   rN   r   )r=   r  r   rm   r   r  r  r   past_seen_tokenscausal_mask_mappingmask_creation_functionsr+   r   r  r.  idxdecoder_layerr  s                    @rC   rs   zZayaModel.forward  s    -t";<YZZ  --i8M0*$++>OCRC^==?de <<(;(;A(>}G[G[\_ooL'11!4L?-F++!."0#2 ,K D"Z'#
 UXX\XcXcXoXoTp#FP
?3J?AA# # +J*XK*X'% "$++"9"9:
 |ZPP
 
 ($*G*GG4KiKiimmMM
 %)!"+DKK"8 	C005J 8E)
8 2*=/33F;  !0$7
$C
8 
84M4		  		-"2"29I9I9O9O"2"PQ%+/8O
 	
>B
 	
K#
s   +H<0I)NNNNNN)rv   rw   rx   r"   r1   r   r    r   rT   
LongTensorry   r
   FloatTensorboolr   r   r   rs   r~   r   s   @rC   r  r    s    z $   .2.204(,26!%L
##d*L
 t+L
 &&-	L

 L
 ((4/L
 $;L
 +,L
 
 L
    L
rD   r  zZyphra/ZAYA1-8B)
checkpointc                   L    e Zd ZddiZdZ fdZee	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  d	edz  d
ej                  dz  dej                  dz  dedz  dedz  deej                  z  dee   defd              Zedd       Z xZS )ZayaForCausalLMzlm_head.weightzmodel.embed_tokens.weightTc                    t         |   |       t        |      | _        |j                  | _        t        j                  |j                  |j                  | j                  j                        | _
        | j                          y )Nr   )r0   r1   r  r  r  r   r   rQ   r&   lm_head_biaslm_headr  )r=   r&   r   rB   s      rC   r1   zZayaForCausalLM.__init__"  s]     v&
 ++yy!3!3V5F5FT[[MeMefrD   Nr  r   rm   r   r  labelsr  output_router_logitslogits_to_keepr   rG   c
                    ||n| j                   j                  } | j                  d|||||||d|
}|j                  }t	        |	t
              rt        |	 d      n|	}| j                  |dd|ddf         }d}| | j                  ||| j                  fi |
}t        |||j                  |j                  |j                  |j                        S )a  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, ZayaForCausalLM

        >>> model = ZayaForCausalLM.from_pretrained("meta-zaya/Zaya-2-7b-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("meta-zaya/Zaya-2-7b-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r  r   rm   r   r  r  r  )losslogitsr   r   r  r\  r0  )r&   r  r  r  re   rS   slicer  loss_functionr  r   r   r   r  r\  )r=   r  r   rm   r   r  r  r  r  r  r   outputsr   slice_indicesr  r  s                   rC   rs   zZayaForCausalLM.forward)  s    B %9$D $++JjJj 	 +5$** 	+
)%+'!5	+
 	+
  118B>SV8W~ot4]kmA}a,?@A%4%%ffdooPPD(#33!//))!//
 	
rD   c                    
 | j                         }|||||d

fd
fdd}t        |j                        D ci c]  }| ||           }	}t        di 
|	d<   |	S c c}w )Nr  c                      t        di  S r  r  r  s   rC   r  z;ZayaForCausalLM.create_masks_for_generate.<locals>.<lambda>v  s    0?;? rD   c                      t        di  S r  r  r  s   rC   r  z;ZayaForCausalLM.create_masks_for_generate.<locals>.<lambda>w  s    &G&V+&V rD   r  r  r0  )get_text_configr6   r7   r   )r&   r  r   r   rm   r2  text_configr  r+   r  r  s             @rC   create_masks_for_generatez)ZayaForCausalLM.create_masks_for_generatei  s     ,,.!*,.(
 @V#

 QTT_TkTkPl
BLJ;/
;==
 
  ?MMV	
s   A )	NNNNNNNNr   ru   )rv   rw   rx   _tied_weights_keys_is_statefulr1   r   r   rT   r  ry   r
   r  r  rS   r   r   r   rs   r{   r  r~   r   s   @rC   r  r    s#   *,GHL  .2.204(,26*.!%,0-.<
##d*<
 t+<
 &&-	<

 <
 ((4/<
   4'<
 $;<
 #Tk<
 ell*<
 +,<
 
#<
  <
|  rD   r  )r  r  r  )r  )r!   )Jcollections.abcr   typingr   r   rT   torch.nn.functionalr   r   r   torch.nnr   activationsr	   cache_utilsr
   r   
generationr   integrationsr   r   masking_utilsr   r   r   modeling_layersr   modeling_outputsr   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   utils.genericr   r   utils.output_capturingr   r    configuration_zayar"   Moduler$   r   ry   rS   r   r   r   r   rX   r   r  r
  r"  r+  r;  rE  rc  r&  r  r  r  __all__r0  rD   rC   <module>r     s@  , %        ! . ) S s s 9 Q K F & I I G E *M<")) M<` Y'J")) J (J(	UU\\ 	U# 	U%,, 	Ul!		 l!^( (,( %II%<<% 
% <<	%
 LL4'% % % '(%4#LI)BII I)X,81 ,8^(")) (,BII , <
 <
~ $#")) $# $#N8 80 /X/ /X /Xd b
# b
 b
J ,-_)? _ ._D BrD   