
    ^jp                     b   d Z ddlmZ ddlmZmZ ddlZddlmc m	Z
 ddlZddlmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZmZ ddlmZ ddlmZ ddlmZmZ ddl m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z* ddl+m,Z,m-Z- ddl.m/Z/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6 ddl7m8Z8m9Z9  e$d      e G d de*                    Z: G d de-      Z; G d  d!e9      Z< G d" d#ejz                        Z> G d$ d%ejz                        Z? G d& d'e3      Z@ G d( d)e/      ZA G d* d+ejz                        ZB G d, d-ejz                        ZC G d. d/ejz                        ZD G d0 d1e8      ZE G d2 d3ejz                        ZF G d4 d5e0      ZGe$ G d6 d7e,             ZH e$d       G d8 d9e(eG             ZIg d:ZJy);zPyTorch Zaya model.    )Callable)AnyLiteralN)strict)nn)init   )CacheDynamicCache)PreTrainedConfig)create_causal_maskcreate_recurrent_attention_mask!create_sliding_window_causal_mask)MoeModelOutputWithPast)ROPE_INIT_FUNCTIONS)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstring)OutputRecorder   )AfmoeForCausalLM)LagunaConfig)LagunaModelLagunaRotaryEmbedding)LlamaDecoderLayerLlamaPreTrainedModel	repeat_kv)Phi3Attention)apply_rotary_pos_embeager_attention_forward)Qwen3MoeExpertsQwen3MoeRMSNormzZyphra/ZAYA1-8B)
checkpointc                      e Zd ZU dZdZdZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZedz  ed<   dZedz  ed<   d	Zedz  ed<   dZeee   z  dz  ed<   dZeed<   dZeed<   dZeed<   dZeed<   d	Zeed<   d	Zeed<    e       Z e       Z e       Z e       Z e       Z e       Z  e       Z! e       Z" e       Z# e       Z$ e       Z%d  Z&d! Z'd" Z(y)#
ZayaConfiga  
    lm_head_bias (`bool`, *optional*, defaults to `False`):
        Whether to add a bias to the language modeling head.
    router_hidden_size (`int`, *optional*, defaults to 256):
        Hidden size used by the ZAYA router.
    cca_time0 (`int`, *optional*, defaults to 2):
        First temporal parameter of the CCA projection.
    cca_time1 (`int`, *optional*, defaults to 2):
        Second temporal parameter of the CCA projection.

    ```python
    >>> from transformers import ZayaConfig, ZayaModel

    >>> configuration = ZayaConfig()
    >>> model = ZayaModel(configuration)

    >>> configuration = model.config
    ```
    zayai  
vocab_sizei   moe_intermediate_size   num_attention_headsr   num_key_value_headsTtie_word_embeddingsgh㈵>rms_norm_epsNsliding_windowr   pad_token_idbos_token_idj   eos_token_id   num_experts_per_tok   num_expertsFlm_head_bias   router_hidden_size	cca_time0	cca_time1c                     | j                   dg| j                  z  nt        | j                         | _         ddddddddd}| j                  || _        t	        j
                  | fi |ddd	hi y )
Nhybriddefaultg    SA      ?)	rope_type
rope_thetapartial_rotary_factorg     @r?   hybrid_slidingignore_keys_at_rope_validationrF   )layer_typesnum_hidden_layerslistrope_parametersr   __post_init__)selfkwargsdefault_rope_paramss      p/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/zaya/modular_zaya.pyrL   zZayaConfig.__post_init__n   s    BFBRBRBZH:(>(>>`deieueu`v ')), '&),Z
 '#6D &&tsvsW_aqVrs    c                     |S N )rM   rN   s     rP   convert_rope_params_to_dictz&ZayaConfig.convert_rope_params_to_dict   s    rQ   c                     | j                   dk7  rt        d      | j                  | j                  z  dk7  rt        d      d| j                  v r| j
                  t        d      y y )Nr5   z5ZAYA currently supports `num_experts_per_tok=1` only.r   zB`num_attention_heads` must be a multiple of `num_key_value_heads`.rF   zJ`sliding_window` must be set when `layer_types` contains `hybrid_sliding`.)r6   
ValueErrorr,   r-   rH   r0   )rM   s    rP   validate_architecturez ZayaConfig.validate_architecture   so    ##q(TUU##d&>&>>!Cabbt///D4G4G4Oijj 5P/rQ   ))__name__
__module____qualname____doc__
model_typer)   int__annotations__r*   r,   r-   r.   boolr/   floatr0   r1   r2   r4   rJ   r6   r8   r9   r;   r<   r=   AttributeErrorbase_model_tp_planbase_model_pp_planintermediate_sizeshared_expert_intermediate_sizerouter_aux_loss_coefnum_attention_heads_per_layergatingmlp_layer_typesmoe_routed_scaling_factor moe_apply_router_weight_on_inputmoe_router_logit_softcappingrL   rU   rX   rT   rQ   rP   r'   r'   3   sA   ( JJ!%3%     $$L%!%NC$J% L#*  L#* +.L#S	/D(.  KL$!!IsIs
 ()')&(&4&6#)+$2$4!F$&O . 0'5'7$#1#3 t(krQ   r'   c                       e Zd Zy)ZayaRotaryEmbeddingNrY   rZ   r[   rT   rQ   rP   ro   ro          rQ   ro   c                       e Zd Zy)ZayaRMSNormNrp   rT   rQ   rP   rs   rs      rq   rQ   rs   c                   x     e Zd ZdZdedef fdZ	 d
dej                  de	dz  dej                  dz  fd	Z
 xZS )ZayaCCAProjectionav  
    Projects hidden states into attention q/k/v states with ZAYA's Compressed Convolutional Attention (CCA) path.
    See https://huggingface.co/papers/2510.04476.

    This follows the usual q/k/v projection flow, with three ZAYA-specific changes: q/k are mixed by a causal 1D
    convolution, q/k keep residual projection paths, and v uses a delayed recurrent state.
    config	layer_idxc                    t         |           || _        || _        |j                  | _        |j
                  | _        |j                  | _        | j                  dz
  | j                  dz
  z   | _	        |j                  | _
        |j                  | _        |j                  | _        | j                  | j                  z  | _        | j                  | j                  z  }| j                  | j                  z  }t        j                  | j                  || j                  j                         | _        t        j                  | j                  || j                  j                         | _        t        j                  | j                  |dz  | j                  j                         | _        t        j                  | j                  |dz  | j                  j                         | _        ||z   }t        j*                  ||| j                  |dd      | _        t        j*                  ||| j                  | j                  | j                  z   dd      | _        y )Nr5   biasr   r   )in_channelsout_channelskernel_sizegroupspaddingstride)super__init__rv   rw   hidden_sizer<   depthwise_kernel_sizer=   grouped_kernel_sizeconv_kernel_sizer-   r,   head_dimnum_key_value_groupsr   Linearattention_biasq_projk_projv_proj_currentv_proj_delayedConv1dconv_qk_depthwiseconv_qk_grouped)rM   rv   rw   query_hidden_sizekey_value_hidden_sizeconv_channels	__class__s         rP   r   zZayaCCAProjection.__init__   s   "!--%+%5%5"#)#3#3 !%!;!;a!?DD\D\_`D` a#)#=#= #)#=#= $($<$<@X@X$X! 44t}}D $ 8 84== Hii 0 02C$++JdJdeii 0 02GdkkNhNhi ii(8(8:OST:T[_[f[f[u[uv ii(8(8:OST:T[_[f[f[u[uv-0AA!#%&22 "
  "yy%&00,,t/G/GG 
rQ   Nhidden_statespast_key_values	conv_maskc                 8   |(||d d d d d f   j                  |j                        z  }|j                  d d }g |d| j                  }| j	                  |      }| j                  |      }t        j                  ||gd      } |j                  | }	 |j                  | j                  dd      }
t        |
| j                        j                  dd      }
|	|
z   dz  }	 |	j                  g |d| j                  | j                   j                  d      }
|j                  dd      }|d uxr |j                  | j                        }|r@|j                  | j                     j                   d   }t        j                  ||gd      }n"t#        j$                  || j&                  df      }|b|d| j&                   d f   }t#        j$                  || j&                  |j                  d   z
  df      }|j)                  || j                         | j+                  |      }| j-                  |      j                  dd      }|	j                  d   |	j                  d   z  } |dd |f   j                  | |	z   } |d|d f   j                  | |
z   }| j/                  |      }| j1                  |      }|r6|j                  | j                     j2                  d   j5                  d      }n/| j1                  |j7                  |d   d| j8                              }t        j                  ||d d d df   gd      }|&|j;                  |d d dd d f   | j                          t        j                  ||gd      j                  | }|||fS )	Ndimr5   r   rA   r   .)todtypeshaper   r   r   torchcatview	transposer   r   meanhas_previous_staterw   layersconv_statesFpadr   update_conv_stater   r   r   r   recurrent_states	unsqueeze	new_zerosr   update_recurrent_state)rM   r   r   r   input_shapehidden_shapeprojected_queriesprojected_keys	qk_statesquery_residualkey_residualuse_precomputed_statescached_qk_statesnew_conv_stater   querykeyvalue_currentdelayed_v_staterecurrent_v_statevalue_delayedvalues                         rP   forwardzZayaCCAProjection.forward   s     )IaDj,A,D,D]EXEX,YYM#))#2.88b8$--8 KK6]3II0.ArJ	/*//>*~**L9CCAqI t/H/HISSTUWXY(<73>*~**fKffT=V=VfX\XeXefkkprks''1-	!0!<!sAcAcdhdrdrAs!.55dnnEQQRST		#3Y"?RHIi$*?*?)CDI&&sT-B-B,B,D'DENUU>D4I4INL`L`acLd4dfg3hiN--ndnnM**95	((3==aC	*004~7K7KB7OO7	#1 111277FW5i.//055|D|S ++M:--m<! / 6 6t~~ F W WXY Z d def g $ 3 3M4K4KKXYN\]_c_o_o4p q		#4oa"f6M"NTUV&22?1b!83Ldnn]F		=-8bAFFUc5  rQ   rS   )rY   rZ   r[   r\   r'   r^   r   r   Tensorr
   r   __classcell__r   s   @rP   ru   ru      sS    (
z (
c (
\ *.	9!||9! 9! <<$&	9!rQ   ru   c                        e Zd ZdZdef fdZdej                  dej                  deej                  ej                  f   fdZ	 xZ
S )
ZayaQKNormzf
    L2-normalizes q/k states to sqrt(head_dim) and applies ZAYA's learned per-KV-head key scale.
    rv   c                     t         |           |j                  dz  | _        t	        j
                  t        j                  |j                              | _	        y )NrA   )
r   r   r   head_dim_scaler   	Parameterr   zerosr-   temprM   rv   r   s     rP   r   zZayaQKNorm.__init__  s>    $oos2LLV-G-G!HI	rQ   query_states
key_statesreturnc                 X   t        j                  |j                        j                  }|| j                  |j                  ddd      j                  |      z  z  }|| j                  |j                  ddd      j                  |      z  z  }|| j                  d d d d d f   z  }||fS )Nr   r   T)pr   keepdim)r   finfor   epsr   norm	clamp_minr   )rM   r   r   norm_epss       rP   r   zZayaQKNorm.forward  s    ;;|11266#,"3"3aR"3"N"X"XYa"bb
  *//A2t/"L"V"VW_"``

  $))D$4,?"@@
Z''rQ   )rY   rZ   r[   r\   r'   r   r   r   tupler   r   r   s   @rP   r   r     sP    Jz J
	(ELL 	(ell 	(uUZUaUachcocoUoOp 	(rQ   r   c                        e Zd Zdedef fdZ	 	 	 ddej                  dee	e
f   dz  dedz  deej                  ej                  f   dz  d	ee   d
eej                  ej                  dz  f   fdZ xZS )ZayaAttentionrv   rw   c                    t         |   ||       ~|j                  |   | _        | j                  dk(  r|j                  nd | _        |j
                  | _        |j                  | _        t        j                  |j                  | j                  z  |j
                  |j                        | _        t        |      | _        t        | j                  |      | _        y )NrF   ry   )rv   rw   )r   r   rH   
layer_typer0   r   r,   r   r   r   r   o_projr   qk_normru   rv   qkv_proj)rM   rv   rw   op_sizer   s       rP   r   zZayaAttention.__init__  s    + ,,Y77;JZ7Zf33`d!--#)#=#= ii&&68J8JQWQfQf
 "&));;
rQ   Nr   attention_maskr   position_embeddingsrN   r   c                    |j                   d d }|xs i }|j                  d      }|j                  d      }	| j                  |||	      \  }
}}| j                  |
|      \  }
}|
j	                  dd      }
|j	                  dd      }|j	                  dd      }|\  }}t        |
|||      \  }
}| |j                  ||| j                        \  }}t        j                  | j                  j                  t              } || |
|||f| j                  sdn| j                  | j                  | j                   d|\  }} |j"                  g |d }| j%                  |      }||fS )Nr   causalconvr5   r           )dropoutscalingr0   )r   getr   r   r   r!   updaterw   r   get_interfacerv   _attn_implementationr"   trainingattention_dropoutr   r0   reshaper   )rM   r   r   r   r   rN   r   mask_mappingcausal_maskr   r   r   value_statescossinattention_interfaceattn_outputattn_weightss                     rP   r   zZayaAttention.forward.  s    $))#2.%+"&&x0 $$V,	 26}o_h1i.j,#'<<j#I j#--a3))!Q/
#--a3&S#7jRUWZ#[ j&'6'='=j,X\XfXf'g$J(?(M(MKK,,.E)
 %8
%
  $}}C$2H2HLL..
%
 
%
!\ *k));;;;kk+.L((rQ   )NNNrY   rZ   r[   r'   r^   r   r   r   dictstrr   r
   r   r   r   r   r   r   s   @rP   r   r     s    
z 
c 
( 15(,HL.)||.) S#X-.) 	.)
 #5<<#=>E.) +,.) 
u||U\\D00	1.)rQ   r   c                       e Zd Zdedef fdZ	 	 	 	 ddej                  dej                  dz  dee	e
f   dz  dedz  d	eej                  ej                  f   dz  d
ee   deej                  ej                  dz  f   fdZ xZS )ZayaDecoderLayerrv   rw   c                     t         |   ||       t        ||      | _        t	        |j
                        | _        t	        |j
                        | _        y rS   )r   r   ZayaSparseMoeBlockmlpZayaResidualScalingr   post_attention_residual_scalepost_mlp_residual_scalerM   rv   rw   r   s      rP   r   zZayaDecoderLayer.__init__`  sF    +%fi8-@ASAS-T*':6;M;M'N$rQ   Nr   prev_router_hidden_statesr   r   r   rN   r   c                    |}| j                  |j                  | j                   j                  j                              } | j                  d||||d|\  }}| j                  ||      }| j                  |j                  | j                  j                  j                              }| j                  ||      \  }}| j                  ||      }||fS )Nr   )r   r   r   r   rT   )	input_layernormr   weightr   	self_attnr  post_attention_layernormr   r  )	rM   r   r  r   r   r   rN   residual_s	            rP   r   zZayaDecoderLayer.forwardf  s     ! ,,X[[t?S?S?Z?Z?`?`[-ab)4>> 
')+ 3	

 
q 55mXN55hkkHeHeHlHlHrHrk6st3788%4
00
 44]HM777rQ   )NNNNr   r   s   @rP   r   r   _  s    Oz Oc O :>04(,HL 8|| 8 $)<<$#6 8 S#X-	 8
  8 #5<<#=>E 8 +, 8 
u||U\\D00	1 8rQ   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )r   r   c                    t         |           t        j                  t	        j
                  |            | _        t        j                  t	        j                  |            | _        t        j                  t	        j
                  |            | _	        t        j                  t	        j                  |            | _
        y rS   )r   r   r   r   r   oneshidden_states_scaler   hidden_states_biasresidual_scaleresidual_bias)rM   r   r   s     rP   r   zZayaResidualScaling.__init__  sv    #%<<

;0G#H "$,,u{{;/G"H ll5::k+BC\\%++k*BCrQ   r   r  c                 |    || j                   z   | j                  z  }|| j                  z   | j                  z  }||z   S rS   )r  r  r  r  )rM   r   r  s      rP   r   zZayaResidualScaling.forward  sC    &)@)@@DD\D\\t111T5H5HHx''rQ   )	rY   rZ   r[   r^   r   r   r   r   r   r   s   @rP   r   r     s,    DC D(U\\ (U\\ (rQ   r   c                   d     e Zd Zdededef fdZdej                  dej                  fdZ xZ	S )ZayaRouterMLPr   r8   r/   c                 &   t         |           t        ||      | _        t	        j
                  ||d      | _        t	        j
                  ||d      | _        t	        j
                  ||d      | _        t	        j                         | _
        y )N)r   Try   F)r   r   rs   r   r   r   fc1fc2out_projGELUact_fn)rM   r   r8   r/   r   s       rP   r   zZayaRouterMLP.__init__  se    >	99[+DA99[+DA		+{GggirQ   r   r   c                     | j                  |      }| j                  | j                  |            }| j                  | j                  |            }| j	                  |      S rS   )r   r  r  r  r  )rM   r   s     rP   r   zZayaRouterMLP.forward  sM    		-0DHH]$;<DHH]$;<}}]++rQ   )
rY   rZ   r[   r^   ra   r   r   r   r   r   r   s   @rP   r  r    s8     C  c    ,U\\ ,ell ,rQ   r  c                        e Zd Zdeddf fdZ	 ddej                  dej                  dz  deej                  ej                  ej                  ej                  f   fdZ xZ	S )	
ZayaRouterrw   r   Nc                    t         |           || _        |j                  | _        || _        |j
                  | _        |j
                  dz   | _        |j                  | _        |j                  | _	        t        j                  | j                  | j                  d      | _        | j                  dk7  | _        | j                  r7t        j                  t        j                   | j                              | _        t%        | j                  | j                  |j&                        | _        | j+                  dt        j,                  | j                  t        j.                               d| j0                  d<   y )	Nr5   Try   r   balancing_biasesr        r   )r   r   rv   r   rw   r8   num_router_classesr6   top_kr;   r   r   	down_projuse_edar   r   r  router_states_scaler  r/   
router_mlpregister_bufferr   float32r!  r  s      rP   r   zZayaRouter.__init__  s   
 	!--"!--"("4"4q"8//
"(";";4#3#3T5L5LSWX~~*<<')||EJJt?V?V4W'XD$'(?(?AXAXZ`ZmZmn/T=T=T\a\i\i1jk$(b!rQ   r   router_statesc                    d| j                   f}|j                  d   }| j                  |      }| j                  r|||| j                  z  z   }|d d | d f   j                         }| j                  |      }t        j                  |d      }|j                         j                  t        j                        | j                  z   }	t        j                  |	| j                   d      \  }
}t        j                  |d|      }|| j                  j                   k(  }|j#                  |d      }|j#                  |d      }|j%                  d| j&                        |j%                  |      |j%                  |      |fS )Nr   r5   r   r   )r   indexr   )r$  r   r%  r&  r'  cloner(  r   softmaxdetachr   r*  r!  topkgatherrv   r8   masked_fillr   r#  )rM   r   r+  final_shape
seq_lengthrouter_hidden_statesrouter_hidden_states_nextrouter_logitsrouter_probsbiased_router_probsr  router_indicesskip_experts                rP   r   zZayaRouter.forward  s`   
 4::&"((+
#~~m<<<M5#7-$JbJb:b#b $8ZKL$I$O$O$Q!(<=}}];*11366u}}EH]H]]!JJ':DJJBO>||La~N %(?(??#//Q?'33KC !!"d&=&=>  -"";/%	
 	
rQ   rS   
rY   rZ   r[   r^   r   r   r   r   r   r   r   s   @rP   r  r    sm    ) ) 
	)> .2
||
 ||d*
 
u||U\\5<<E	F	
rQ   r  c                       e Zd Zy)ZayaExpertsNrp   rT   rQ   rP   r?  r?    rq   rQ   r?  c            
            e Zd Zdef fdZ	 ddej                  dej                  dz  deej                  ej                  dz  f   fdZ xZ	S )	r   rw   c                 d    t         |           t        ||      | _        t	        |      | _        y rS   )r   r   r  gater?  expertsr  s      rP   r   zZayaSparseMoeBlock.__init__  s(    vy1	"6*rQ   Nr   r  r   c                     | j                  ||      \  }}}}|j                  \  }}}|j                  ||z  |      }	| j                  |	||      }
|
j                  |||      }
|
|fS )N)r+  )rB  r   r   rC  )rM   r   r  r  r9  r;  
batch_sizer5  emb_dimhidden_states_flatexpert_outputs              rP   r   zZayaSparseMoeBlock.forward  s     FJYY)B FO F
B<)B +8*=*='
J*//
Z0GQ%7V%**:z7K777rQ   rS   r=  r   s   @rP   r   r     sY    +# + :>8||8 $)<<$#68 
u||U\\D00	1	8rQ   r   c                   f    e Zd ZU eed<   dZ eed      ee	dZ
 ej                         d        Zy)ZayaPreTrainedModelrv   Fr   )r-  )r8  r   
attentionsc                 D   t        j                  | |       t        |t              r}t	        j
                  |j                         t	        j                  |j                         t	        j
                  |j                         t	        j                  |j                         y t        |t              r?t	        j
                  |j                         t	        j                  |j                         y t        |t              r t	        j                  |j                         y t        |t               rZ|j"                  rt	        j
                  |j$                         t	        j                  |j&                         d|j&                  d<   y t        |t(              r[| j*                  j,                  }t	        j.                  |j0                  d|       t	        j.                  |j2                  d|       y t        |t4              r|j6                  D ]  }|j8                  }|j:                  |   dk7  rt<        |j:                  |      } ||j*                  |      \  }}t?        || d      jA                  |       t?        || d      jA                  |        y y )	Nr"  r   r   )r   stdr@   )r   	_inv_freq_original_inv_freq)!r   _init_weights
isinstancer   r   ones_r  zeros_r  r  r  	ZayaModelinput_hidden_states_scaleinput_hidden_states_biasr   r   r  r&  r'  r!  r?  rv   initializer_rangenormal_gate_up_projr%  ro   rH   compute_default_rope_parametersrB   r   getattrcopy_)rM   modulerM  r   rope_init_fncurr_inv_freqr  s          rP   rP  z!ZayaPreTrainedModel._init_weights  s   %%dF3f12JJv112KK112JJv,,-KK,,-	*JJv778KK778
+KK$
+~~

6556KK//0*.F##B',++//CLL,,3C@LL))= 34$00 X
%EE##J/9<#6v7G7G
7S#TL#/*#U q:,i 89??N:,.@ ABHHWX 5rQ   N)rY   rZ   r[   r'   r_   _can_compile_fullgraphr   r  r   r   _can_record_outputsr   no_gradrP  rT   rQ   rP   rJ  rJ    sD    "'
!<)# U]]_X XrQ   rJ  c                        e Zd Zdef fdZ	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	e
dz  d
ee   defdZ xZS )rT  rv   c                     t         |   |       t        j                  t	        j
                  |j                              | _        t        j                  t	        j                  |j                              | _	        y rS   )
r   r   r   r   r   r  r   rU  r   rV  r   s     rP   r   zZayaModel.__init__/  sP     )+ejjASAS6T)U&(*U[[ASAS5T(U%rQ   N	input_idsr   position_idsr   inputs_embeds	use_cacherN   r   c           	         |d u |d uz  rt        d      || j                  |      }|r|t        | j                        }|V||j	                         nd}t        j                  |j                  d   |j                        |z   }|j                  d      }t        |x}	t              s`| j                  ||||dfdfdd	}
t        | j                  j                        D ci c]  }| |
|           }	}t        di |	d
<   |}t        | j                  j                        D ci c]  }|| j                  |||       }}|| j                   z   | j"                  z  j%                  t
        j&                        }d }t)        | j*                        D ]G  \  }}| j                  j                  |   } |||f|	|   |	j-                  d
      d|||   d|\  }}I | j/                  |j%                  | j.                  j0                  j2                              }t5        ||r|      S d       S c c}w c c}w )Nz:You must specify exactly one of input_ids or inputs_embeds)rv   r   r5   )devicerv   rg  r   r   rf  c                      t        di  S NrT   r   mask_kwargss   rP   <lambda>z#ZayaModel.forward.<locals>.<lambda>U  s    "4"C{"C rQ   c                      t        di  S rm  r   ro  s   rP   rq  z#ZayaModel.forward.<locals>.<lambda>V  s    *K*Zk*Z rQ   rE   r   )r   r   )r   r   r   r  )last_hidden_stater   rT   )rW   embed_tokensr   rv   get_seq_lengthr   aranger   rj  r   rQ  r   setrH   r   
rotary_embrV  rU  r   r*  	enumerater   r   r   r  r   r   )rM   re  r   rf  r   rg  rh  rN   past_seen_tokenscausal_mask_mappingmask_creation_functionsr   r   r   r  idxdecoder_layerrp  s                    @rP   r   zZayaModel.forward4  s    -t";<YZZ  --i8M0*$++>OCRC^==?de <<(;(;A(>}G[G[\_ooL'11!4L?-F++!."0#2 ,K D"Z'#
 UXX\XcXcXoXoTp#FP
?3J?AA# # +J*XK*X'% "$++"9"9:
 |ZPP
 
 ($*G*GG4KiKiimmMM
 %)!"+DKK"8 	C005J 8E)
8 2*=/33F;  !0$7
$C
8 
84M4		  		-"2"29I9I9O9O"2"PQ%+/8O
 	
>B
 	
K#
s   +H<0I)NNNNNN)rY   rZ   r[   r'   r   r   
LongTensorr   r
   FloatTensorr`   r   r   r   r   r   r   s   @rP   rT  rT  -  s    Vz V .2.204(,26!%L
##d*L
 t+L
 &&-	L

 L
 ((4/L
 $;L
 +,L
 
 L
rQ   rT  c                   X     e Zd ZddiZdZ e       Z e       Z fdZe	dd       Z
 xZS )ZayaForCausalLMzlm_head.weightzmodel.embed_tokens.weightTc                     t        |   |fi | t        j                  |j                  |j
                  | j                  j                        | _        y )Nry   )	r   r   r   r   r   r)   rv   r9   lm_head)rM   rv   rN   r   s      rP   r   zZayaForCausalLM.__init__  s?    *6*yy!3!3V5F5FT[[MeMefrQ   c                    
 | j                         }|||||d

fd
fdd}t        |j                        D ci c]  }| ||           }	}t        di 
|	d<   |	S c c}w )Nrk  c                      t        di  S rm  rn  ro  s   rP   rq  z;ZayaForCausalLM.create_masks_for_generate.<locals>.<lambda>  s    0?;? rQ   c                      t        di  S rm  rs  ro  s   rP   rq  z;ZayaForCausalLM.create_masks_for_generate.<locals>.<lambda>  s    &G&V+&V rQ   rE   r   rT   )get_text_configrx  rH   r   )rv   rg  r   r   rf  r  text_configr}  r   r   rp  s             @rP   create_masks_for_generatez)ZayaForCausalLM.create_masks_for_generate  s     ,,.!*,.(
 @V#

 QTT_TkTkPl
BLJ;/
;==
 
  ?MMV	
s   A rS   )rY   rZ   r[   _tied_weights_keys_is_statefulrb   _tp_plan_pp_planr   staticmethodr  r   r   s   @rP   r  r    s>    *,GHLHHg  rQ   r  )r'   rJ  rT  r  )Kr\   collections.abcr   typingr   r   r   torch.nn.functionalr   
functionalr   torch.utils.checkpointhuggingface_hub.dataclassesr   torch.nnr   cache_utilsr
   r   configuration_utilsr   masking_utilsr   r   r   modeling_outputsr   modeling_rope_utilsr   modeling_utilsr   r   processing_utilsr   utilsr   r   utils.output_capturingr   afmoe.modeling_afmoer   laguna.configuration_lagunar   laguna.modeling_lagunar   r   llama.modeling_llamar   r   r   phi3.modeling_phi3r     qwen3_5_moe.modeling_qwen3_5_moer!   r"   qwen3_moe.modeling_qwen3_moer#   r$   r'   ro   rs   Moduleru   r   r   r   r   r  r  r?  r   rJ  rT  r  __all__rT   rQ   rP   <module>r     s    $      .   . 3 s s 6 6 F & 5 3 6 G U U . L ,-Wk Wk  .Wkt	/ 		/ 	l!		 l!^( (,@)M @)F'8( '8T(")) (,BII , <
 <
~	/ 	8 80'X. 'XT R
 R
 R
j ,-&(;  .DrQ   