Ë
    ÜÓ^jŠ�  ã                   ó^  — d dl mZ d dlmZmZ d dlZd dlmc mZ	 d dlmZ d dl
mZ ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZmZmZ ddlmZ ddlmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z* ddl+m,Z,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2  G d„ dejf                  «      Z4 ed«       G d„ dejf                  «      «       Z5dejl                  de7dejl                  fd„Z8 G d„ d ejf                  «      Z9 G d!„ d"ejf                  «      Z:d#„ Z;	 dEd$ejf                  d%ejl                  d&ejl                  d'ejl                  d(ejl                  dz  d)e<d*e<d+e&e(   fd,„Z=dFd-„Z> G d.„ d/ejf                  «      Z? G d0„ d1e«      Z@ G d2„ d3ejf                  «      ZA G d4„ d5ejf                  «      ZB G d6„ d7ejf                  «      ZCe G d8„ d9ejf                  «      «       ZD G d:„ d;ejf                  «      ZEe) G d<„ d=e$«      «       ZFe) G d>„ d?eF«      «       ZG e)d@¬A«       G dB„ dCeFe«      «       ZHg dD¢ZIy)Gé    )ÚCallable)ÚAnyÚOptionalN)Únn)Úinité   )ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hub)Úcreate_causal_maskÚcreate_recurrent_attention_maskÚ!create_sliding_window_causal_mask)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú
ZayaConfigc                   óÐ   ‡ — e Zd ZU ej                  ed<   defˆ fd„Ze	 	 	 	 ddedz  de	d   de
dz  dedz  d	ed
ef   f
d„«       Z ej                  «       edd„«       «       Zˆ xZS )ÚZayaRotaryEmbeddingÚinv_freqÚconfigc                 ót  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        t        t        |j                  «      «      | _        i | _	        | j                  D ]Ç  }| j
                  j                  |   }|€Œ|d   | j                  |<   | j                  }| j                  |   dk7  rt        | j                  |      } || j
                  |¬«      \  }}| j                  |› d�|d¬«       | j                  |› d�|j                  «       d¬«       t        | |› d�|«       ŒÉ y )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqF)Ú
persistentÚ_original_inv_freqÚ_attention_scaling)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr&   ÚlistÚsetÚlayer_typesr(   Úrope_parametersÚcompute_default_rope_parametersr   Úregister_bufferÚcloneÚsetattr)Úselfr&   r+   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingÚ	__class__s          €úq/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/zaya/modeling_zaya.pyr1   zZayaRotaryEmbedding.__init__1   s6  ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØˆŒÜ¤ F×$6Ñ$6Ó 7Ó8ˆÔØˆŒØ×*Ñ*ò 	UˆJØŸ+™+×5Ñ5°jÑAˆKØÐ"Øà)4°[Ñ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÑ)¨YÒ6Ü2°4·>±>À*Ñ3MÑN�Ù4@ÀÇÁÐYcÔ4dÑ1ˆMÐ1Ø× Ñ  J <¨yÐ!9¸=ÐUZÐ Ô[Ø× Ñ  J <Ð/AÐ!BÀM×DWÑDWÓDYÐfkÐ ÔlÜ�D˜Z˜LÐ(:Ð;Ð=SÕTñ	Uó    NÚdeviceztorch.deviceÚseq_lenr+   Úreturnztorch.Tensorc                 óz  — | j                   |   d   }| j                   |   j                  dd«      }t        | dd«      xs | j                  | j                  z  }t        ||z  «      }d}d|t        j                  d|dt        j                  ¬«      j                  |t        j                  ¬	«      |z  z  z  }	|	|fS )
a{  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimNr   é   ©Údtype)rE   rN   )r8   ÚgetÚgetattrÚhidden_sizeÚnum_attention_headsÚintÚtorchÚarangeÚint64ÚtoÚfloat)
r&   rE   rF   r+   ÚbaserJ   rK   ÚdimÚattention_factorr%   s
             rC   r9   z3ZayaRotaryEmbedding.compute_default_rope_parametersF   sÅ   € ð. ×%Ñ% jÑ1°,Ñ?ˆà &× 6Ñ 6°zÑ B× FÑ FÐG^Ð`cÓ dÐÜ˜6 :¨tÓ4Òh¸×8JÑ8JÈf×NhÑNhÑ8hˆÜ�(Ð2Ñ2Ó3ˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)rD   c                 óN  — t        | |› d�«      }t        | |› d�«      }|d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬	«      5  |j                  «       |j                  «       z  j                  dd
«      }	t        j                  |	|	fd¬«      }
|
j                  «       |z  }|
j                  «       |z  }d d d «       j	                  |j                  ¬«      j	                  |j                  ¬«      fS # 1 sw Y   ŒAxY w)Nr,   r/   r   éÿÿÿÿr!   ÚmpsÚcpuF)Údevice_typeÚenabledrL   ©rZ   rM   )rP   rX   ÚexpandÚshaperW   rE   Ú
isinstanceÚtypeÚstrr   Ú	transposerT   ÚcatÚcosÚsinrN   )r=   ÚxÚposition_idsr+   r%   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedr`   ÚfreqsÚembrj   rk   s                rC   ÚforwardzZayaRotaryEmbedding.forwardk   sl  € ô ˜4 J <¨yÐ!9Ó:ˆÜ# D¨Z¨LÐ8JÐ*KÓLÐà$ Tª1¨d ]Ñ3×9Ñ9Ó;×BÑBÀ<×CUÑCUÐVWÑCXÐZ\Ð^_Ó`×cÑcÐde×dlÑdlÓmÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	0Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)Ð/Ñ/ˆCØ—'‘'“)Ð/Ñ/ˆC÷		0ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	0ð 	0ús   Ã*A1FÆF$©NNNN©N)Ú__name__Ú
__module__Ú__qualname__rT   ÚTensorÚ__annotations__r"   r1   Ústaticmethodr   rS   rg   ÚtuplerX   r9   Úno_gradr   rs   Ú__classcell__©rB   s   @rC   r$   r$   .   sª   ø… Ø�l‰lÓðU˜zõ Uð* à$(Ø+/Ø"Ø!%ñ	"*Ø˜TÑ!ð"*à˜Ñ(ð"*ð �t‘ð"*ð ˜$‘Jð	"*ð
 
ˆ~˜uÐ$Ñ	%ò"*ó ð"*ðH €U‡]�]ƒ_Øò<ó ó ô<rD   r$   ÚRMSNormc                   óh   ‡ — e Zd Zddeddfˆ fd„Zdej                  dej                  fd„Zd„ Zˆ xZ	S )	ÚZayaRMSNormÚepsrG   Nc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)z:
        ZayaRMSNorm is equivalent to T5LayerNorm
        N)r0   r1   r   Ú	ParameterrT   ÚonesÚweightÚvariance_epsilon)r=   rQ   rƒ   rB   s      €rC   r1   zZayaRMSNorm.__init__€   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕrD   Úhidden_statesc                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S )NrL   r]   T)Úkeepdim)	rN   rW   rT   Úfloat32ÚpowÚmeanÚrsqrtrˆ   r‡   )r=   r‰   Úinput_dtypeÚvariances       rC   rs   zZayaRMSNorm.forwardˆ   sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:rD   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)r|   r‡   rd   rˆ   )r=   s    rC   Ú
extra_reprzZayaRMSNorm.extra_repr�   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIrD   )g�íµ ÷Æ°>)
rv   rw   rx   rX   r1   rT   ry   rs   r“   r~   r   s   @rC   r‚   r‚   ~   s7   ø„ ñ$¨ð $¸$õ $ð; U§\¡\ð ;°e·l±ló ;öJrD   r‚   r‰   Ún_reprG   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r!   N)rd   rc   Úreshape)r‰   r”   ÚbatchÚnum_key_value_headsÚslenrK   s         rC   Ú	repeat_kvrš   “   so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTrD   c                   óx   ‡ — e Zd ZdZdedefˆ fd„Z	 d
dej                  de	dz  dej                  dz  fd	„Z
ˆ xZS )ÚZayaCCAProjectionav  
    Projects hidden states into attention q/k/v states with ZAYA's Compressed Convolutional Attention (CCA) path.
    See https://huggingface.co/papers/2510.04476.

    This follows the usual q/k/v projection flow, with three ZAYA-specific changes: q/k are mixed by a causal 1D
    convolution, q/k keep residual projection paths, and v uses a delayed recurrent state.
    r&   Ú	layer_idxc                 ó¶  •— t         ‰| �  «        || _        || _        |j                  | _        |j
                  | _        |j                  | _        | j                  dz
  | j                  dz
  z   | _	        |j                  | _
        |j                  | _        |j                  | _        | j                  | j                  z  | _        | j                  | j                  z  }| j                  | j                  z  }t        j                  | j                  || j                  j                   ¬«      | _        t        j                  | j                  || j                  j                   ¬«      | _        t        j                  | j                  |dz  | j                  j                   ¬«      | _        t        j                  | j                  |dz  | j                  j                   ¬«      | _        ||z   }t        j*                  ||| j                  |dd¬«      | _        t        j*                  ||| j                  | j                  | j                  z   dd¬«      | _        y )Nr!   ©ÚbiasrL   r   )Úin_channelsÚout_channelsÚkernel_sizeÚgroupsÚpaddingÚstride)r0   r1   r&   r�   rQ   Ú	cca_time0Údepthwise_kernel_sizeÚ	cca_time1Úgrouped_kernel_sizeÚconv_kernel_sizer˜   rR   rK   Únum_key_value_groupsr   ÚLinearÚattention_biasÚq_projÚk_projÚv_proj_currentÚv_proj_delayedÚConv1dÚconv_qk_depthwiseÚconv_qk_grouped)r=   r&   r�   Úquery_hidden_sizeÚkey_value_hidden_sizeÚconv_channelsrB   s         €rC   r1   zZayaCCAProjection.__init__¨   sì  ø€ Ü‰ÑÔØˆŒØ"ˆŒà!×-Ñ-ˆÔà%+×%5Ñ%5ˆÔ"Ø#)×#3Ñ#3ˆÔ Ø!%×!;Ñ!;¸aÑ!?ÀD×D\ÑD\Ð_`ÑD`Ñ aˆÔà#)×#=Ñ#=ˆÔ Ø#)×#=Ñ#=ˆÔ ØŸ™ˆŒØ$(×$<Ñ$<À×@XÑ@XÑ$XˆÔ!à ×4Ñ4°t·}±}ÑDÐØ $× 8Ñ 8¸4¿=¹=Ñ HÐä—i‘i × 0Ñ 0Ð2CÈ$Ï+É+×JdÑJdÔeˆŒÜ—i‘i × 0Ñ 0Ð2GÈdÏkÉk×NhÑNhÔiˆŒÜ Ÿi™i¨×(8Ñ(8Ð:OÐSTÑ:TÐ[_×[fÑ[f×[uÑ[uÔvˆÔÜ Ÿi™i¨×(8Ñ(8Ð:OÐSTÑ:TÐ[_×[fÑ[f×[uÑ[uÔvˆÔà-Ð0AÑAˆÜ!#§¡Ø%Ø&Ø×2Ñ2Ø ØØô"
ˆÔô  "Ÿy™yØ%Ø&Ø×0Ñ0Ø×,Ñ,¨t×/GÑ/GÑGØØô 
ˆÕrD   Nr‰   Úpast_key_valuesÚ	conv_maskc                 ó8  — |�(||d d …d d …d f   j                  |j                  «      z  }|j                  d d }g |¢d‘| j                  ‘­}| j	                  |«      }| j                  |«      }t        j                  ||gd¬«      } |j                  |Ž }	 |j                  |Ž j                  dd«      }
t        |
| j                  «      j                  dd«      }
|	|
z   dz  }	 |	j                  g |¢d‘| j                  ‘| j                  ‘­Ž j                  d¬«      }
|j                  dd«      }|d uxr |j                  | j                  «      }|r@|j                  | j                     j                   d   }t        j                  ||gd¬«      }n"t#        j$                  || j&                  df«      }|�b|d| j&                   d …f   }t#        j$                  || j&                  |j                  d   z
  df«      }|j)                  || j                  «       | j+                  |«      }| j-                  |«      j                  dd«      }|	j                  d   |	j                  d   z  } |dd |…f   j                  |Ž |	z   } |d|d …f   j                  |Ž |
z   }| j/                  |«      }| j1                  |«      }|r6|j                  | j                     j2                  d   j5                  d«      }n/| j1                  |j7                  |d   d| j8                  «      «      }t        j                  ||d d …d d…f   gd¬«      }|�&|j;                  |d d …dd d …f   | j                  «        t        j                  ||gd¬«      j                  |Ž }|||fS )	Nr]   rb   r!   rL   ç      à?éþÿÿÿr   .)rW   rN   rd   rK   r¯   r°   rT   ri   Úviewrh   rš   r¬   rŽ   Úhas_previous_stater�   ÚlayersÚconv_statesÚFÚpadr«   Úupdate_conv_stater´   rµ   r±   r²   Úrecurrent_statesÚ	unsqueezeÚ	new_zerosrQ   Úupdate_recurrent_state)r=   r‰   r¹   rº   Úinput_shapeÚhidden_shapeÚprojected_queriesÚprojected_keysÚ	qk_statesÚquery_residualÚkey_residualÚuse_precomputed_statesÚcached_qk_statesÚnew_conv_stater¶   ÚqueryÚkeyÚvalue_currentÚdelayed_v_stateÚrecurrent_v_stateÚvalue_delayedÚvalues                         rC   rs   zZayaCCAProjection.forwardÒ   s¬  € ð Ð Ø)¨I²aº¸D°jÑ,A×,DÑ,DÀ]×EXÑEXÓ,YÑYˆMà#×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà ŸK™K¨Ó6ÐØŸ™ ]Ó3ˆÜ—I‘IÐ0°.ÐAÀrÔJˆ	à/Ð*×/Ñ/°Ð>ˆØ*�~×*Ñ*¨LÐ9×CÑCÀAÀqÓIˆÜ  ¨t×/HÑ/HÓI×SÑSÐTUÐWXÓYˆØ(¨<Ñ7¸3Ñ>ˆØ*�~×*Ñ*Ðf¨KÐf¸Ðf¸T×=VÑ=VÐfÐX\×XeÑXeÒf×kÑkÐprÐkÓsˆà×'Ñ'¨¨1Ó-ˆ	Ø!0¸Ð!<Ò!sÀ×AcÑAcÐdh×drÑdrÓAsÐÙ!Ø.×5Ñ5°d·n±nÑE×QÑQÐRSÑTÐÜŸ	™	Ð#3°YÐ"?ÀRÔH‰IäŸ™˜i¨$×*?Ñ*?ÀÐ)CÓDˆIàÐ&Ø& s¨T×-BÑ-BÐ,BÑ,DÐ'DÑEˆNÜŸU™U >°D×4IÑ4IÈN×L`ÑL`ÐacÑLdÑ4dÐfgÐ3hÓiˆNØ×-Ñ-¨n¸d¿n¹nÔMà×*Ñ*¨9Ó5ˆ	Ø×(Ñ(¨Ó3×=Ñ=¸aÀÓCˆ	à*×0Ñ0°Ñ4°~×7KÑ7KÈBÑ7OÑOÐØ7�	˜#Ð1Ð 1Ð1Ð1Ñ2×7Ñ7¸ÐFÈÑWˆØ5ˆi˜Ð.Ñ/Ð/Ñ0×5Ñ5°|ÐDÀ|ÑSˆð ×+Ñ+¨MÓ:ˆØ×-Ñ-¨mÓ<ˆÙ!Ø /× 6Ñ 6°t·~±~Ñ F× WÑ WÐXYÑ Z× dÑ dÐefÓ gÑà $× 3Ñ 3°M×4KÑ4KÈKÐXYÉNÐ\]Ð_c×_oÑ_oÓ4pÓ qÐÜŸ	™	Ð#4°oÂaÈÈ"ÈÀfÑ6MÐ"NÐTUÔVˆàÐ&Ø×2Ñ2°?Â1ÀbÊ!À8Ñ3LÈdÏnÉnÔ]àF”—	‘	˜=¨-Ð8¸bÔA×FÑFÈÐUˆà�c˜5Ð Ð rD   ru   )rv   rw   rx   Ú__doc__r"   rS   r1   rT   ry   r
   rs   r~   r   s   @rC   rœ   rœ   Ÿ   sS   ø„ ñð(
˜zð (
°cõ (
ð\ *.ñ	9!à—|‘|ð9!ð  ™ð9!ð —<‘< $Ñ&÷	9!rD   rœ   c                   ó–   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  deej                  ej                  f   fd„Z	ˆ xZ
S )Ú
ZayaQKNormzf
    L2-normalizes q/k states to sqrt(head_dim) and applies ZAYA's learned per-KV-head key scale.
    r&   c                 ó¸   •— t         ‰| �  «        |j                  dz  | _        t	        j
                  t        j                  |j                  «      «      | _	        y )Nr¼   )
r0   r1   rK   Úhead_dim_scaler   r…   rT   Úzerosr˜   Útemp©r=   r&   rB   s     €rC   r1   zZayaQKNorm.__init__  s>   ø€ Ü‰ÑÔØ$Ÿo™o¨sÑ2ˆÔÜ—L‘L¤§¡¨V×-GÑ-GÓ!HÓIˆ�	rD   Úquery_statesÚ
key_statesrG   c                 óX  — t        j                  |j                  «      j                  }|| j                  |j                  ddd¬«      j                  |«      z  z  }|| j                  |j                  ddd¬«      j                  |«      z  z  }|| j                  d d d d …d f   z  }||fS )NrL   r]   T)ÚprZ   r‹   )rT   ÚfinforN   rƒ   rÞ   ÚnormÚ	clamp_minrà   )r=   râ   rã   Únorm_epss       rC   rs   zZayaQKNorm.forward  s°   € Ü—;‘;˜|×1Ñ1Ó2×6Ñ6ˆØ#Ø×Ñ ,×"3Ñ"3°a¸RÈÐ"3Ó"N×"XÑ"XÐYaÓ"bÑbñ
ˆð  Ø×Ñ *§/¡/°A¸2Àt /Ó"L×"VÑ"VÐW_Ó"`Ñ`ñ
ˆ
ð   $§)¡)¨D°$º¸4Ð,?Ñ"@Ñ@ˆ
Ø˜ZÐ'Ð'rD   )rv   rw   rx   rÚ   r"   r1   rT   ry   r|   rs   r~   r   s   @rC   rÜ   rÜ     sP   ø„ ñðJ˜zõ Jð
	( E§L¡Lð 	(¸e¿l¹lð 	(ÈuÐUZ×UaÑUaÐch×coÑcoÐUoÑOp÷ 	(rD   rÜ   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Nr]   rL   rb   )rd   rT   ri   )rl   Úx1Úx2s      rC   Úrotate_halfrí   $  sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'rD   ÚmodulerÓ   rÔ   rÙ   Úattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t        || j                  «      }t        || j                  «      }	t        j                  ||j	                  dd«      «      |z  }
|�|
|z   }
t
        j                  j                  |
dt        j                  ¬«      j                  |j                  «      }
t
        j                  j                  |
|| j                  ¬«      }
t        j                  |
|	«      }|j	                  dd«      j                  «       }||
fS )NrL   r   r]   )rZ   rN   )rå   Útrainingr!   )rš   r¬   rT   Úmatmulrh   r   Ú
functionalÚsoftmaxrŒ   rW   rN   rñ   rô   Ú
contiguous)rî   rÓ   rÔ   rÙ   rï   rð   rñ   rò   rã   Úvalue_statesÚattn_weightsÚattn_outputs               rC   Úeager_attention_forwardrü   +  sà   € ô ˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$rD   c                 óh  — |j                  |«      }|j                  |«      }|j                  d   }| dd|…f   | d|d…f   }}|dd|…f   |d|d…f   }	}||z  t        |«      |z  z   }
||z  t        |«      |z  z   }t        j                  |
|gd¬«      }
t        j                  ||	gd¬«      }|
|fS )a»  Applies Rotary Position Embedding to the query and key tensors.

    Removes the interleaving of cos and sin from GLM

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    r]   .Nrb   )rÆ   rd   rí   rT   ri   )ÚqÚkrj   rk   Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds               rC   Úapply_rotary_pos_embr  E  sÞ   € ð( �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cð —‘˜2‘€JØ�c˜;˜J˜;Ð&Ñ'¨¨3°
±Ð+;Ñ)<ˆ6€EØ�c˜;˜J˜;Ð&Ñ'¨¨3°
±Ð+;Ñ)<ˆ6€Eð �s‰{œ{¨5Ó1°CÑ7Ñ8€GØ�s‰{œ{¨5Ó1°CÑ7Ñ8€Gô �i‰i˜ &Ð)¨rÔ2€GÜ�i‰i˜ &Ð)¨rÔ2€GØ�GÐÐrD   c                   óô   ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej                  de	e
ef   dz  dedz  d	eej                  ej                  f   dz  d
ee   deej                  ej                  dz  f   fd„Zˆ xZS )ÚZayaAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr&   r�   c                 óÞ  •— t         ‰| �  «        || _        || _        t	        |d|j
                  |j                  z  «      | _        |j                  |j                  z  | _	        |j                  | _        | j                  dz  | _
        |j                  | _        d| _        t        j                  |j                  | j                  z  |j
                  |j                  ¬«      | _        t#        | j                  |¬«      | _        |j&                  |   | _        | j(                  dk(  r|j*                  nd | _        |j
                  | _        |j                  | _        t-        |«      | _        y )NrK   g      à¿TrŸ   ©r&   r�   Úhybrid_sliding)r0   r1   r&   r�   rP   rQ   rR   rK   r˜   r¬   rð   Úattention_dropoutÚ	is_causalr   r­   r®   Úo_projrœ   Úqkv_projr7   r+   Úsliding_windowrÜ   Úqk_norm©r=   r&   r�   rB   s      €rC   r1   zZayaAttention.__init__n  s-  ø€ Ü‰ÑÔØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø#)×#=Ñ#=ˆÔ Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒô *Ø—;‘;Øô
ˆŒð !×,Ñ,¨YÑ7ˆŒØ7;·±ÐJZÒ7Z˜f×3Ò3Ð`dˆÔØ!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ Ü! &Ó)ˆ�rD   Nr‰   rï   r¹   Úposition_embeddingsrò   rG   c                 óÒ  — |j                   d d }|xs i }|j                  d«      }|j                  d«      }	| j                  |||	«      \  }
}}| j                  |
|«      \  }
}|
j	                  dd«      }
|j	                  dd«      }|j	                  dd«      }|\  }}t        |
|||«      \  }
}|� |j                  ||| j                  «      \  }}t        j                  | j                  j                  t        «      } || |
|||f| j                  sdn| j                  | j                  | j                   dœ|¤Ž\  }} |j"                  g |¢d‘­Ž }| j%                  |«      }||fS )Nr]   ÚcausalÚconvr!   rL   ç        )rñ   rð   r  )rd   rO   r  r  rh   r  Úupdater�   r   Úget_interfacer&   Ú_attn_implementationrü   rô   r  rð   r  r–   r  )r=   r‰   rï   r¹   r  rò   rÉ   Úmask_mappingÚcausal_maskrº   râ   rã   rù   rj   rk   Úattention_interfacerû   rú   s                     rC   rs   zZayaAttention.forward†  s•  € ð $×)Ñ)¨#¨2Ð.ˆà%Ò+¨ˆØ"×&Ñ& xÓ0ˆØ ×$Ñ$ VÓ,ˆ	ð 26·±¸}ÈoÐ_hÓ1iÑ.ˆ�j ,Ø#'§<¡<°¸jÓ#IÑ ˆ�jà#×-Ñ-¨a°Ó3ˆØ×)Ñ)¨!¨QÓ/ˆ
Ø#×-Ñ-¨a°Ó3ˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;ˆØ—k‘k +Ó.ˆà˜LÐ(Ð(rD   )NNN)rv   rw   rx   rÚ   r"   rS   r1   rT   ry   Údictrg   r   r
   r|   r   r   rs   r~   r   s   @rC   r
  r
  k  s¬   ø„ ÙGð*˜zð *°cõ *ð6 15Ø(,ØHLñ.)à—|‘|ð.)ð ˜S #˜X™¨Ñ-ð.)ð  ™ð	.)ð
 # 5§<¡<°·±Ð#=Ñ>ÀÑEð.)ð Ð+Ñ,ð.)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷.)rD   r
  c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej                  dej                  dz  dee	e
f   dz  dedz  d	eej                  ej                  f   dz  d
ee   deej                  ej                  dz  f   fd„Zˆ xZS )ÚZayaDecoderLayerr&   r�   c                 óŠ  •— t         ‰| �  «        |j                  | _        t        ||¬«      | _        t        ||«      | _        t        |j                  |j                  ¬«      | _	        t        |j                  |j                  ¬«      | _
        t        |j                  «      | _        t        |j                  «      | _        y )Nr  ©rƒ   )r0   r1   rQ   r
  Ú	self_attnÚZayaSparseMoeBlockÚmlpr‚   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormÚZayaResidualScalingÚpost_attention_residual_scaleÚpost_mlp_residual_scaler  s      €rC   r1   zZayaDecoderLayer.__init__¸  s”   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä&¨fÀ	ÔJˆŒÜ% f¨iÓ8ˆŒÜ*¨6×+=Ñ+=À6×CVÑCVÔWˆÔÜ(3°F×4FÑ4FÈF×L_ÑL_Ô(`ˆÔ%Ü-@À×ASÑASÓ-TˆÔ*Ü':¸6×;MÑ;MÓ'NˆÕ$rD   Nr‰   Úprev_router_hidden_statesrï   r¹   r  rò   rG   c                 ó°  — |}| j                  |j                  | j                   j                  j                  ¬«      «      } | j                  d||||dœ|¤Ž\  }}| j                  ||«      }| j                  |j                  | j                  j                  j                  ¬«      «      }| j                  ||«      \  }}| j                  ||«      }||fS )NrM   )r‰   rï   r¹   r  © )	r)  rW   r‡   rN   r%  r,  r*  r'  r-  )	r=   r‰   r.  rï   r¹   r  rò   ÚresidualÚ_s	            rC   rs   zZayaDecoderLayer.forwardÃ  sã   € ð !ˆð ×,Ñ,¨X¯[©[¸t×?SÑ?S×?ZÑ?Z×?`Ñ?`¨[Ó-aÓbˆà)˜4Ÿ>™>ð 
Ø'Ø)Ø+Ø 3ñ	
ð
 ñ
Ñˆ�qð ×5Ñ5°mÀXÓNˆØ×5Ñ5°h·k±kÈ×HeÑHe×HlÑHl×HrÑHr°kÓ6sÓtˆà37·8±8ØØ%ó4
Ñ0ˆÐ0ð
 ×4Ñ4°]ÀHÓMˆàÐ7Ð7Ð7rD   rt   )rv   rw   rx   r"   rS   r1   rT   ry   r   rg   r   r
   r|   r   r   rs   r~   r   s   @rC   r"  r"  ·  sÂ   ø„ ð	O˜zð 	O°cõ 	Oð :>Ø04Ø(,ØHLñ 8à—|‘|ð 8ð $)§<¡<°$Ñ#6ð 8ð ˜S #˜X™¨Ñ-ð	 8ð
  ™ð 8ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð 8ð Ð+Ñ,ð 8ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷ 8rD   r"  c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )r+  rQ   c                 óŠ  •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        t        j                  t	        j                  |«      «      | _        t        j                  t	        j
                  |«      «      | _	        t        j                  t	        j                  |«      «      | _
        y ru   )r0   r1   r   r…   rT   r†   Úhidden_states_scalerß   Úhidden_states_biasÚresidual_scaleÚresidual_bias)r=   rQ   rB   s     €rC   r1   zZayaResidualScaling.__init__ç  sv   ø€ Ü‰ÑÔÜ#%§<¡<´·
±
¸;Ó0GÓ#HˆÔ Ü"$§,¡,¬u¯{©{¸;Ó/GÓ"HˆÔÜ Ÿl™l¬5¯:©:°kÓ+BÓCˆÔÜŸ\™\¬%¯+©+°kÓ*BÓCˆÕrD   r‰   r1  c                 ó|   — || j                   z   | j                  z  }|| j                  z   | j                  z  }||z   S ru   )r6  r5  r8  r7  )r=   r‰   r1  s      rC   rs   zZayaResidualScaling.forwardî  sC   € à&¨×)@Ñ)@Ñ@ÀD×D\ÑD\Ñ\ˆØ˜t×1Ñ1Ñ1°T×5HÑ5HÑHˆØ˜xÑ'Ð'rD   )	rv   rw   rx   rS   r1   rT   ry   rs   r~   r   s   @rC   r+  r+  æ  s,   ø„ ðD Cõ Dð( U§\¡\ð (¸U¿\¹\÷ (rD   r+  c                   ód   ‡ — e Zd Zdededefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚZayaRouterMLPrQ   Únum_expertsr(  c                 ó&  •— t         ‰| �  «        t        ||¬«      | _        t	        j
                  ||d¬«      | _        t	        j
                  ||d¬«      | _        t	        j
                  ||d¬«      | _        t	        j                  «       | _
        y )Nr$  TrŸ   F)r0   r1   r‚   rç   r   r­   Úfc1Úfc2Úout_projÚGELUÚact_fn)r=   rQ   r<  r(  rB   s       €rC   r1   zZayaRouterMLP.__init__ö  se   ø€ Ü‰ÑÔÜ °Ô>ˆŒ	Ü—9‘9˜[¨+¸DÔAˆŒÜ—9‘9˜[¨+¸DÔAˆŒÜŸ	™	 +¨{ÀÔGˆŒÜ—g‘g“iˆ�rD   r‰   rG   c                 óÆ   — | j                  |«      }| j                  | j                  |«      «      }| j                  | j                  |«      «      }| j	                  |«      S ru   )rç   rB  r>  r?  r@  )r=   r‰   s     rC   rs   zZayaRouterMLP.forwardþ  sM   € ØŸ	™	 -Ó0ˆØŸ™ D§H¡H¨]Ó$;Ó<ˆØŸ™ D§H¡H¨]Ó$;Ó<ˆØ�}‰}˜]Ó+Ð+rD   )
rv   rw   rx   rS   rX   r1   rT   ry   rs   r~   r   s   @rC   r;  r;  õ  s8   ø„ ð  Cð  °cð  Èõ  ð, U§\¡\ð ,°e·l±l÷ ,rD   r;  c                   óÌ   ‡ — e Zd Zdeddfˆ fd„Z	 ddej                  dej                  dz  deej                  ej                  ej                  ej                  f   fd„Zˆ xZ	S )	Ú
ZayaRouterr�   rG   Nc                 óú  •— t         ‰| �  «        || _        |j                  | _        || _        |j
                  | _        |j
                  dz   | _        |j                  | _        |j                  | _	        t        j                  | j                  | j                  d¬«      | _        | j                  dk7  | _        | j                  r7t        j                  t        j                   | j                  «      «      | _        t%        | j                  | j                  |j&                  «      | _        | j+                  dt        j,                  | j                  t        j.                  ¬«      «       d| j0                  d<   y )	Nr!   TrŸ   r   Úbalancing_biasesrM   ç      ð¿r]   )r0   r1   r&   rQ   r�   r<  Únum_router_classesÚnum_experts_per_tokÚtop_kÚrouter_hidden_sizer   r­   Ú	down_projÚuse_edar…   rT   r†   Úrouter_states_scaler;  r(  Ú
router_mlpr:   rß   rŒ   rG  r  s      €rC   r1   zZayaRouter.__init__  s  ø€ ô
 	‰ÑÔàˆŒØ!×-Ñ-ˆÔØ"ˆŒà!×-Ñ-ˆÔà"(×"4Ñ"4°qÑ"8ˆÔØ×/Ñ/ˆŒ
Ø"(×";Ñ";ˆÔäŸ™ 4×#3Ñ#3°T×5LÑ5LÐSWÔXˆŒà—~‘~¨Ñ*ˆŒØ�<Š<Ü')§|¡|´E·J±J¸t×?VÑ?VÓ4WÓ'XˆDÔ$ä'¨×(?Ñ(?À×AXÑAXÐZ`×ZmÑZmÓnˆŒà×ÑÐ/´·±¸T×=TÑ=TÔ\a×\iÑ\iÔ1jÔkØ$(ˆ×Ñ˜bÒ!rD   r‰   Úrouter_statesc                 ó  — d| j                   f}|j                  d   }| j                  |«      }| j                  r|�||| j                  z  z   }|d d …| d …f   j                  «       }| j                  |«      }t        j                  |d¬«      }|j                  «       j                  t        j                  «      | j                  z   }	t        j                  |	| j                   d¬«      \  }
}t        j                  |d|¬«      }|| j                  j                   k(  }|j#                  |d«      }|j#                  |d«      }|j%                  d| j&                  «      |j%                  |«      |j%                  |«      |fS )Nr]   r!   rb   rL   )rZ   Úindexr   )rK  rd   rM  rN  rO  r;   rP  rT   r÷   ÚdetachrW   rŒ   rG  ÚtopkÚgatherr&   r<  Úmasked_fillr–   rI  )r=   r‰   rQ  Úfinal_shapeÚ
seq_lengthÚrouter_hidden_statesÚrouter_hidden_states_nextÚrouter_logitsÚrouter_probsÚbiased_router_probsr2  Úrouter_indicesÚskip_experts                rC   rs   zZayaRouter.forward"  s`  € ð
 ˜4Ÿ:™:Ð&ˆØ"×(Ñ(¨Ñ+ˆ
à#Ÿ~™~¨mÓ<Ðà�<Š<˜MÐ5Ø#7¸-È$×JbÑJbÑ:bÑ#bÐ à$8º¸Z¸K¹L¸Ñ$I×$OÑ$OÓ$QÐ!ØŸ™Ð(<Ó=ˆÜ—}‘} ]¸Ô;ˆà*×1Ñ1Ó3×6Ñ6´u·}±}ÓEÈ×H]ÑH]Ñ]ÐÜ!ŸJ™JÐ':¸D¿J¹JÈBÔOÑˆˆ>Ü—|‘| L°a¸~ÔNˆð %¨¯©×(?Ñ(?Ñ?ˆØ#×/Ñ/°¸QÓ?ˆØ'×3Ñ3°KÀÓCˆð ×!Ñ! " d×&=Ñ&=Ó>Ø× Ñ  Ó-Ø×"Ñ" ;Ó/Ø%ð	
ð 	
rD   ru   ©
rv   rw   rx   rS   r1   rT   ry   r|   rs   r~   r   s   @rC   rE  rE    sm   ø„ ð)ð ð)ð 
õ	)ð> .2ñ
à—|‘|ð
ð —|‘| dÑ*ð
ð 
ˆu�|‰|˜UŸ\™\¨5¯<©<¸¿¹ÐEÑ	F÷	
rD   rE  c                   óŠ   ‡ — e Zd ZdZˆ fd„Zdej                  dej                  dej                  dej                  fd„Zˆ xZS )ÚZayaExpertsz2Collection of expert weights stored as 3D tensors.c                 óò  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        t        j                  t        j                  | j                  d| j                  z  | j                  «      «      | _        t        j                  t        j                  | j                  | j                  | j                  «      «      | _        t        |j                     | _        y )NrL   )r0   r1   r<  rQ   Ú
hidden_dimÚmoe_intermediate_sizeÚintermediate_dimr   r…   rT   ÚemptyÚgate_up_projrM  r	   Ú
hidden_actrB  rá   s     €rC   r1   zZayaExperts.__init__H  s­   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ ×,Ñ,ˆŒØ &× <Ñ <ˆÔÜŸL™L¬¯©°T×5EÑ5EÀqÈ4×K`ÑK`ÑG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸ™¤e§k¡k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Ñ/ˆ�rD   r‰   Útop_k_indexÚtop_k_weightsrG   c                 óf  — t        j                  |«      }t        j                  «       5  t         j                  j                  j                  || j                  ¬«      }|j                  ddd«      }t        j                  |j                  d¬«      d«      j                  «       }d d d «       D ]ð  }|d   }|| j                  k(  rŒt        j                  |   «      \  }}	||	   }
t        j                  j                  |
| j                  |   «      j                  dd¬«      \  }}| j                  |«      |z  }t        j                  j                  || j                   |   «      }|||	|d f   z  }|j#                  d|	|j%                  |j&                  «      «       Œò |S # 1 sw Y   �ŒxY w)N)Únum_classesrL   r!   r   )r]   r½   rb   r]   )rT   Ú
zeros_liker}   r   rö   Úone_hotr<  ÚpermuteÚgreaterÚsumÚnonzeroÚwhereÚlinearri  ÚchunkrB  rM  Ú
index_add_rW   rN   )r=   r‰   rk  rl  Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess                 rC   rs   zZayaExperts.forwardQ  s™  € ô $×.Ñ.¨}Ó=ÐÜ�]‰]‹_ñ 	SÜŸ(™(×-Ñ-×5Ñ5°kÈt×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸ™ {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷	Sð
 %ò 
	nˆJØ# A™ˆJØ˜T×-Ñ-Ò-ØÜ#(§;¡;¨{¸:Ñ/FÓ#GÑ ˆI�yØ)¨)Ñ4ˆMÜ—}‘}×+Ñ+¨M¸4×;LÑ;LÈZÑ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆD�"Ø$(§K¡K°Ó$5¸Ñ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9NÐPT×P^ÑP^Ð_iÑPjÓ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÑ<eÑ$eÐ!Ø×*Ñ*¨1¨iÐ9N×9QÑ9QÐRe×RkÑRkÓ9lÕmð
	nð #Ð"÷#	Sñ 	Sús   ªA=F&Æ&F0)	rv   rw   rx   rÚ   r1   rT   ry   rs   r~   r   s   @rC   rc  rc  D  sF   ø„ á<ô0ð#à—|‘|ð#ð —\‘\ð#ð —|‘|ð	#ð
 
�‰÷#rD   rc  c            
       ó¢   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dej                  dz  deej                  ej                  dz  f   fd„Zˆ xZ	S )	r&  r�   c                 ód   •— t         ‰| �  «        t        ||«      | _        t	        |«      | _        y ru   )r0   r1   rE  r€  rc  Úexpertsr  s      €rC   r1   zZayaSparseMoeBlock.__init__m  s(   ø€ Ü‰ÑÔÜ˜v yÓ1ˆŒ	Ü" 6Ó*ˆ�rD   Nr‰   r.  rG   c                 óÐ   — | j                  ||¬«      \  }}}}|j                  \  }}}|j                  ||z  |«      }	| j                  |	||«      }
|
j                  |||«      }
|
|fS )N)rQ  )r€  rd   r¾   r…  )r=   r‰   r.  r2  r]  r_  Ú
batch_sizerY  Úemb_dimÚhidden_states_flatÚexpert_outputs              rC   rs   zZayaSparseMoeBlock.forwardr  s‹   € ð FJÇYÁYØÐ)Bð FOó F
ÑBˆˆ<˜Ð)Bð +8×*=Ñ*=Ñ'ˆ
�J Ø*×/Ñ/°
¸ZÑ0GÈÓQÐØŸ™Ð%7¸ÈÓVˆØ%×*Ñ*¨:°zÀ7ÓKˆàÐ7Ð7Ð7rD   ru   ra  r   s   @rC   r&  r&  l  sY   ø„ ð+¨#õ +ð :>ñ8à—|‘|ð8ð $)§<¡<°$Ñ#6ð8ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷	8rD   r&  c                   ó–   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZ eed¬«      eed	œZ ej(                  «       ˆ fd
„«       Zˆ xZS )ÚZayaPreTrainedModelr&   ÚmodelTr"  r¹   Fr   )rS  )r\  r‰   Ú
attentionsc                 ó8  •— t         ‰| �  |«       t        |t        «      r}t	        j
                  |j                  «       t	        j                  |j                  «       t	        j
                  |j                  «       t	        j                  |j                  «       y t        |t        «      r?t	        j
                  |j                  «       t	        j                  |j                  «       y t        |t        «      r t	        j                  |j                  «       y t        |t         «      rZ|j"                  rt	        j
                  |j$                  «       t	        j                  |j&                  «       d|j&                  d<   y t        |t(        «      r[| j*                  j,                  }t	        j.                  |j0                  d|¬«       t	        j.                  |j2                  d|¬«       y t        |t4        «      r™|j6                  D ]‰  }|j8                  }|j:                  |   dk7  rt<        |j:                  |      } ||j*                  |¬«      \  }}t?        ||› d�«      jA                  |«       t?        ||› d�«      jA                  |«       Œ‹ y y )	NrH  r]   r  )rŽ   Ústdr)   r*   r,   r.   )!r0   Ú_init_weightsre   r+  r   Úones_r5  Úzeros_r6  r7  r8  Ú	ZayaModelÚinput_hidden_states_scaleÚinput_hidden_states_biasrÜ   rà   rE  rN  rO  rG  rc  r&   Úinitializer_rangeÚnormal_ri  rM  r$   r7   r9   r(   r   rP   Úcopy_)r=   rî   r�  r+   r?   r@   r2  rB   s          €rC   r‘  z!ZayaPreTrainedModel._init_weights—  sÝ  ø€ ä‰Ñ˜fÔ%Ü�fÔ1Ô2Ü�J‰J�v×1Ñ1Ô2Ü�K‰K˜×1Ñ1Ô2Ü�J‰J�v×,Ñ,Ô-Ü�K‰K˜×,Ñ,Õ-Ü˜¤	Ô*Ü�J‰J�v×7Ñ7Ô8Ü�K‰K˜×7Ñ7Õ8Ü˜¤
Ô+Ü�K‰K˜Ÿ™Õ$Ü˜¤
Ô+Ø�~Š~Ü—
‘
˜6×5Ñ5Ô6Ü�K‰K˜×/Ñ/Ô0Ø*.ˆF×#Ñ# BÒ'Ü˜¤Ô,Ø—+‘+×/Ñ/ˆCÜ�L‰L˜×,Ñ,°3¸CÕ@Ü�L‰L˜×)Ñ)°¸Ö=Ü˜Ô 3Ô4Ø$×0Ñ0ò X�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ò<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ô#UÑ �˜qÜ˜ : ,¨iÐ 8Ó9×?Ñ?ÀÔNÜ˜ : ,Ð.@Ð AÓB×HÑHÈÕWñXð 5rD   )rv   rw   rx   r"   rz   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr   rE  r"  r
  Ú_can_record_outputsrT   r}   r‘  r~   r   s   @rC   rŒ  rŒ  „  sz   ø… àÓØÐØ&*Ð#Ø+Ð,ÐØ#4Ð"5ÐØÐØ€NØÐà"ÐØ"&Ðá'¨
¸!Ô<Ø)Ø#ñÐð €U‡]�]ƒ_óXó ôXrD   rŒ  c                   óò   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   defd„«       «       «       Zˆ xZS )r”  r&   c           	      óè  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                  ¬«      | _        t#        |¬«      | _        d| _        t        j(                  t+        j,                  |j                  «      «      | _        t        j(                  t+        j0                  |j                  «      «      | _        | j5                  «        y c c}w )Nr$  ©r&   F)r0   r1   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	EmbeddingrQ   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersr"  rÀ   r‚   r(  rç   r$   Ú
rotary_embÚgradient_checkpointingr…   rT   r†   r•  rß   r–  Ú	post_initr  s      €rC   r1   zZayaModel.__init__¹  s  ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m‘mÜBGÈ×H`ÑH`ÓBaÖb°YÔ˜f iÕ0Òbó
ˆŒô   × 2Ñ 2¸×8KÑ8KÔLˆŒ	Ü-°VÔ<ˆŒØ&+ˆÔ#Ü)+¯©´e·j±jÀ×ASÑASÓ6TÓ)UˆÔ&Ü(*¯©´U·[±[À×ASÑASÓ5TÓ(UˆÔ%ð 	�‰Õùò cs   ÂE/NÚ	input_idsrï   rm   r¹   Úinputs_embedsÚ	use_cacherò   rG   c           	      óŒ  ‡— |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      s`| j                  ||||dœŠˆfd„ˆfd„d	œ}
t        | j                  j                  «      D �ci c]  }| |
|   «       “Œ }	}t        di ‰¤Ž|	d
<   |}t        | j                  j                  «      D �ci c]  }|| j                  |||«      “Œ }}|| j                   z   | j"                  z  j%                  t
        j&                  «      }d }t)        | j*                  «      D ]G  \  }}| j                  j                  |   } |||f|	|   |	j-                  d
«      dœ|||   dœ|¤Ž\  }}ŒI | j/                  |j%                  | j.                  j0                  j2                  ¬«      «      }t5        ||r|¬«      S d ¬«      S c c}w c c}w )Nz:You must specify exactly one of input_ids or inputs_embedsr¦  r   r!   )rE   ©r&   r³  rï   r¹   rm   c                  ó   •— t        di ‰ ¤ŽS ©Nr0  ©r   ©Úmask_kwargss   €rC   ú<lambda>z#ZayaModel.forward.<locals>.<lambda>ï  s   ø€ Ô"4Ñ"C°{Ñ"C€ rD   c                  ó   •— t        di ‰ ¤ŽS r¸  ©r   rº  s   €rC   r¼  z#ZayaModel.forward.<locals>.<lambda>ð  s   ø€ Ô*KÑ*ZÈkÑ*Z€ rD   ©Úhybridr  r  )r  r  )rï   r¹   r  rM   )Úlast_hidden_stater¹   r0  )Ú
ValueErrorr«  r   r&   Úget_seq_lengthrT   rU   rd   rE   rÆ   re   r   r6   r7   r   r¯  r–  r•  rW   rŒ   Ú	enumeraterÀ   rO   rç   r‡   rN   r   )r=   r²  rï   rm   r¹   r³  r´  rò   Úpast_seen_tokensÚcausal_mask_mappingÚmask_creation_functionsr+   r‰   r  r.  ÚidxÚdecoder_layerr»  s                    @rC   rs   zZayaModel.forwardË  s‘  ø€ ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-ÄÔFàŸ+™+Ø!.Ø"0Ø#2Ø ,ñˆKó DÛ"Zñ'Ð#ô
 UXÐX\×XcÑXc×XoÑXoÓTpö#ØFP�
Ð?Ð3°JÑ?ÓAÑAð#Ðð #ô +JÑ*XÈKÑ*XÐ Ñ'à%ˆô " $§+¡+×"9Ñ"9Ó:ö
àð ˜Ÿ™¨°|ÀZÓPÑPð
Ðð 
ð (¨$×*GÑ*GÑGÈ4×KiÑKiÑi×mÑmÜ�M‰Mó
ˆð %)Ð!ä"+¨D¯K©KÓ"8ò 	ÑˆC�ØŸ™×0Ñ0°Ñ5ˆJñ 8EØØ)ð
8ð 2°*Ñ=Ø/×3Ñ3°FÓ;ñ ð !0Ø$7¸
Ñ$Cñ
8ð ñ
8Ñ4ˆMÑ4ð		ð  Ÿ	™	 -×"2Ñ"2¸¿¹×9IÑ9I×9OÑ9OÐ"2Ó"PÓQˆä%Ø+Ù/8˜Oô
ð 	
à>Bô
ð 	
ùòK#ùò
s   Ã+H<Ä0I)NNNNNN)rv   rw   rx   r"   r1   r   r    r   rT   Ú
LongTensorry   r
   ÚFloatTensorÚboolr   r   r   rs   r~   r   s   @rC   r”  r”  ·  s×   ø„ ð˜zõ ð$  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñL
à×#Ñ# dÑ*ðL
ð Ÿ™ tÑ+ðL
ð ×&Ñ&¨Ñ-ð	L
ð
  ™ðL
ð ×(Ñ(¨4Ñ/ðL
ð ˜$‘;ðL
ð Ð+Ñ,ðL
ð 
 òL
ó ó ó  ôL
rD   r”  zZyphra/ZAYA1-8B)Ú
checkpointc                   óL  ‡ — e Zd ZddiZdZˆ fd„Zee	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  d	edz  d
ej                  dz  dej                  dz  dedz  dedz  deej                  z  dee   defd„«       «       Zedd„«       Zˆ xZS )ÚZayaForCausalLMzlm_head.weightzmodel.embed_tokens.weightTc                 ó  •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  |j                  | j                  j                  ¬«      | _
        | j                  «        y )NrŸ   )r0   r1   r”  r�  r©  r   r­   rQ   r&   Úlm_head_biasÚlm_headr±  )r=   r&   rò   rB   s      €rC   r1   zZayaForCausalLM.__init__"  s]   ø€ Ü‰Ñ˜Ô Ü˜vÓ&ˆŒ
Ø ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈTÏ[É[×MeÑMeÔfˆŒØ�‰ÕrD   Nr²  rï   rm   r¹   r³  Úlabelsr´  Úoutput_router_logitsÚlogits_to_keeprò   rG   c
                 ó®  — |�|n| j                   j                  } | j                  d|||||||dœ|
¤Ž}|j                  }t	        |	t
        «      rt        |	 d«      n|	}| j                  |dd…|dd…f   «      }d}|� | j                  ||| j                  fi |
¤Ž}t        |||j                  |j                  |j                  |j                  ¬«      S )aÉ  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, ZayaForCausalLM

        >>> model = ZayaForCausalLM.from_pretrained("meta-zaya/Zaya-2-7b-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("meta-zaya/Zaya-2-7b-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)r²  rï   rm   r¹   r³  r´  rÔ  )ÚlossÚlogitsr¹   r‰   rŽ  r\  r0  )r&   rÔ  r�  rÁ  re   rS   ÚslicerÒ  Úloss_functionr©  r   r¹   r‰   rŽ  r\  )r=   r²  rï   rm   r¹   r³  rÓ  r´  rÔ  rÕ  rò   Úoutputsr‰   Úslice_indicesrØ  r×  s                   rC   rs   zZayaForCausalLM.forward)  sø   € ðB %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð +5¨$¯*©*ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ñ	+
ð ñ	+
ˆð  ×1Ñ1ˆÜ8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ% f¨f°d·o±oÑPÈÑPˆDä(ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
rD   c                 óÊ   ‡
— | j                  «       }|||||dœŠ
ˆ
fd„ˆ
fd„dœ}t        |j                  «      D �ci c]  }| ||   «       “Œ }	}t        di ‰
¤Ž|	d<   |	S c c}w )Nr¶  c                  ó   •— t        di ‰ ¤ŽS r¸  r¹  rº  s   €rC   r¼  z;ZayaForCausalLM.create_masks_for_generate.<locals>.<lambda>v  s   ø€ Ô0Ñ?°;Ñ?€ rD   c                  ó   •— t        di ‰ ¤ŽS r¸  r¾  rº  s   €rC   r¼  z;ZayaForCausalLM.create_masks_for_generate.<locals>.<lambda>w  s   ø€ Ô&GÑ&VÈ+Ñ&V€ rD   r¿  r  r0  )Úget_text_configr6   r7   r   )r&   r³  rï   r¹   rm   r2  Útext_configrÇ  r+   r  r»  s             @rC   Úcreate_masks_for_generatez)ZayaForCausalLM.create_masks_for_generatei  s”   ø€ ð ×,Ñ,Ó.ˆà!Ø*Ø,Ø.Ø(ñ
ˆó @ÛVñ#
Ðô
 QTÐT_×TkÑTkÓPlö
ØBLˆJÐ;Ð/°
Ñ;Ó=Ñ=ð
ˆð 
ô  ?ÑMÀÑMˆ�VÑØÐùò	
s   ¼A )	NNNNNNNNr   ru   )rv   rw   rx   Ú_tied_weights_keysÚ_is_statefulr1   r   r   rT   rÊ  ry   r
   rË  rÌ  rS   r   r   r   rs   r{   râ  r~   r   s   @rC   rÏ  rÏ    s#  ø„ à*Ð,GÐHÐØ€Lôð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ñ<
à×#Ñ# dÑ*ð<
ð Ÿ™ tÑ+ð<
ð ×&Ñ&¨Ñ-ð	<
ð
  ™ð<
ð ×(Ñ(¨4Ñ/ð<
ð × Ñ  4Ñ'ð<
ð ˜$‘;ð<
ð # T™kð<
ð ˜eŸl™lÑ*ð<
ð Ð+Ñ,ð<
ð 
#ò<
ó ó ð<
ð| òó ôrD   rÏ  )rŒ  r”  rÏ  )r  )r!   )JÚcollections.abcr   Útypingr   r   rT   Útorch.nn.functionalr   rö   rÂ   Útorch.nnr   Úactivationsr	   Úcache_utilsr
   r   Ú
generationr   Úintegrationsr   r   Úmasking_utilsr   r   r   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   r   Úutils.output_capturingr   r    Úconfiguration_zayar"   ÚModuler$   r‚   ry   rS   rš   rœ   rÜ   rí   rX   rü   r  r
  r"  r+  r;  rE  rc  r&  rŒ  r”  rÏ  Ú__all__r0  rD   rC   ú<module>rù     s@  ðõ, %ß  ã ß Ð Ý Ý å !ß .Ý )ß Sß sÑ sÝ 9ß Qß Kß FÝ &ß IÑ Iß Gß EÝ *ôM<˜"Ÿ)™)ô M<ñ` ˜YÓ'ôJ�"—)‘)ó Jó (ðJð(	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uôl!˜Ÿ	™	ô l!ô^(�—‘ô (ò,(ð ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(ó%ó4#ôLI)�B—I‘Iô I)ôX,8Ð1ô ,8ô^(˜"Ÿ)™)ô (ô,�B—I‘Iô ,ô <
�—‘ô <
ð~ ô$#�"—)‘)ó $#ó ð$#ôN8˜Ÿ™ô 8ð0 ô/X˜/ó /Xó ð/Xðd ôb
Ð#ó b
ó ðb
ñJ Ð,Ô-ô_Ð)¨?ó _ó .ð_òD B�rD   