Ë
    ÜÓ^j¥p  ã                   ób  — d Z ddlmZ ddlmZmZ ddlZddlmc m	Z
 ddlZddlmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZmZ ddlmZ ddlmZ ddlmZmZ ddl m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z* ddl+m,Z,m-Z- ddl.m/Z/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6 ddl7m8Z8m9Z9  e$d¬«      e G d„ de*«      «       «       Z: G d„ de-«      Z; G d „ d!e9«      Z< G d"„ d#ejz                  «      Z> G d$„ d%ejz                  «      Z? G d&„ d'e3«      Z@ G d(„ d)e/«      ZA G d*„ d+ejz                  «      ZB G d,„ d-ejz                  «      ZC G d.„ d/ejz                  «      ZD G d0„ d1e8«      ZE G d2„ d3ejz                  «      ZF G d4„ d5e0«      ZGe$ G d6„ d7e,«      «       ZH e$d¬«       G d8„ d9e(eG«      «       ZIg d:¢ZJy);zPyTorch Zaya model.é    )ÚCallable)ÚAnyÚLiteralN)Ústrict)Únn)Úinité   )ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_maskÚcreate_recurrent_attention_maskÚ!create_sliding_window_causal_mask)ÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONS)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstring)ÚOutputRecorderé   )ÚAfmoeForCausalLM)ÚLagunaConfig)ÚLagunaModelÚLagunaRotaryEmbedding)ÚLlamaDecoderLayerÚLlamaPreTrainedModelÚ	repeat_kv)ÚPhi3Attention)Úapply_rotary_pos_embÚeager_attention_forward)ÚQwen3MoeExpertsÚQwen3MoeRMSNormzZyphra/ZAYA1-8B)Ú
checkpointc                   óÆ  — e Zd ZU dZdZdZeed<   dZeed<   dZ	eed<   d	Z
eed
<   dZeed<   dZeed<   dZedz  ed<   dZedz  ed<   d	Zedz  ed<   dZeee   z  dz  ed<   dZeed<   dZeed<   dZeed<   dZeed<   d	Zeed<   d	Zeed<    e«       Z e«       Z e«       Z e«       Z e«       Z e«       Z  e«       Z! e«       Z" e«       Z# e«       Z$ e«       Z%d „ Z&d!„ Z'd"„ Z(y)#Ú
ZayaConfiga€  
    lm_head_bias (`bool`, *optional*, defaults to `False`):
        Whether to add a bias to the language modeling head.
    router_hidden_size (`int`, *optional*, defaults to 256):
        Hidden size used by the ZAYA router.
    cca_time0 (`int`, *optional*, defaults to 2):
        First temporal parameter of the CCA projection.
    cca_time1 (`int`, *optional*, defaults to 2):
        Second temporal parameter of the CCA projection.

    ```python
    >>> from transformers import ZayaConfig, ZayaModel

    >>> configuration = ZayaConfig()
    >>> model = ZayaModel(configuration)

    >>> configuration = model.config
    ```
    Úzayai€  Ú
vocab_sizei   Úmoe_intermediate_sizeé   Únum_attention_headsr   Únum_key_value_headsTÚtie_word_embeddingsgñhãˆµøä>Úrms_norm_epsNÚsliding_windowr   Úpad_token_idÚbos_token_idéj   Úeos_token_idé   Únum_experts_per_toké   Únum_expertsFÚlm_head_biasé   Úrouter_hidden_sizeÚ	cca_time0Ú	cca_time1c                 óè   — | j                   €dg| j                  z  nt        | j                   «      | _         ddddœddddœdœ}| j                  €|| _        t	        j
                  | fi |¤ddd	hi¤Ž y )
NÚhybridÚdefaultg    ÐSAç      à?)Ú	rope_typeÚ
rope_thetaÚpartial_rotary_factorg     ˆÃ@©r?   Úhybrid_slidingÚignore_keys_at_rope_validationrF   )Úlayer_typesÚnum_hidden_layersÚlistÚrope_parametersr   Ú__post_init__)ÚselfÚkwargsÚdefault_rope_paramss      úp/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/zaya/modular_zaya.pyrL   zZayaConfig.__post_init__n   s�   € ØBF×BRÑBRÐBZ˜H˜:¨×(>Ñ(>Ò>Ô`dÐei×euÑeuÓ`vˆÔð 'Ø)Ø),ñð 'Ø&Ø),ññZ
Ðð ×ÑÐ'Ø#6ˆDÔ ä×&Ñ& tÑs¨vÑsÐW_ÐaqÐVrÔsó    c                 ó   — |S ©N© )rM   rN   s     rP   Úconvert_rope_params_to_dictz&ZayaConfig.convert_rope_params_to_dict‚   s   € àˆrQ   c                 óÒ   — | j                   dk7  rt        d«      ‚| j                  | j                  z  dk7  rt        d«      ‚d| j                  v r| j
                  €t        d«      ‚y y )Nr5   z5ZAYA currently supports `num_experts_per_tok=1` only.r   zB`num_attention_heads` must be a multiple of `num_key_value_heads`.rF   zJ`sliding_window` must be set when `layer_types` contains `hybrid_sliding`.)r6   Ú
ValueErrorr,   r-   rH   r0   )rM   s    rP   Úvalidate_architecturez ZayaConfig.validate_architecture†   so   € Ø×#Ñ# qÒ(ÜÐTÓUÐUØ×#Ñ# d×&>Ñ&>Ñ>À!ÒCÜÐaÓbÐbØ˜t×/Ñ/Ñ/°D×4GÑ4GÐ4OÜÐiÓjÐjð 5PÐ/rQ   ))Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typer)   ÚintÚ__annotations__r*   r,   r-   r.   Úboolr/   Úfloatr0   r1   r2   r4   rJ   r6   r8   r9   r;   r<   r=   ÚAttributeErrorÚbase_model_tp_planÚbase_model_pp_planÚintermediate_sizeÚshared_expert_intermediate_sizeÚrouter_aux_loss_coefÚnum_attention_heads_per_layerÚgatingÚmlp_layer_typesÚmoe_routed_scaling_factorÚ moe_apply_router_weight_on_inputÚmoe_router_logit_softcappingrL   rU   rX   rT   rQ   rP   r'   r'   3   sA  … ñð( €Jà€J�ÓØ!%Ð˜3Ó%Ø Ð˜Ó Ø Ð˜Ó Ø $Ð˜Ó$Ø€L�%ÓØ!%€N�C˜$‘JÓ%Ø €L�#˜‘*Ó Ø €L�#˜‘*Ó Ø+.€L�#˜˜S™	‘/ DÑ(Ó.à Ð˜Ó Ø€K�Óà€L�$ÓØ!Ð˜Ó!Ø€IˆsÓØ€IˆsÓñ
 (Ó)ÐÙ'Ó)ÐÙ&Ó(ÐÙ&4Ó&6Ð#Ù)Ó+ÐÙ$2Ó$4Ð!ÙÓ€FÙ$Ó&€OÙ .Ó 0ÐÙ'5Ó'7Ð$Ù#1Ó#3Ð òtò(ókrQ   r'   c                   ó   — e Zd Zy)ÚZayaRotaryEmbeddingN©rY   rZ   r[   rT   rQ   rP   ro   ro   �   ó   „ ØrQ   ro   c                   ó   — e Zd Zy)ÚZayaRMSNormNrp   rT   rQ   rP   rs   rs   “   rq   rQ   rs   c                   óx   ‡ — e Zd ZdZdedefˆ fd„Z	 d
dej                  de	dz  dej                  dz  fd	„Z
ˆ xZS )ÚZayaCCAProjectionav  
    Projects hidden states into attention q/k/v states with ZAYA's Compressed Convolutional Attention (CCA) path.
    See https://huggingface.co/papers/2510.04476.

    This follows the usual q/k/v projection flow, with three ZAYA-specific changes: q/k are mixed by a causal 1D
    convolution, q/k keep residual projection paths, and v uses a delayed recurrent state.
    ÚconfigÚ	layer_idxc                 ó¶  •— t         ‰| �  «        || _        || _        |j                  | _        |j
                  | _        |j                  | _        | j                  dz
  | j                  dz
  z   | _	        |j                  | _
        |j                  | _        |j                  | _        | j                  | j                  z  | _        | j                  | j                  z  }| j                  | j                  z  }t        j                  | j                  || j                  j                   ¬«      | _        t        j                  | j                  || j                  j                   ¬«      | _        t        j                  | j                  |dz  | j                  j                   ¬«      | _        t        j                  | j                  |dz  | j                  j                   ¬«      | _        ||z   }t        j*                  ||| j                  |dd¬«      | _        t        j*                  ||| j                  | j                  | j                  z   dd¬«      | _        y )Nr5   ©Úbiasr   r   )Úin_channelsÚout_channelsÚkernel_sizeÚgroupsÚpaddingÚstride)ÚsuperÚ__init__rv   rw   Úhidden_sizer<   Údepthwise_kernel_sizer=   Úgrouped_kernel_sizeÚconv_kernel_sizer-   r,   Úhead_dimÚnum_key_value_groupsr   ÚLinearÚattention_biasÚq_projÚk_projÚv_proj_currentÚv_proj_delayedÚConv1dÚconv_qk_depthwiseÚconv_qk_grouped)rM   rv   rw   Úquery_hidden_sizeÚkey_value_hidden_sizeÚconv_channelsÚ	__class__s         €rP   r‚   zZayaCCAProjection.__init__    sì  ø€ Ü‰ÑÔØˆŒØ"ˆŒà!×-Ñ-ˆÔà%+×%5Ñ%5ˆÔ"Ø#)×#3Ñ#3ˆÔ Ø!%×!;Ñ!;¸aÑ!?ÀD×D\ÑD\Ð_`ÑD`Ñ aˆÔà#)×#=Ñ#=ˆÔ Ø#)×#=Ñ#=ˆÔ ØŸ™ˆŒØ$(×$<Ñ$<À×@XÑ@XÑ$XˆÔ!à ×4Ñ4°t·}±}ÑDÐØ $× 8Ñ 8¸4¿=¹=Ñ HÐä—i‘i × 0Ñ 0Ð2CÈ$Ï+É+×JdÑJdÔeˆŒÜ—i‘i × 0Ñ 0Ð2GÈdÏkÉk×NhÑNhÔiˆŒÜ Ÿi™i¨×(8Ñ(8Ð:OÐSTÑ:TÐ[_×[fÑ[f×[uÑ[uÔvˆÔÜ Ÿi™i¨×(8Ñ(8Ð:OÐSTÑ:TÐ[_×[fÑ[f×[uÑ[uÔvˆÔà-Ð0AÑAˆÜ!#§¡Ø%Ø&Ø×2Ñ2Ø ØØô"
ˆÔô  "Ÿy™yØ%Ø&Ø×0Ñ0Ø×,Ñ,¨t×/GÑ/GÑGØØô 
ˆÕrQ   NÚhidden_statesÚpast_key_valuesÚ	conv_maskc                 ó8  — |�(||d d …d d …d f   j                  |j                  «      z  }|j                  d d }g |¢d‘| j                  ‘­}| j	                  |«      }| j                  |«      }t        j                  ||gd¬«      } |j                  |Ž }	 |j                  |Ž j                  dd«      }
t        |
| j                  «      j                  dd«      }
|	|
z   dz  }	 |	j                  g |¢d‘| j                  ‘| j                  ‘­Ž j                  d¬«      }
|j                  dd«      }|d uxr |j                  | j                  «      }|r@|j                  | j                     j                   d   }t        j                  ||gd¬«      }n"t#        j$                  || j&                  df«      }|�b|d| j&                   d …f   }t#        j$                  || j&                  |j                  d   z
  df«      }|j)                  || j                  «       | j+                  |«      }| j-                  |«      j                  dd«      }|	j                  d   |	j                  d   z  } |dd |…f   j                  |Ž |	z   } |d|d …f   j                  |Ž |
z   }| j/                  |«      }| j1                  |«      }|r6|j                  | j                     j2                  d   j5                  d«      }n/| j1                  |j7                  |d   d| j8                  «      «      }t        j                  ||d d …d d…f   gd¬«      }|�&|j;                  |d d …dd d …f   | j                  «        t        j                  ||gd¬«      j                  |Ž }|||fS )	Néÿÿÿÿ©Údimr5   r   rA   éþÿÿÿr   .)ÚtoÚdtypeÚshaper‡   r‹   rŒ   ÚtorchÚcatÚviewÚ	transposer   rˆ   ÚmeanÚhas_previous_staterw   ÚlayersÚconv_statesÚFÚpadr†   Úupdate_conv_stater�   r‘   r�   rŽ   Úrecurrent_statesÚ	unsqueezeÚ	new_zerosrƒ   Úupdate_recurrent_state)rM   r–   r—   r˜   Úinput_shapeÚhidden_shapeÚprojected_queriesÚprojected_keysÚ	qk_statesÚquery_residualÚkey_residualÚuse_precomputed_statesÚcached_qk_statesÚnew_conv_stater’   ÚqueryÚkeyÚvalue_currentÚdelayed_v_stateÚrecurrent_v_stateÚvalue_delayedÚvalues                         rP   ÚforwardzZayaCCAProjection.forwardÊ   s¬  € ð Ð Ø)¨I²aº¸D°jÑ,A×,DÑ,DÀ]×EXÑEXÓ,YÑYˆMà#×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà ŸK™K¨Ó6ÐØŸ™ ]Ó3ˆÜ—I‘IÐ0°.ÐAÀrÔJˆ	à/Ð*×/Ñ/°Ð>ˆØ*�~×*Ñ*¨LÐ9×CÑCÀAÀqÓIˆÜ  ¨t×/HÑ/HÓI×SÑSÐTUÐWXÓYˆØ(¨<Ñ7¸3Ñ>ˆØ*�~×*Ñ*Ðf¨KÐf¸Ðf¸T×=VÑ=VÐfÐX\×XeÑXeÒf×kÑkÐprÐkÓsˆà×'Ñ'¨¨1Ó-ˆ	Ø!0¸Ð!<Ò!sÀ×AcÑAcÐdh×drÑdrÓAsÐÙ!Ø.×5Ñ5°d·n±nÑE×QÑQÐRSÑTÐÜŸ	™	Ð#3°YÐ"?ÀRÔH‰IäŸ™˜i¨$×*?Ñ*?ÀÐ)CÓDˆIàÐ&Ø& s¨T×-BÑ-BÐ,BÑ,DÐ'DÑEˆNÜŸU™U >°D×4IÑ4IÈN×L`ÑL`ÐacÑLdÑ4dÐfgÐ3hÓiˆNØ×-Ñ-¨n¸d¿n¹nÔMà×*Ñ*¨9Ó5ˆ	Ø×(Ñ(¨Ó3×=Ñ=¸aÀÓCˆ	à*×0Ñ0°Ñ4°~×7KÑ7KÈBÑ7OÑOÐØ7�	˜#Ð1Ð 1Ð1Ð1Ñ2×7Ñ7¸ÐFÈÑWˆØ5ˆi˜Ð.Ñ/Ð/Ñ0×5Ñ5°|ÐDÀ|ÑSˆð ×+Ñ+¨MÓ:ˆØ×-Ñ-¨mÓ<ˆÙ!Ø /× 6Ñ 6°t·~±~Ñ F× WÑ WÐXYÑ Z× dÑ dÐefÓ gÑà $× 3Ñ 3°M×4KÑ4KÈKÐXYÉNÐ\]Ð_c×_oÑ_oÓ4pÓ qÐÜŸ	™	Ð#4°oÂaÈÈ"ÈÀfÑ6MÐ"NÐTUÔVˆàÐ&Ø×2Ñ2°?Â1ÀbÊ!À8Ñ3LÈdÏnÉnÔ]àF”—	‘	˜=¨-Ð8¸bÔA×FÑFÈÐUˆà�c˜5Ð Ð rQ   rS   )rY   rZ   r[   r\   r'   r^   r‚   r¡   ÚTensorr
   rÁ   Ú__classcell__©r•   s   @rP   ru   ru   —   sS   ø„ ñð(
˜zð (
°cõ (
ð\ *.ñ	9!à—|‘|ð9!ð  ™ð9!ð —<‘< $Ñ&÷	9!rQ   ru   c                   ó–   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  deej                  ej                  f   fd„Z	ˆ xZ
S )Ú
ZayaQKNormzf
    L2-normalizes q/k states to sqrt(head_dim) and applies ZAYA's learned per-KV-head key scale.
    rv   c                 ó¸   •— t         ‰| �  «        |j                  dz  | _        t	        j
                  t        j                  |j                  «      «      | _	        y )NrA   )
r�   r‚   r‡   Úhead_dim_scaler   Ú	Parameterr¡   Úzerosr-   Útemp©rM   rv   r•   s     €rP   r‚   zZayaQKNorm.__init__  s>   ø€ Ü‰ÑÔØ$Ÿo™o¨sÑ2ˆÔÜ—L‘L¤§¡¨V×-GÑ-GÓ!HÓIˆ�	rQ   Úquery_statesÚ
key_statesÚreturnc                 óX  — t        j                  |j                  «      j                  }|| j                  |j                  ddd¬«      j                  |«      z  z  }|| j                  |j                  ddd¬«      j                  |«      z  z  }|| j                  d d d d …d f   z  }||fS )Nr   rš   T)Úprœ   Úkeepdim)r¡   ÚfinforŸ   ÚepsrÈ   ÚnormÚ	clamp_minrË   )rM   rÍ   rÎ   Únorm_epss       rP   rÁ   zZayaQKNorm.forward  s°   € Ü—;‘;˜|×1Ñ1Ó2×6Ñ6ˆØ#Ø×Ñ ,×"3Ñ"3°a¸RÈÐ"3Ó"N×"XÑ"XÐYaÓ"bÑbñ
ˆð  Ø×Ñ *§/¡/°A¸2Àt /Ó"L×"VÑ"VÐW_Ó"`Ñ`ñ
ˆ
ð   $§)¡)¨D°$º¸4Ð,?Ñ"@Ñ@ˆ
Ø˜ZÐ'Ð'rQ   )rY   rZ   r[   r\   r'   r‚   r¡   rÂ   ÚtuplerÁ   rÃ   rÄ   s   @rP   rÆ   rÆ     sP   ø„ ñðJ˜zõ Jð
	( E§L¡Lð 	(¸e¿l¹lð 	(ÈuÐUZ×UaÑUaÐch×coÑcoÐUoÑOp÷ 	(rQ   rÆ   c                   óð   ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej                  dee	e
f   dz  dedz  deej                  ej                  f   dz  d	ee   d
eej                  ej                  dz  f   fd„Zˆ xZS )ÚZayaAttentionrv   rw   c                 óÀ  •— t         ‰| �  ||«       ~|j                  |   | _        | j                  dk(  r|j                  nd | _        |j
                  | _        |j                  | _        t        j                  |j                  | j                  z  |j
                  |j                  ¬«      | _        t        |«      | _        t        | j                  |¬«      | _        y )NrF   ry   )rv   rw   )r�   r‚   rH   Ú
layer_typer0   rƒ   r,   r   r‰   r‡   rŠ   Úo_projrÆ   Úqk_normru   rv   Úqkv_proj)rM   rv   rw   Úop_sizer•   s       €rP   r‚   zZayaAttention.__init__  s·   ø€ Ü‰Ñ˜ Ô+ØØ ×,Ñ,¨YÑ7ˆŒØ7;·±ÐJZÒ7Z˜f×3Ò3Ð`dˆÔØ!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ ä—i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒô " &Ó)ˆŒÜ)Ø—;‘;Øô
ˆ�rQ   Nr–   Úattention_maskr—   Úposition_embeddingsrN   rÏ   c                 óÒ  — |j                   d d }|xs i }|j                  d«      }|j                  d«      }	| j                  |||	«      \  }
}}| j                  |
|«      \  }
}|
j	                  dd«      }
|j	                  dd«      }|j	                  dd«      }|\  }}t        |
|||«      \  }
}|� |j                  ||| j                  «      \  }}t        j                  | j                  j                  t        «      } || |
|||f| j                  sdn| j                  | j                  | j                   dœ|¤Ž\  }} |j"                  g |¢d‘­Ž }| j%                  |«      }||fS )Nrš   ÚcausalÚconvr5   r   ç        )ÚdropoutÚscalingr0   )r    Úgetrß   rÞ   r¤   r!   Úupdaterw   r   Úget_interfacerv   Ú_attn_implementationr"   ÚtrainingÚattention_dropoutrè   r0   ÚreshaperÝ   )rM   r–   rá   r—   râ   rN   r°   Úmask_mappingÚcausal_maskr˜   rÍ   rÎ   Úvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                     rP   rÁ   zZayaAttention.forward.  s•  € ð $×)Ñ)¨#¨2Ð.ˆà%Ò+¨ˆØ"×&Ñ& xÓ0ˆØ ×$Ñ$ VÓ,ˆ	ð 26·±¸}ÈoÐ_hÓ1iÑ.ˆ�j ,Ø#'§<¡<°¸jÓ#IÑ ˆ�jà#×-Ñ-¨a°Ó3ˆØ×)Ñ)¨!¨QÓ/ˆ
Ø#×-Ñ-¨a°Ó3ˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;ˆØ—k‘k +Ó.ˆà˜LÐ(Ð(rQ   )NNN©rY   rZ   r[   r'   r^   r‚   r¡   rÂ   ÚdictÚstrr   r
   rØ   r   r   rÁ   rÃ   rÄ   s   @rP   rÚ   rÚ     s©   ø„ ð
˜zð 
°cõ 
ð( 15Ø(,ØHLñ.)à—|‘|ð.)ð ˜S #˜X™¨Ñ-ð.)ð  ™ð	.)ð
 # 5§<¡<°·±Ð#=Ñ>ÀÑEð.)ð Ð+Ñ,ð.)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷.)rQ   rÚ   c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej                  dej                  dz  dee	e
f   dz  dedz  d	eej                  ej                  f   dz  d
ee   deej                  ej                  dz  f   fd„Zˆ xZS )ÚZayaDecoderLayerrv   rw   c                 ó°   •— t         ‰| �  ||«       t        ||«      | _        t	        |j
                  «      | _        t	        |j
                  «      | _        y rS   )r�   r‚   ÚZayaSparseMoeBlockÚmlpÚZayaResidualScalingrƒ   Úpost_attention_residual_scaleÚpost_mlp_residual_scale©rM   rv   rw   r•   s      €rP   r‚   zZayaDecoderLayer.__init__`  sF   ø€ Ü‰Ñ˜ Ô+Ü% f¨iÓ8ˆŒÜ-@À×ASÑASÓ-TˆÔ*Ü':¸6×;MÑ;MÓ'NˆÕ$rQ   Nr–   Úprev_router_hidden_statesrá   r—   râ   rN   rÏ   c                 ó°  — |}| j                  |j                  | j                   j                  j                  ¬«      «      } | j                  d||||dœ|¤Ž\  }}| j                  ||«      }| j                  |j                  | j                  j                  j                  ¬«      «      }| j                  ||«      \  }}| j                  ||«      }||fS )N©rŸ   )r–   rá   r—   râ   rT   )	Úinput_layernormrž   ÚweightrŸ   Ú	self_attnr  Úpost_attention_layernormrÿ   r  )	rM   r–   r  rá   r—   râ   rN   ÚresidualÚ_s	            rP   rÁ   zZayaDecoderLayer.forwardf  sã   € ð !ˆð ×,Ñ,¨X¯[©[¸t×?SÑ?S×?ZÑ?Z×?`Ñ?`¨[Ó-aÓbˆà)˜4Ÿ>™>ð 
Ø'Ø)Ø+Ø 3ñ	
ð
 ñ
Ñˆ�qð ×5Ñ5°mÀXÓNˆØ×5Ñ5°h·k±kÈ×HeÑHe×HlÑHl×HrÑHr°kÓ6sÓtˆà37·8±8ØØ%ó4
Ñ0ˆÐ0ð
 ×4Ñ4°]ÀHÓMˆàÐ7Ð7Ð7rQ   )NNNNrø   rÄ   s   @rP   rü   rü   _  sÂ   ø„ ðO˜zð O°cõ Oð :>Ø04Ø(,ØHLñ 8à—|‘|ð 8ð $)§<¡<°$Ñ#6ð 8ð ˜S #˜X™¨Ñ-ð	 8ð
  ™ð 8ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð 8ð Ð+Ñ,ð 8ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷ 8rQ   rü   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )r   rƒ   c                 óŠ  •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        t        j                  t	        j                  |«      «      | _        t        j                  t	        j
                  |«      «      | _	        t        j                  t	        j                  |«      «      | _
        y rS   )r�   r‚   r   rÉ   r¡   ÚonesÚhidden_states_scalerÊ   Úhidden_states_biasÚresidual_scaleÚresidual_bias)rM   rƒ   r•   s     €rP   r‚   zZayaResidualScaling.__init__Š  sv   ø€ Ü‰ÑÔÜ#%§<¡<´·
±
¸;Ó0GÓ#HˆÔ Ü"$§,¡,¬u¯{©{¸;Ó/GÓ"HˆÔÜ Ÿl™l¬5¯:©:°kÓ+BÓCˆÔÜŸ\™\¬%¯+©+°kÓ*BÓCˆÕrQ   r–   r  c                 ó|   — || j                   z   | j                  z  }|| j                  z   | j                  z  }||z   S rS   )r  r  r  r  )rM   r–   r  s      rP   rÁ   zZayaResidualScaling.forward‘  sC   € à&¨×)@Ñ)@Ñ@ÀD×D\ÑD\Ñ\ˆØ˜t×1Ñ1Ñ1°T×5HÑ5HÑHˆØ˜xÑ'Ð'rQ   )	rY   rZ   r[   r^   r‚   r¡   rÂ   rÁ   rÃ   rÄ   s   @rP   r   r   ‰  s,   ø„ ðD Cõ Dð( U§\¡\ð (¸U¿\¹\÷ (rQ   r   c                   ód   ‡ — e Zd Zdededefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚZayaRouterMLPrƒ   r8   r/   c                 ó&  •— t         ‰| �  «        t        ||¬«      | _        t	        j
                  ||d¬«      | _        t	        j
                  ||d¬«      | _        t	        j
                  ||d¬«      | _        t	        j                  «       | _
        y )N)rÔ   Try   F)r�   r‚   rs   rÕ   r   r‰   Úfc1Úfc2Úout_projÚGELUÚact_fn)rM   rƒ   r8   r/   r•   s       €rP   r‚   zZayaRouterMLP.__init__™  se   ø€ Ü‰ÑÔÜ °Ô>ˆŒ	Ü—9‘9˜[¨+¸DÔAˆŒÜ—9‘9˜[¨+¸DÔAˆŒÜŸ	™	 +¨{ÀÔGˆŒÜ—g‘g“iˆ�rQ   r–   rÏ   c                 óÆ   — | j                  |«      }| j                  | j                  |«      «      }| j                  | j                  |«      «      }| j	                  |«      S rS   )rÕ   r  r  r  r  )rM   r–   s     rP   rÁ   zZayaRouterMLP.forward¡  sM   € ØŸ	™	 -Ó0ˆØŸ™ D§H¡H¨]Ó$;Ó<ˆØŸ™ D§H¡H¨]Ó$;Ó<ˆØ�}‰}˜]Ó+Ð+rQ   )
rY   rZ   r[   r^   ra   r‚   r¡   rÂ   rÁ   rÃ   rÄ   s   @rP   r  r  ˜  s8   ø„ ð  Cð  °cð  Èõ  ð, U§\¡\ð ,°e·l±l÷ ,rQ   r  c                   óÌ   ‡ — e Zd Zdeddfˆ fd„Z	 ddej                  dej                  dz  deej                  ej                  ej                  ej                  f   fd„Zˆ xZ	S )	Ú
ZayaRouterrw   rÏ   Nc                 óú  •— t         ‰| �  «        || _        |j                  | _        || _        |j
                  | _        |j
                  dz   | _        |j                  | _        |j                  | _	        t        j                  | j                  | j                  d¬«      | _        | j                  dk7  | _        | j                  r7t        j                  t        j                   | j                  «      «      | _        t%        | j                  | j                  |j&                  «      | _        | j+                  dt        j,                  | j                  t        j.                  ¬«      «       d| j0                  d<   y )	Nr5   Try   r   Úbalancing_biasesr  ç      ð¿rš   )r�   r‚   rv   rƒ   rw   r8   Únum_router_classesr6   Útop_kr;   r   r‰   Ú	down_projÚuse_edarÉ   r¡   r  Úrouter_states_scaler  r/   Ú
router_mlpÚregister_bufferrÊ   Úfloat32r!  r  s      €rP   r‚   zZayaRouter.__init__©  s  ø€ ô
 	‰ÑÔàˆŒØ!×-Ñ-ˆÔØ"ˆŒà!×-Ñ-ˆÔà"(×"4Ñ"4°qÑ"8ˆÔØ×/Ñ/ˆŒ
Ø"(×";Ñ";ˆÔäŸ™ 4×#3Ñ#3°T×5LÑ5LÐSWÔXˆŒà—~‘~¨Ñ*ˆŒØ�<Š<Ü')§|¡|´E·J±J¸t×?VÑ?VÓ4WÓ'XˆDÔ$ä'¨×(?Ñ(?À×AXÑAXÐZ`×ZmÑZmÓnˆŒà×ÑÐ/´·±¸T×=TÑ=TÔ\a×\iÑ\iÔ1jÔkØ$(ˆ×Ñ˜bÒ!rQ   r–   Úrouter_statesc                 ó  — d| j                   f}|j                  d   }| j                  |«      }| j                  r|�||| j                  z  z   }|d d …| d …f   j                  «       }| j                  |«      }t        j                  |d¬«      }|j                  «       j                  t        j                  «      | j                  z   }	t        j                  |	| j                   d¬«      \  }
}t        j                  |d|¬«      }|| j                  j                   k(  }|j#                  |d«      }|j#                  |d«      }|j%                  d| j&                  «      |j%                  |«      |j%                  |«      |fS )Nrš   r5   r›   r   )rœ   Úindexr   )r$  r    r%  r&  r'  Úcloner(  r¡   ÚsoftmaxÚdetachrž   r*  r!  ÚtopkÚgatherrv   r8   Úmasked_fillrï   r#  )rM   r–   r+  Úfinal_shapeÚ
seq_lengthÚrouter_hidden_statesÚrouter_hidden_states_nextÚrouter_logitsÚrouter_probsÚbiased_router_probsr  Úrouter_indicesÚskip_experts                rP   rÁ   zZayaRouter.forwardÅ  s`  € ð
 ˜4Ÿ:™:Ð&ˆØ"×(Ñ(¨Ñ+ˆ
à#Ÿ~™~¨mÓ<Ðà�<Š<˜MÐ5Ø#7¸-È$×JbÑJbÑ:bÑ#bÐ à$8º¸Z¸K¹L¸Ñ$I×$OÑ$OÓ$QÐ!ØŸ™Ð(<Ó=ˆÜ—}‘} ]¸Ô;ˆà*×1Ñ1Ó3×6Ñ6´u·}±}ÓEÈ×H]ÑH]Ñ]ÐÜ!ŸJ™JÐ':¸D¿J¹JÈBÔOÑˆˆ>Ü—|‘| L°a¸~ÔNˆð %¨¯©×(?Ñ(?Ñ?ˆØ#×/Ñ/°¸QÓ?ˆØ'×3Ñ3°KÀÓCˆð ×!Ñ! " d×&=Ñ&=Ó>Ø× Ñ  Ó-Ø×"Ñ" ;Ó/Ø%ð	
ð 	
rQ   rS   ©
rY   rZ   r[   r^   r‚   r¡   rÂ   rØ   rÁ   rÃ   rÄ   s   @rP   r  r  ¨  sm   ø„ ð)ð ð)ð 
õ	)ð> .2ñ
à—|‘|ð
ð —|‘| dÑ*ð
ð 
ˆu�|‰|˜UŸ\™\¨5¯<©<¸¿¹ÐEÑ	F÷	
rQ   r  c                   ó   — e Zd Zy)ÚZayaExpertsNrp   rT   rQ   rP   r?  r?  ç  rq   rQ   r?  c            
       ó¢   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dej                  dz  deej                  ej                  dz  f   fd„Zˆ xZ	S )	rþ   rw   c                 ód   •— t         ‰| �  «        t        ||«      | _        t	        |«      | _        y rS   )r�   r‚   r  Úgater?  Úexpertsr  s      €rP   r‚   zZayaSparseMoeBlock.__init__ì  s(   ø€ Ü‰ÑÔÜ˜v yÓ1ˆŒ	Ü" 6Ó*ˆ�rQ   Nr–   r  rÏ   c                 óÐ   — | j                  ||¬«      \  }}}}|j                  \  }}}|j                  ||z  |«      }	| j                  |	||«      }
|
j                  |||«      }
|
|fS )N)r+  )rB  r    r£   rC  )rM   r–   r  r  r9  r;  Ú
batch_sizer5  Úemb_dimÚhidden_states_flatÚexpert_outputs              rP   rÁ   zZayaSparseMoeBlock.forwardñ  s‹   € ð FJÇYÁYØÐ)Bð FOó F
ÑBˆˆ<˜Ð)Bð +8×*=Ñ*=Ñ'ˆ
�J Ø*×/Ñ/°
¸ZÑ0GÈÓQÐØŸ™Ð%7¸ÈÓVˆØ%×*Ñ*¨:°zÀ7ÓKˆàÐ7Ð7Ð7rQ   rS   r=  rÄ   s   @rP   rþ   rþ   ë  sY   ø„ ð+¨#õ +ð :>ñ8à—|‘|ð8ð $)§<¡<°$Ñ#6ð8ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷	8rQ   rþ   c                   óf   — e Zd ZU eed<   dZ eed¬«      ee	dœZ
 ej                  «       d„ «       Zy)ÚZayaPreTrainedModelrv   Fr   )r-  )r8  r–   Ú
attentionsc                 óD  — t        j                  | |«       t        |t        «      r}t	        j
                  |j                  «       t	        j                  |j                  «       t	        j
                  |j                  «       t	        j                  |j                  «       y t        |t        «      r?t	        j
                  |j                  «       t	        j                  |j                  «       y t        |t        «      r t	        j                  |j                  «       y t        |t         «      rZ|j"                  rt	        j
                  |j$                  «       t	        j                  |j&                  «       d|j&                  d<   y t        |t(        «      r[| j*                  j,                  }t	        j.                  |j0                  d|¬«       t	        j.                  |j2                  d|¬«       y t        |t4        «      r™|j6                  D ]‰  }|j8                  }|j:                  |   dk7  rt<        |j:                  |      } ||j*                  |¬«      \  }}t?        ||› d�«      jA                  |«       t?        ||› d�«      jA                  |«       Œ‹ y y )	Nr"  rš   ræ   )r¥   Ústdr@   )rÜ   Ú	_inv_freqÚ_original_inv_freq)!r   Ú_init_weightsÚ
isinstancer   r   Úones_r  Úzeros_r  r  r  Ú	ZayaModelÚinput_hidden_states_scaleÚinput_hidden_states_biasrÆ   rË   r  r&  r'  r!  r?  rv   Úinitializer_rangeÚnormal_Úgate_up_projr%  ro   rH   Úcompute_default_rope_parametersrB   r   ÚgetattrÚcopy_)rM   ÚmodulerM  rÜ   Úrope_init_fnÚcurr_inv_freqr  s          rP   rP  z!ZayaPreTrainedModel._init_weights  sß  € ä×%Ñ% d¨FÔ3Ü�fÔ1Ô2Ü�J‰J�v×1Ñ1Ô2Ü�K‰K˜×1Ñ1Ô2Ü�J‰J�v×,Ñ,Ô-Ü�K‰K˜×,Ñ,Õ-Ü˜¤	Ô*Ü�J‰J�v×7Ñ7Ô8Ü�K‰K˜×7Ñ7Õ8Ü˜¤
Ô+Ü�K‰K˜Ÿ™Õ$Ü˜¤
Ô+Ø�~Š~Ü—
‘
˜6×5Ñ5Ô6Ü�K‰K˜×/Ñ/Ô0Ø*.ˆF×#Ñ# BÒ'Ü˜¤Ô,Ø—+‘+×/Ñ/ˆCÜ�L‰L˜×,Ñ,°3¸CÕ@Ü�L‰L˜×)Ñ)°¸Ö=Ü˜Ô 3Ô4Ø$×0Ñ0ò X�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ò<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ô#UÑ �˜qÜ˜ : ,¨iÐ 8Ó9×?Ñ?ÀÔNÜ˜ : ,Ð.@Ð AÓB×HÑHÈÕWñXð 5rQ   N)rY   rZ   r[   r'   r_   Ú_can_compile_fullgraphr   r  rü   rÚ   Ú_can_record_outputsr¡   Úno_gradrP  rT   rQ   rP   rJ  rJ    sD   … ØÓà"Ðá'¨
¸!Ô<Ø)Ø#ñÐð €U‡]�]ƒ_ñXó ñXrQ   rJ  c                   óÔ   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	e
dz  d
ee   defd„Zˆ xZS )rT  rv   c                 ó   •— t         ‰| �  |«       t        j                  t	        j
                  |j                  «      «      | _        t        j                  t	        j                  |j                  «      «      | _	        y rS   )
r�   r‚   r   rÉ   r¡   r  rƒ   rU  rÊ   rV  rÌ   s     €rP   r‚   zZayaModel.__init__/  sP   ø€ Ü‰Ñ˜Ô Ü)+¯©´e·j±jÀ×ASÑASÓ6TÓ)UˆÔ&Ü(*¯©´U·[±[À×ASÑASÓ5TÓ(UˆÕ%rQ   NÚ	input_idsrá   Úposition_idsr—   Úinputs_embedsÚ	use_cacherN   rÏ   c           	      óŒ  ‡— |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      s`| j                  ||||dœŠˆfd„ˆfd„d	œ}
t        | j                  j                  «      D �ci c]  }| |
|   «       “Œ }	}t        di ‰¤Ž|	d
<   |}t        | j                  j                  «      D �ci c]  }|| j                  |||«      “Œ }}|| j                   z   | j"                  z  j%                  t
        j&                  «      }d }t)        | j*                  «      D ]G  \  }}| j                  j                  |   } |||f|	|   |	j-                  d
«      dœ|||   dœ|¤Ž\  }}ŒI | j/                  |j%                  | j.                  j0                  j2                  ¬«      «      }t5        ||r|¬«      S d ¬«      S c c}w c c}w )Nz:You must specify exactly one of input_ids or inputs_embeds)rv   r   r5   )Údevice©rv   rg  rá   r—   rf  c                  ó   •— t        di ‰ ¤ŽS ©NrT   ©r   ©Úmask_kwargss   €rP   ú<lambda>z#ZayaModel.forward.<locals>.<lambda>U  s   ø€ Ô"4Ñ"C°{Ñ"C€ rQ   c                  ó   •— t        di ‰ ¤ŽS rm  ©r   ro  s   €rP   rq  z#ZayaModel.forward.<locals>.<lambda>V  s   ø€ Ô*KÑ*ZÈkÑ*Z€ rQ   rE   rå   )rä   rå   )rá   r—   râ   r  )Úlast_hidden_stater—   rT   )rW   Úembed_tokensr   rv   Úget_seq_lengthr¡   Úaranger    rj  r­   rQ  rù   ÚsetrH   r   Ú
rotary_embrV  rU  rž   r*  Ú	enumerater§   ré   rÕ   r  rŸ   r   )rM   re  rá   rf  r—   rg  rh  rN   Úpast_seen_tokensÚcausal_mask_mappingÚmask_creation_functionsrÜ   r–   râ   r  ÚidxÚdecoder_layerrp  s                    @rP   rÁ   zZayaModel.forward4  s‘  ø€ ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-ÄÔFàŸ+™+Ø!.Ø"0Ø#2Ø ,ñˆKó DÛ"Zñ'Ð#ô
 UXÐX\×XcÑXc×XoÑXoÓTpö#ØFP�
Ð?Ð3°JÑ?ÓAÑAð#Ðð #ô +JÑ*XÈKÑ*XÐ Ñ'à%ˆô " $§+¡+×"9Ñ"9Ó:ö
àð ˜Ÿ™¨°|ÀZÓPÑPð
Ðð 
ð (¨$×*GÑ*GÑGÈ4×KiÑKiÑi×mÑmÜ�M‰Mó
ˆð %)Ð!ä"+¨D¯K©KÓ"8ò 	ÑˆC�ØŸ™×0Ñ0°Ñ5ˆJñ 8EØØ)ð
8ð 2°*Ñ=Ø/×3Ñ3°FÓ;ñ ð !0Ø$7¸
Ñ$Cñ
8ð ñ
8Ñ4ˆMÑ4ð		ð  Ÿ	™	 -×"2Ñ"2¸¿¹×9IÑ9I×9OÑ9OÐ"2Ó"PÓQˆä%Ø+Ù/8˜Oô
ð 	
à>Bô
ð 	
ùòK#ùò
s   Ã+H<Ä0I)NNNNNN)rY   rZ   r[   r'   r‚   r¡   Ú
LongTensorrÂ   r
   ÚFloatTensorr`   r   r   r   rÁ   rÃ   rÄ   s   @rP   rT  rT  -  s¹   ø„ ðV˜zõ Vð .2Ø.2Ø04Ø(,Ø26Ø!%ñL
à×#Ñ# dÑ*ðL
ð Ÿ™ tÑ+ðL
ð ×&Ñ&¨Ñ-ð	L
ð
  ™ðL
ð ×(Ñ(¨4Ñ/ðL
ð ˜$‘;ðL
ð Ð+Ñ,ðL
ð 
 ÷L
rQ   rT  c                   óX   ‡ — e Zd ZddiZdZ e«       Z e«       Zˆ fd„Ze	dd„«       Z
ˆ xZS )ÚZayaForCausalLMzlm_head.weightzmodel.embed_tokens.weightTc                 ó°   •— t        ‰| �  |fi |¤Ž t        j                  |j                  |j
                  | j                  j                  ¬«      | _        y )Nry   )	r�   r‚   r   r‰   rƒ   r)   rv   r9   Úlm_head)rM   rv   rN   r•   s      €rP   r‚   zZayaForCausalLM.__init__‹  s?   ø€ Ü‰Ñ˜Ñ* 6Ò*Ü—y‘y ×!3Ñ!3°V×5FÑ5FÈTÏ[É[×MeÑMeÔfˆ�rQ   c                 óÊ   ‡
— | j                  «       }|||||dœŠ
ˆ
fd„ˆ
fd„dœ}t        |j                  «      D �ci c]  }| ||   «       “Œ }	}t        di ‰
¤Ž|	d<   |	S c c}w )Nrk  c                  ó   •— t        di ‰ ¤ŽS rm  rn  ro  s   €rP   rq  z;ZayaForCausalLM.create_masks_for_generate.<locals>.<lambda>œ  s   ø€ Ô0Ñ?°;Ñ?€ rQ   c                  ó   •— t        di ‰ ¤ŽS rm  rs  ro  s   €rP   rq  z;ZayaForCausalLM.create_masks_for_generate.<locals>.<lambda>�  s   ø€ Ô&GÑ&VÈ+Ñ&V€ rQ   rE   rå   rT   )Úget_text_configrx  rH   r   )rv   rg  rá   r—   rf  r  Útext_configr}  rÜ   rð   rp  s             @rP   Úcreate_masks_for_generatez)ZayaForCausalLM.create_masks_for_generate�  s”   ø€ ð ×,Ñ,Ó.ˆà!Ø*Ø,Ø.Ø(ñ
ˆó @ÛVñ#
Ðô
 QTÐT_×TkÑTkÓPlö
ØBLˆJÐ;Ð/°
Ñ;Ó=Ñ=ð
ˆð 
ô  ?ÑMÀÑMˆ�VÑØÐùò	
s   ¼A rS   )rY   rZ   r[   Ú_tied_weights_keysÚ_is_statefulrb   Ú_tp_planÚ_pp_planr‚   Ústaticmethodr‹  rÃ   rÄ   s   @rP   rƒ  rƒ  ƒ  s>   ø„ à*Ð,GÐHÐØ€LáÓ€HÙÓ€Hôgð òó ôrQ   rƒ  )r'   rJ  rT  rƒ  )Kr\   Úcollections.abcr   Útypingr   r   r¡   Útorch.nn.functionalr   Ú
functionalr©   Útorch.utils.checkpointÚhuggingface_hub.dataclassesr   Útorch.nnr   Úcache_utilsr
   r   Úconfiguration_utilsr   Úmasking_utilsr   r   r   Úmodeling_outputsr   Úmodeling_rope_utilsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.output_capturingr   Úafmoe.modeling_afmoer   Úlaguna.configuration_lagunar   Úlaguna.modeling_lagunar   r   Úllama.modeling_llamar   r   r   Úphi3.modeling_phi3r    Ú qwen3_5_moe.modeling_qwen3_5_moer!   r"   Úqwen3_moe.modeling_qwen3_moer#   r$   r'   ro   rs   ÚModuleru   rÆ   rÚ   rü   r   r  r  r?  rþ   rJ  rT  rƒ  Ú__all__rT   rQ   rP   ú<module>rª     s–  ðñ å $ß ã ß Ð Û Ý .Ý Ý ç .Ý 3ß sÑ sÝ 6Ý 6ß FÝ &÷õ 5Ý 3Ý 6ß Gß UÑ UÝ .÷÷ Lñ Ð,Ô-ØôWk�ó Wkó ó .ðWkôt	Ð/ô 	ô	�/ô 	ôl!˜Ÿ	™	ô l!ô^(�—‘ô (ô,@)�Mô @)ôF'8Ð(ô '8ôT(˜"Ÿ)™)ô (ô,�B—I‘Iô ,ô <
�—‘ô <
ô~	�/ô 	ô8˜Ÿ™ô 8ô0'XÐ.ô 'XðT ôR
�ó R
ó ðR
ñj Ð,Ô-ôÐ&Ð(;ó ó .ðòD�rQ   