Ë
    ÜÓ^j¶  ã                   ó‚   — d dl mZmZ d dlmZ ddlmZ ddlmZ ddl	m
Z
  e
d¬«      e G d	„ d
e«      «       «       Zd
gZy)é    )ÚAnyÚLiteral)Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringzZyphra/ZAYA1-8B)Ú
checkpointc                   ó  ‡ — e Zd ZU dZdZdgZdddddœZdZee	d	<   d
Z
ee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZeez  dz  e	d<   dZedz  e	d<   d Zeez  e	d!<   d
Zee	d"<   d#Zee	d$<   d%Zee	d&<   d'Zee	d(<   dZ e!e   dz  e	d)<   d*Z"edz  e	d+<   dZ#edz  e	d,<   d-Z$ee!e   z  dz  e	d.<   d/Z%ee	d0<   d'Z&ee	d1<   d'Z'ee	d2<   d3Z(ee	d4<   dZ)ee	d5<   dZ*ee	d6<   ˆ fd7„Z+d8„ Z,d9„ Z-ˆ xZ.S ):Ú
ZayaConfiga€  
    lm_head_bias (`bool`, *optional*, defaults to `False`):
        Whether to add a bias to the language modeling head.
    router_hidden_size (`int`, *optional*, defaults to 256):
        Hidden size used by the ZAYA router.
    cca_time0 (`int`, *optional*, defaults to 2):
        First temporal parameter of the CCA projection.
    cca_time1 (`int`, *optional*, defaults to 2):
        Second temporal parameter of the CCA projection.

    ```python
    >>> from transformers import ZayaConfig, ZayaModel

    >>> configuration = ZayaConfig()
    >>> model = ZayaModel(configuration)

    >>> configuration = model.config
    ```
    ÚzayaÚpast_key_valuesÚ	ep_routerÚgrouped_gemmÚmoe_tp_experts)zlayers.*.mlp.gatez!layers.*.mlp.experts.gate_up_projzlayers.*.mlp.experts.down_projzlayers.*.mlp.expertsi€  Ú
vocab_sizei   Úhidden_sizeé(   Únum_hidden_layersé   Únum_attention_headsé   Únum_key_value_headsÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangegñhãˆµøä>Úrms_norm_epsTÚ	use_cacheÚtie_word_embeddingsNÚrope_parametersÚsliding_windowg        Úattention_dropoutÚmoe_intermediate_sizeé   Únum_experts_per_toké   Únum_expertsFÚoutput_router_logitsÚlayer_typesr   Úpad_token_idÚbos_token_idéj   Úeos_token_idé€   Úhead_dimÚattention_biasÚlm_head_biasé   Úrouter_hidden_sizeÚ	cca_time0Ú	cca_time1c                 óÜ   •— | j                   €dg| j                  z  nt        | j                   «      | _         ddddœddddœdœ}| j                  €|| _        t	        ‰| �  d
i |¤ddd	hi¤Ž y )NÚhybridÚdefaultg    ÐSAg      à?)Ú	rope_typeÚ
rope_thetaÚpartial_rotary_factorg     ˆÃ@)r8   Úhybrid_slidingÚignore_keys_at_rope_validationr=   © )r*   r   Úlistr!   ÚsuperÚ__post_init__)ÚselfÚkwargsÚdefault_rope_paramsÚ	__class__s      €úv/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/zaya/configuration_zaya.pyrB   zZayaConfig.__post_init___   sŒ   ø€ ØBF×BRÑBRÐBZ˜H˜:¨×(>Ñ(>Ò>Ô`dÐei×euÑeuÓ`vˆÔð 'Ø)Ø),ñð 'Ø&Ø),ññZ
Ðð ×ÑÐ'Ø#6ˆDÔ ä‰ÑÑd ÑdÈÐRbÐGcÔdó    c                 ó   — |S )Nr?   )rC   rD   s     rG   Úconvert_rope_params_to_dictz&ZayaConfig.convert_rope_params_to_dicts   s   € àˆrH   c                 óÒ   — | j                   dk7  rt        d«      ‚| j                  | j                  z  dk7  rt        d«      ‚d| j                  v r| j
                  €t        d«      ‚yy)z'Part of ``@strict``-powered validation.r%   z5ZAYA currently supports `num_experts_per_tok=1` only.r   zB`num_attention_heads` must be a multiple of `num_key_value_heads`.r=   NzJ`sliding_window` must be set when `layer_types` contains `hybrid_sliding`.)r&   Ú
ValueErrorr   r   r*   r"   )rC   s    rG   Úvalidate_architecturez ZayaConfig.validate_architecturew   so   € à×#Ñ# qÒ(ÜÐTÓUÐUØ×#Ñ# d×&>Ñ&>Ñ>À!ÒCÜÐaÓbÐbØ˜t×/Ñ/Ñ/°D×4GÑ4GÐ4OÜÐiÓjÐjð 5PÐ/rH   )/Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_ep_planr   ÚintÚ__annotations__r   r   r   r   r   Ústrr   r   Úfloatr   r   Úboolr    r!   r   Údictr"   r#   r$   r&   r(   r)   r*   r@   r+   r,   r.   r0   r1   r2   r4   r5   r6   rB   rJ   rM   Ú__classcell__)rF   s   @rG   r   r      s”  ø… ñð( €JØ#4Ð"5Ðà(Ø-;Ø*8Ø 0ñ	Ðð €J�ÓØ€K�ÓØÐ�sÓØ Ð˜Ó Ø Ð˜Ó Ø€J�ÓØ#)Ð˜SÓ)Ø#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ $Ð˜Ó$Ø48€O�^ dÑ*¨TÑ1Ó8Ø!%€N�C˜$‘JÓ%Ø%(Ð�u˜s‘{Ó(Ø!%Ð˜3Ó%à Ð˜Ó Ø€K�ÓØ!&Ð˜$Ó&Ø$(€K��c‘˜TÑ!Ó(Ø €L�#˜‘*Ó Ø €L�#˜‘*Ó Ø+.€L�#˜˜S™	‘/ DÑ(Ó.ð €HˆcÓØ €N�DÓ à€L�$ÓØ!Ð˜Ó!Ø€IˆsÓØ€IˆsÓôeò(ökrH   r   N)Útypingr   r   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr	   r   Ú__all__r?   rH   rG   ú<module>rb      sP   ð÷*  å .å 3Ý 1Ý #ñ Ð,Ô-Øô^kÐ!ó ^kó ó .ð^kðB ˆ.�rH   