
    ^j                         d dl mZmZ d dlmZ ddlmZ ddlmZ ddl	m
Z
  e
d      e G d	 d
e                    Zd
gZy)    )AnyLiteral)strict   )PreTrainedConfig)RopeParameters)auto_docstringzZyphra/ZAYA1-8B)
checkpointc                       e Zd ZU dZdZdgZdddddZdZee	d	<   d
Z
ee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZeez  dz  e	d<   dZedz  e	d<   d Zeez  e	d!<   d
Zee	d"<   d#Zee	d$<   d%Zee	d&<   d'Zee	d(<   dZ e!e   dz  e	d)<   d*Z"edz  e	d+<   dZ#edz  e	d,<   d-Z$ee!e   z  dz  e	d.<   d/Z%ee	d0<   d'Z&ee	d1<   d'Z'ee	d2<   d3Z(ee	d4<   dZ)ee	d5<   dZ*ee	d6<    fd7Z+d8 Z,d9 Z- xZ.S ):
ZayaConfiga  
    lm_head_bias (`bool`, *optional*, defaults to `False`):
        Whether to add a bias to the language modeling head.
    router_hidden_size (`int`, *optional*, defaults to 256):
        Hidden size used by the ZAYA router.
    cca_time0 (`int`, *optional*, defaults to 2):
        First temporal parameter of the CCA projection.
    cca_time1 (`int`, *optional*, defaults to 2):
        Second temporal parameter of the CCA projection.

    ```python
    >>> from transformers import ZayaConfig, ZayaModel

    >>> configuration = ZayaConfig()
    >>> model = ZayaModel(configuration)

    >>> configuration = model.config
    ```
    zayapast_key_values	ep_routergrouped_gemmmoe_tp_experts)zlayers.*.mlp.gatez!layers.*.mlp.experts.gate_up_projzlayers.*.mlp.experts.down_projzlayers.*.mlp.expertsi  
vocab_sizei   hidden_size(   num_hidden_layers   num_attention_heads   num_key_value_headssilu
hidden_acti   max_position_embeddingsg{Gz?initializer_rangegh㈵>rms_norm_epsT	use_cachetie_word_embeddingsNrope_parameterssliding_windowg        attention_dropoutmoe_intermediate_size   num_experts_per_tok   num_expertsFoutput_router_logitslayer_typesr   pad_token_idbos_token_idj   eos_token_id   head_dimattention_biaslm_head_bias   router_hidden_size	cca_time0	cca_time1c                     | j                   dg| j                  z  nt        | j                         | _         ddddddddd}| j                  || _        t	        |   d
i |ddd	hi y )Nhybriddefaultg    SAg      ?)	rope_type
rope_thetapartial_rotary_factorg     @)r8   hybrid_slidingignore_keys_at_rope_validationr=    )r*   r   listr!   super__post_init__)selfkwargsdefault_rope_params	__class__s      v/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/zaya/configuration_zaya.pyrB   zZayaConfig.__post_init___   s    BFBRBRBZH:(>(>>`deieueu`v ')), '&),Z
 '#6D ddRbGcd    c                     |S )Nr?   )rC   rD   s     rG   convert_rope_params_to_dictz&ZayaConfig.convert_rope_params_to_dicts   s    rH   c                     | j                   dk7  rt        d      | j                  | j                  z  dk7  rt        d      d| j                  v r| j
                  t        d      yy)z'Part of ``@strict``-powered validation.r%   z5ZAYA currently supports `num_experts_per_tok=1` only.r   zB`num_attention_heads` must be a multiple of `num_key_value_heads`.r=   NzJ`sliding_window` must be set when `layer_types` contains `hybrid_sliding`.)r&   
ValueErrorr   r   r*   r"   )rC   s    rG   validate_architecturez ZayaConfig.validate_architecturew   so    ##q(TUU##d&>&>>!Cabbt///D4G4G4Oijj 5P/rH   )/__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencebase_model_ep_planr   int__annotations__r   r   r   r   r   strr   r   floatr   r   boolr    r!   r   dictr"   r#   r$   r&   r(   r)   r*   r@   r+   r,   r.   r0   r1   r2   r4   r5   r6   rB   rJ   rM   __classcell__)rF   s   @rG   r   r      s   ( J#4"5(-;*8 0	 JKs    J#)S)#u#L%It $$48O^d*T18!%NC$J%%(us{(!%3%  K!&$&$(KcT!( L#*  L#* +.L#S	/D(. Hc ND L$!!IsIse(krH   r   N)typingr   r   huggingface_hub.dataclassesr   configuration_utilsr   modeling_rope_utilsr   utilsr	   r   __all__r?   rH   rG   <module>rb      sP   *   . 3 1 # ,-^k! ^k  .^kB .rH   