
    ^j!                         d dl mZmZ d dlmZ ddlmZ ddlmZ ddl	m
Z
  e
d      e G d	 d
e                    Zd
gZy)    )AnyLiteral)strict   )PreTrainedConfig)RopeParameters)auto_docstringzpoolside/laguna-XS.2)
checkpointc                       e Zd ZU dZdZdgZi ddddddddd	d
dddddddddd
dddd
dddddddd
ZdgdgfddgdgfdgdgfdZddddd Zd!Z	e
ed"<   d#Ze
ed$<   d%Ze
ed&<   d'Ze
ed(<   d)Ze
ed*<   d+Ze
ed,<   d-Zeed.<   d/Ze
ed0<   d1Zeed2<   d3Zeed4<   d5Zeed6<   d7Zeed8<   d9Zeez  d9z  ed:<   d;Ze
ed<<   d=Zee
z  ed><   d;Ze
ed?<   d;Z e
ed@<   d+Z!e
edA<   dBZ"e
edC<   d7Z#eedD<   dEZ$eedF<   d9Z%e&e   d9z  edG<   d9Z'e
d9z  edH<   d9Z(e
d9z  edI<   d9Z)e
e&e
   z  d9z  edJ<   dKZ*e
edL<   d7Z+eedM<   d5Z,eez  edN<   d9Z-e&e
   d9z  edO<   d9Z.e&e   d9z  edP<   dQZ/eedR<   d7Z0eedS<   d=Z1eedT<    fdUZ2dV Z3dW Z4 xZ5S )XLagunaConfigu  
    gating (`bool` or `str`, *optional*, defaults to `True`):
        Softplus output-gate granularity. ``True`` or ``"per-head"`` applies one gate per head,
        broadcast across ``head_dim``; ``"per-element"`` applies one gate per ``(head, head_dim)``
        channel.
    num_attention_heads_per_layer (`list[int]`, *optional*):
        Per-layer override for ``num_attention_heads``. Length must equal ``num_hidden_layers``.
    mlp_layer_types (`list[str]`, *optional*):
        Per-layer MLP type — ``"dense"`` or ``"sparse"``. Length must equal
        ``num_hidden_layers``. Defaults to first layer dense, rest sparse.
    moe_routed_scaling_factor (`float`, *optional*, defaults to 1.0):
        Scalar applied to routed-expert output before combining with the shared-expert output.
    moe_apply_router_weight_on_input (`bool`, *optional*, defaults to `False`):
        Whether to apply router weights to the MoE input rather than the output. Not supported
        in transformers yet; ``True`` will raise a ``NotImplementedError`` for now.
    moe_router_logit_softcapping (`float`, *optional*, defaults to 0.0):
        Scaling factor when applying tanh softcapping on the logits of the MoE router logits.

    Example:

    ```python
    >>> from transformers import LagunaModel, LagunaConfig

    >>> configuration = LagunaConfig()
    >>> model = LagunaModel(configuration)
    >>> configuration = model.config
    ```
    lagunapast_key_valueszlayers.*.self_attn.q_projcolwisezlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.g_projzlayers.*.self_attn.o_projrowwisezlayers.*.self_attn.q_normreplicated_with_grad_allreducezlayers.*.self_attn.k_normzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proj!layers.*.mlp.experts.gate_up_projpacked_colwiselayers.*.mlp.experts.down_projlayers.*.mlp.expertsmoe_tp_expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_proj	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnorm	ep_routergrouped_gemm)zlayers.*.mlp.gater   r   r   i  
vocab_sizei   hidden_sizei    intermediate_size(   num_hidden_layers0   num_attention_heads   num_key_value_headssilu
hidden_acti   max_position_embeddingsg{Gz?initializer_rangegư>rms_norm_epsT	use_cacheFtie_word_embeddingsNrope_parametersi   sliding_windowg        attention_dropoutmoe_intermediate_sizeshared_expert_intermediate_sizenum_experts_per_tok   num_expertsoutput_router_logitsgMbP?router_aux_loss_coeflayer_typespad_token_idbos_token_ideos_token_id   head_dimattention_biasgatingnum_attention_heads_per_layermlp_layer_types      ?moe_routed_scaling_factor moe_apply_router_weight_on_inputmoe_router_logit_softcappingc                 X   | j                   dg| j                  z  | _         | j                  dgdg| j                  dz
  z  z   | _        | j                  | j                  g| j                  z  | _        dddddd	d
dd}| j
                  || _        t        |   di |dddhi y )Nfull_attentiondensesparse   defaultg    Ag      ?)	rope_type
rope_thetapartial_rotary_factorg     @rD   )rI   sliding_attentionignore_keys_at_rope_validationrQ    )r:   r$   rC   rB   r&   r0   super__post_init__)selfkwargsdefault_rope_params	__class__s      z/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/laguna/configuration_laguna.pyrU   zLagunaConfig.__post_init__   s    # 01D4J4JJD'$+9zT=S=SVW=W/X#XD --5262J2J1KdNdNd1dD. -6Xhkl/8jm!ne
 '#6D  	ooH[]mGno    c                     |S )NrS   )rV   rW   s     rZ   convert_rope_params_to_dictz(LagunaConfig.convert_rope_params_to_dict   s    r[   c                 0   | j                   rt        d      | j                  Qt        | j                        | j                  k7  r/t        dt        | j                         d| j                   d      t        | j                        | j                  k7  r/t        dt        | j                         d| j                   d      t        | j                        | j                  k7  r/t        dt        | j                         d| j                   d      y)z'Part of ``@strict``-powered validation.zhmoe_apply_router_weight_on_input=True is not yet supported in the transformers implementation of Laguna.Nz&num_attention_heads_per_layer length (z ) must equal num_hidden_layers (z).zlayer_types length (zmlp_layer_types length ()rF   NotImplementedErrorrB   lenr$   
ValueErrorr:   rC   )rV   s    rZ   validate_architecturez"LagunaConfig.validate_architecture   s,   00%9 
 ..:D6674;Q;QQ8T=_=_9`8a b1151G1G0HL  t D$:$::&s4+;+;'<&= >1151G1G0HL  t##$(>(>>*3t/C/C+D*E F1151G1G0HL  ?r[   )6__name__
__module____qualname____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planbase_model_ep_planr    int__annotations__r!   r"   r$   r&   r(   r*   strr+   r,   floatr-   r.   boolr/   r0   r   dictr1   r2   r3   r4   r5   r7   r8   r9   r:   listr;   r<   r=   r?   r@   rA   rB   rC   rE   rF   rG   rU   r]   rb   __classcell__)rY   s   @rZ   r   r      s   : J#4"5#Y#Y 	$Y 	$Y	
 	$Y 	$%E 	$%E 	!) 		 	!) 	,-= 	)) 	 0 	0 	.y  	0!& &(9:#%568IJ!"_$56 )-;*8 0	 JK!s!s!!  J#)S)#u#L%It %%48O^d*T18NC%(us{(!$3$+.#S.  K!&$&"'%'$(KcT!(#L#*##L#*#+/L#S	/D(/ Hc ND FD3J6:!49t#3:(,OT#Y%,'*u*-2$d2*- %-p$r[   r   N)typingr   r   huggingface_hub.dataclassesr   configuration_utilsr   modeling_rope_utilsr   utilsr	   r   __all__rS   r[   rZ   <module>rz      sN   (   . 3 1 # 12Q# Q  3Qh 
r[   