Ë
    ÛÓ^j!  ã                   ó‚   — d dl mZmZ d dlmZ ddlmZ ddlmZ ddl	m
Z
  e
d¬«      e G d	„ d
e«      «       «       Zd
gZy)é    )ÚAnyÚLiteral)Ústricté   )ÚPreTrainedConfig)ÚRopeParameters)Úauto_docstringzpoolside/laguna-XS.2)Ú
checkpointc                   óú  ‡ — e Zd ZU dZdZdgZi dd“dd“dd“dd“d	d
“dd“dd“dd“dd“dd
“dd“dd
“dd“dd“dd“dd
“ZdgdgfddgdgfdgdgfdœZddddd œZd!Z	e
ed"<   d#Ze
ed$<   d%Ze
ed&<   d'Ze
ed(<   d)Ze
ed*<   d+Ze
ed,<   d-Zeed.<   d/Ze
ed0<   d1Zeed2<   d3Zeed4<   d5Zeed6<   d7Zeed8<   d9Zeez  d9z  ed:<   d;Ze
ed<<   d=Zee
z  ed><   d;Ze
ed?<   d;Z e
ed@<   d+Z!e
edA<   dBZ"e
edC<   d7Z#eedD<   dEZ$eedF<   d9Z%e&e   d9z  edG<   d9Z'e
d9z  edH<   d9Z(e
d9z  edI<   d9Z)e
e&e
   z  d9z  edJ<   dKZ*e
edL<   d7Z+eedM<   d5Z,eez  edN<   d9Z-e&e
   d9z  edO<   d9Z.e&e   d9z  edP<   dQZ/eedR<   d7Z0eedS<   d=Z1eedT<   ˆ fdU„Z2dV„ Z3dW„ Z4ˆ xZ5S )XÚLagunaConfigu¤  
    gating (`bool` or `str`, *optional*, defaults to `True`):
        Softplus output-gate granularity. ``True`` or ``"per-head"`` applies one gate per head,
        broadcast across ``head_dim``; ``"per-element"`` applies one gate per ``(head, head_dim)``
        channel.
    num_attention_heads_per_layer (`list[int]`, *optional*):
        Per-layer override for ``num_attention_heads``. Length must equal ``num_hidden_layers``.
    mlp_layer_types (`list[str]`, *optional*):
        Per-layer MLP type â€” ``"dense"`` or ``"sparse"``. Length must equal
        ``num_hidden_layers``. Defaults to first layer dense, rest sparse.
    moe_routed_scaling_factor (`float`, *optional*, defaults to 1.0):
        Scalar applied to routed-expert output before combining with the shared-expert output.
    moe_apply_router_weight_on_input (`bool`, *optional*, defaults to `False`):
        Whether to apply router weights to the MoE input rather than the output. Not supported
        in transformers yet; ``True`` will raise a ``NotImplementedError`` for now.
    moe_router_logit_softcapping (`float`, *optional*, defaults to 0.0):
        Scaling factor when applying tanh softcapping on the logits of the MoE router logits.

    Example:

    ```python
    >>> from transformers import LagunaModel, LagunaConfig

    >>> configuration = LagunaConfig()
    >>> model = LagunaModel(configuration)
    >>> configuration = model.config
    ```
    ÚlagunaÚpast_key_valueszlayers.*.self_attn.q_projÚcolwisezlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.g_projzlayers.*.self_attn.o_projÚrowwisezlayers.*.self_attn.q_normÚreplicated_with_grad_allreducezlayers.*.self_attn.k_normzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projú!layers.*.mlp.experts.gate_up_projÚpacked_colwiseúlayers.*.mlp.experts.down_projúlayers.*.mlp.expertsÚmoe_tp_expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormÚ	ep_routerÚgrouped_gemm)zlayers.*.mlp.gater   r   r   i ˆ Ú
vocab_sizei   Úhidden_sizei    Úintermediate_sizeé(   Únum_hidden_layersé0   Únum_attention_headsé   Únum_key_value_headsÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeg�íµ ÷Æ°>Úrms_norm_epsTÚ	use_cacheFÚtie_word_embeddingsNÚrope_parametersi   Úsliding_windowg        Úattention_dropoutÚmoe_intermediate_sizeÚshared_expert_intermediate_sizeÚnum_experts_per_toké   Únum_expertsÚoutput_router_logitsgü©ñÒMbP?Úrouter_aux_loss_coefÚlayer_typesÚpad_token_idÚbos_token_idÚeos_token_idé€   Úhead_dimÚattention_biasÚgatingÚnum_attention_heads_per_layerÚmlp_layer_typesç      ð?Úmoe_routed_scaling_factorÚ moe_apply_router_weight_on_inputÚmoe_router_logit_softcappingc                 óX  •— | j                   €dg| j                  z  | _         | j                  €dgdg| j                  dz
  z  z   | _        | j                  €| j                  g| j                  z  | _        ddddœdd	d
dœdœ}| j
                  €|| _        t        ‰| �  di |¤dddhi¤Ž y )NÚfull_attentionÚdenseÚsparseé   Údefaultg    €„Ag      à?)Ú	rope_typeÚ
rope_thetaÚpartial_rotary_factorg     ˆÃ@rD   )rI   Úsliding_attentionÚignore_keys_at_rope_validationrQ   © )r:   r$   rC   rB   r&   r0   ÚsuperÚ__post_init__)ÚselfÚkwargsÚdefault_rope_paramsÚ	__class__s      €úz/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/laguna/configuration_laguna.pyrU   zLagunaConfig.__post_init__‚   sÐ   ø€ Ø×ÑÐ#Ø 0Ð1°D×4JÑ4JÑJˆDÔØ×ÑÐ'Ø$+ 9°¨z¸T×=SÑ=SÐVWÑ=WÑ/XÑ#XˆDÔ Ø×-Ñ-Ð5Ø26×2JÑ2JÐ1KÈd×NdÑNdÑ1dˆDÔ.ð -6ÀXÐhkÑlØ/8ÈÐjmÑ!nñe
Ðð ×ÑÐ'Ø#6ˆDÔ ô 	‰ÑÑo ÑoÐH[Ð]mÐGnÔoó    c                 ó   — |S )NrS   )rV   rW   s     rZ   Úconvert_rope_params_to_dictz(LagunaConfig.convert_rope_params_to_dict”   s   € àˆr[   c                 ó0  — | j                   rt        d«      ‚| j                  �Qt        | j                  «      | j                  k7  r/t        dt        | j                  «      › d| j                  › d�«      ‚t        | j                  «      | j                  k7  r/t        dt        | j                  «      › d| j                  › d�«      ‚t        | j                  «      | j                  k7  r/t        dt        | j                  «      › d| j                  › d�«      ‚y)z'Part of ``@strict``-powered validation.zhmoe_apply_router_weight_on_input=True is not yet supported in the transformers implementation of Laguna.Nz&num_attention_heads_per_layer length (z ) must equal num_hidden_layers (z).zlayer_types length (zmlp_layer_types length ()rF   ÚNotImplementedErrorrB   Úlenr$   Ú
ValueErrorr:   rC   )rV   s    rZ   Úvalidate_architecturez"LagunaConfig.validate_architecture˜   s,  € à×0Ò0Ü%ð9óð ð
 ×.Ñ.Ð:Ü�D×6Ñ6Ó7¸4×;QÑ;QÒQäØ8¼¸T×=_Ñ=_Ó9`Ð8að b1Ø15×1GÑ1GÐ0HÈðLóð ô ˆt×ÑÓ  D×$:Ñ$:Ò:ÜØ&¤s¨4×+;Ñ+;Ó'<Ð&=ð >1Ø15×1GÑ1GÐ0HÈðLóð ô ˆt×#Ñ#Ó$¨×(>Ñ(>Ò>ÜØ*¬3¨t×/CÑ/CÓ+DÐ*Eð F1Ø15×1GÑ1GÐ0HÈðLóð ð ?r[   )6Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planÚbase_model_ep_planr    ÚintÚ__annotations__r!   r"   r$   r&   r(   r*   Ústrr+   r,   Úfloatr-   r.   Úboolr/   r0   r   Údictr1   r2   r3   r4   r5   r7   r8   r9   r:   Úlistr;   r<   r=   r?   r@   rA   rB   rC   rE   rF   rG   rU   r]   rb   Ú__classcell__)rY   s   @rZ   r   r      sÝ  ø… ñð: €JØ#4Ð"5ÐðØ# Yðà# Yðð 	$ Yðð 	$ Yð	ð
 	$ Yðð 	$Ð%Eðð 	$Ð%Eðð 	! )ðð 	 	ðð 	! )ðð 	,Ð-=ðð 	)¨)ðð 	Ð 0ðð 	0°ðð 	.¨yðð  	0°ð!Ðð& &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð )Ø-;Ø*8Ø 0ñ	Ðð €J�ÓØ€K�ÓØ!Ð�sÓ!ØÐ�sÓØ!Ð˜Ó!Ø Ð˜Ó Ø€J�ÓØ#)Ð˜SÓ)Ø#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ %Ð˜Ó%Ø48€O�^ dÑ*¨TÑ1Ó8Ø€N�CÓØ%(Ð�u˜s‘{Ó(Ø!$Ð˜3Ó$Ø+.Ð# SÓ.Ø Ð˜Ó Ø€K�ÓØ!&Ð˜$Ó&Ø"'Ð˜%Ó'Ø$(€K��c‘˜TÑ!Ó(Ø#€L�#˜‘*Ó#Ø#€L�#˜‘*Ó#Ø+/€L�#˜˜S™	‘/ DÑ(Ó/ð €HˆcÓØ €N�DÓ Ø€FˆD�3‰JÓØ6:Ð! 4¨¡9¨tÑ#3Ó:à(,€O�T˜#‘Y Ñ%Ó,Ø'*Ð˜uÓ*Ø-2Ð$ dÓ2Ø*-Ð  %Ó-ôpò$ör[   r   N)Útypingr   r   Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_rope_utilsr   Úutilsr	   r   Ú__all__rS   r[   rZ   ú<module>rz      sN   ð÷(  å .å 3Ý 1Ý #ñ Ð1Ô2ØôQÐ#ó Qó ó 3ðQðh Ð
�r[   