Ë
    ÛÓ^j)Q  ã                   óÆ  — d Z ddlmZ ddlmZmZmZ ddlZddlm	c m
Z ddlmZ ddlm	Z	 ddlmZ dd	lmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddl!m"Z" ddl#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6m7Z7m8Z8 ddl9m:Z:m;Z; ddl<m=Z=m>Z>  e%j~                  e@«      ZA e$d¬«      e G d„ de3«      «       «       ZB G d „ d!e8«      ZC G d"„ d#e,«      ZD G d$„ d%e6«      ZE G d&„ d'e:«      ZF G d(„ d)e=«      ZG G d*„ d+e>«      ZHe( G d,„ d-e*«      «       ZI G d.„ d/e.«      ZJ G d0„ d1e7«      ZK G d2„ d3e0«      ZL G d4„ d5e5«      ZMg d6¢ZNy)7zPyTorch Laguna model.é    )ÚCallable)ÚAnyÚLiteralÚOptionalN)Ústrict)Únné   )Úinitialization)ÚCacheÚDynamicCache)ÚPreTrainedConfig)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONS)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)Úauto_docstringÚlogging)ÚTransformersKwargsÚno_inherit_decoratoré   )ÚAfmoeAttention)ÚGemma3RotaryEmbedding)ÚGlm4MoeLiteDecoderLayer)Ú
LlamaModelÚeager_attention_forward)ÚQwen2MoeConfig)ÚQwen2MoeForCausalLMÚQwen2MoeMLPÚQwen2MoePreTrainedModelÚQwen2MoeRMSNorm)ÚQwen3_5MoeTopKRouterÚapply_rotary_pos_emb)ÚQwen3MoeExpertsÚQwen3MoeSparseMoeBlockzpoolside/laguna-XS.2)Ú
checkpointc                   ó  — e Zd ZU dZdZi dd“dd“dd“dd“dd	“d
d“dd“dd“dd“dd	“dd“dd	“dd“dd“dd“dd	“ZdZeed<   dZ	eed<   dZ
eed<   dZeed<   d Zeed!<   d"Zeed#<   d$Zeed%<   d Zeed&<   d'Zeed(<   d'Zeed)<   d'Zeed*<   d+Zeed,<   d-Zeed.<   d/Zeez  ed0<   d1Zee   d1z  ed2<   d1Zee   d1z  ed3<   d4Zeed5<   d-Zeed6<   d7Zeed8<    e«       Z  e«       Z! e«       Z" e«       Z# e«       Z$ e«       Z%d9„ Z&d:„ Z'd;„ Z(y1)<ÚLagunaConfigu¤  
    gating (`bool` or `str`, *optional*, defaults to `True`):
        Softplus output-gate granularity. ``True`` or ``"per-head"`` applies one gate per head,
        broadcast across ``head_dim``; ``"per-element"`` applies one gate per ``(head, head_dim)``
        channel.
    num_attention_heads_per_layer (`list[int]`, *optional*):
        Per-layer override for ``num_attention_heads``. Length must equal ``num_hidden_layers``.
    mlp_layer_types (`list[str]`, *optional*):
        Per-layer MLP type â€” ``"dense"`` or ``"sparse"``. Length must equal
        ``num_hidden_layers``. Defaults to first layer dense, rest sparse.
    moe_routed_scaling_factor (`float`, *optional*, defaults to 1.0):
        Scalar applied to routed-expert output before combining with the shared-expert output.
    moe_apply_router_weight_on_input (`bool`, *optional*, defaults to `False`):
        Whether to apply router weights to the MoE input rather than the output. Not supported
        in transformers yet; ``True`` will raise a ``NotImplementedError`` for now.
    moe_router_logit_softcapping (`float`, *optional*, defaults to 0.0):
        Scaling factor when applying tanh softcapping on the logits of the MoE router logits.

    Example:

    ```python
    >>> from transformers import LagunaModel, LagunaConfig

    >>> configuration = LagunaConfig()
    >>> model = LagunaModel(configuration)
    >>> configuration = model.config
    ```
    Úlagunazlayers.*.self_attn.q_projÚcolwisezlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.g_projzlayers.*.self_attn.o_projÚrowwisezlayers.*.self_attn.q_normÚreplicated_with_grad_allreducezlayers.*.self_attn.k_normzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projz!layers.*.mlp.experts.gate_up_projÚpacked_colwisezlayers.*.mlp.experts.down_projzlayers.*.mlp.expertsÚmoe_tp_expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_proji ˆ Ú
vocab_sizei    Úintermediate_sizeé(   Únum_hidden_layersé0   Únum_attention_headsé   Únum_key_value_headsi   Úmax_position_embeddingsé   Únum_expertsÚnum_experts_per_toki   Úmoe_intermediate_sizeÚshared_expert_intermediate_sizeÚsliding_windowé€   Úhead_dimFÚattention_biasTÚgatingNÚnum_attention_heads_per_layerÚmlp_layer_typesç      ð?Úmoe_routed_scaling_factorÚ moe_apply_router_weight_on_inputç        Úmoe_router_logit_softcappingc                 ód  — | j                   €dg| j                  z  | _         | j                  €dgdg| j                  dz
  z  z   | _        | j                  €| j                  g| j                  z  | _        ddddœdd	d
dœdœ}| j
                  €|| _        t        j                  | fi |¤dddhi¤Ž y )NÚfull_attentionÚdenseÚsparseé   Údefaultg    €„Ag      à?)Ú	rope_typeÚ
rope_thetaÚpartial_rotary_factorg     ˆÃ@rF   ©rL   Úsliding_attentionÚignore_keys_at_rope_validationrU   )Úlayer_typesr4   rE   rD   r6   Úrope_parametersr   Ú__post_init__)ÚselfÚkwargsÚdefault_rope_paramss      út/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/laguna/modular_laguna.pyrY   zLagunaConfig.__post_init__„   sÚ   € Ø×ÑÐ#Ø 0Ð1°D×4JÑ4JÑJˆDÔØ×ÑÐ'Ø$+ 9°¨z¸T×=SÑ=SÐVWÑ=WÑ/XÑ#XˆDÔ Ø×-Ñ-Ð5Ø26×2JÑ2JÐ1KÈd×NdÑNdÑ1dˆDÔ.ð -6ÀXÐhkÑlØ/8ÈÐjmÑ!nñe
Ðð ×ÑÐ'Ø#6ˆDÔ ô 	×&Ñ&Øñ	
Øñ	
Ø<OÐQaÐ;bô	
ó    c                 ó   — |S ©N© )rZ   r[   s     r]   Úconvert_rope_params_to_dictz(LagunaConfig.convert_rope_params_to_dict˜   s   € àˆr^   c                 ó0  — | j                   rt        d«      ‚| j                  �Qt        | j                  «      | j                  k7  r/t        dt        | j                  «      › d| j                  › d�«      ‚t        | j                  «      | j                  k7  r/t        dt        | j                  «      › d| j                  › d�«      ‚t        | j                  «      | j                  k7  r/t        dt        | j                  «      › d| j                  › d�«      ‚y)z'Part of ``@strict``-powered validation.zhmoe_apply_router_weight_on_input=True is not yet supported in the transformers implementation of Laguna.Nz&num_attention_heads_per_layer length (z ) must equal num_hidden_layers (z).zlayer_types length (zmlp_layer_types length ()rH   ÚNotImplementedErrorrD   Úlenr4   Ú
ValueErrorrW   rE   )rZ   s    r]   Úvalidate_architecturez"LagunaConfig.validate_architectureœ   s,  € à×0Ò0Ü%ð9óð ð
 ×.Ñ.Ð:Ü�D×6Ñ6Ó7¸4×;QÑ;QÒQäØ8¼¸T×=_Ñ=_Ó9`Ð8að b1Ø15×1GÑ1GÐ0HÈðLóð ô ˆt×ÑÓ  D×$:Ñ$:Ò:ÜØ&¤s¨4×+;Ñ+;Ó'<Ð&=ð >1Ø15×1GÑ1GÐ0HÈðLóð ô ˆt×#Ñ#Ó$¨×(>Ñ(>Ò>ÜØ*¬3¨t×/CÑ/CÓ+DÐ*Eð F1Ø15×1GÑ1GÐ0HÈðLóð ð ?r^   ))Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚbase_model_tp_planr1   ÚintÚ__annotations__r2   r4   r6   r8   r9   r;   r<   r=   r>   r?   rA   rB   ÚboolrC   ÚstrrD   ÚlistrE   rG   ÚfloatrH   rJ   ÚAttributeErrorÚdecoder_sparse_stepÚmlp_only_layersÚqkv_biasÚnorm_topk_probÚuse_sliding_windowÚmax_window_layersrY   rb   rg   ra   r^   r]   r*   r*   0   sþ  … ñð: €JðØ# Yðà# Yðð 	$ Yðð 	$ Yð	ð
 	$ Yðð 	$Ð%Eðð 	$Ð%Eðð 	! )ðð 	 	ðð 	! )ðð 	,Ð-=ðð 	)¨)ðð 	Ð 0ðð 	0°ðð 	.¨yðð  	0°ð!Ðð& €J�ÓØ!Ð�sÓ!ØÐ�sÓØ!Ð˜Ó!Ø Ð˜Ó Ø#)Ð˜SÓ)Ø€K�ÓØ Ð˜Ó Ø!$Ð˜3Ó$Ø+.Ð# SÓ.Ø€N�CÓð €HˆcÓØ €N�DÓ Ø€FˆD�3‰JÓØ6:Ð! 4¨¡9¨tÑ#3Ó:à(,€O�T˜#‘Y Ñ%Ó,Ø'*Ð˜uÓ*Ø-2Ð$ dÓ2Ø*-Ð  %Ó-ñ )Ó*ÐÙ$Ó&€OÙÓ€HÙ#Ó%€NÙ'Ó)ÐÙ&Ó(Ðò
ò(ór^   r*   c                   ó   — e Zd Zy)ÚLagunaRMSNormN©rh   ri   rj   ra   r^   r]   r|   r|   ·   ó   „ Ør^   r|   c                   óv   ‡ — e Zd Zdefˆ fd„Ze	 	 	 	 ddedz  ded   dedz  dedz  de	d	e
f   f
d
„«       Zˆ xZS )ÚLagunaRotaryEmbeddingÚconfigc                 ó$   •— t         ‰| �  |«       y r`   )ÚsuperÚ__init__©rZ   r�   Ú	__class__s     €r]   r„   zLagunaRotaryEmbedding.__init__¼   s   ø€ Ü‰Ñ˜Õ r^   NÚdeviceztorch.deviceÚseq_lenÚ
layer_typeÚreturnztorch.Tensorc                 óz  — | j                   |   d   }| j                   |   j                  dd«      }t        | dd«      xs | j                  | j                  z  }t        ||z  «      }d}d|t        j                  d|dt        j                  ¬«      j                  |t        j                  ¬	«      |z  z  z  }	|	|fS )
a{  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        rR   rS   rF   rA   Nr   r   )Údtype)r‡   rŒ   )rX   ÚgetÚgetattrÚhidden_sizer6   rn   ÚtorchÚarangeÚint64Útors   )
r�   r‡   rˆ   r‰   ÚbaserS   rA   ÚdimÚattention_factorÚinv_freqs
             r]   Úcompute_default_rope_parametersz5LagunaRotaryEmbedding.compute_default_rope_parameters¿   sÅ   € ð. ×%Ñ% jÑ1°,Ñ?ˆà &× 6Ñ 6°zÑ B× FÑ FÐG^Ð`cÓ dÐÜ˜6 :¨tÓ4Òh¸×8JÑ8JÈf×NhÑNhÑ8hˆÜ�(Ð2Ñ2Ó3ˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r^   )NNNN)rh   ri   rj   r*   r„   Ústaticmethodr   rn   rq   Útuplers   r˜   Ú__classcell__©r†   s   @r]   r€   r€   »   sy   ø„ ð!˜|õ !ð à&*Ø+/Ø"Ø!%ñ	"*Ø˜tÑ#ð"*à˜Ñ(ð"*ð �t‘ð"*ð ˜$‘Jð	"*ð
 
ˆ~˜uÐ$Ñ	%ò"*ó ô"*r^   r€   c                   ó   — e Zd Zy)Ú	LagunaMLPNr}   ra   r^   r]   rž   rž   å   r~   r^   rž   c                   óŠ   ‡ — e Zd Zˆ fd„Zdej
                  deej
                  ej
                  ej
                  f   fd„Zˆ xZS )ÚLagunaTopKRouterc                 ó¶   •— t         ‰| �  «        t        j                  t	        j
                  |j                  «      d¬«      | _        |j                  | _	        y )NF)Úrequires_grad)
rƒ   r„   r   Ú	Parameterr�   Úzerosr;   Úe_score_correction_biasrJ   Úrouter_logit_softcappingr…   s     €r]   r„   zLagunaTopKRouter.__init__ê   s?   ø€ Ü‰ÑÔÜ')§|¡|´E·K±KÀ×@RÑ@RÓ4SÐchÔ'iˆÔ$Ø(.×(KÑ(KˆÕ%r^   Úhidden_statesrŠ   c                 óf  — |j                  d| j                  «      }t        j                  || j                  «      j                  «       }| j                  dkD  r/t        j                  || j                  z  «      | j                  z  }t        j                  |«      }|| j                  j                  |j                  «      z   }t        j                  || j                  d¬«      \  }}|j                  d|«      }||j!                  dd¬«      z  }|j                  |j                  «      }|||fS )NéÿÿÿÿrI   )r•   T)r•   Úkeepdim)ÚreshapeÚ
hidden_dimÚFÚlinearÚweightrs   r¦   r�   ÚtanhÚsigmoidr¥   r“   rŒ   ÚtopkÚtop_kÚgatherÚsum)rZ   r§   Úrouter_logitsÚrouting_scoresÚscores_for_selectionÚ_Úselected_expertsÚrouting_weightss           r]   ÚforwardzLagunaTopKRouter.forwardï   s
  € ð &×-Ñ-¨b°$·/±/ÓBˆÜŸ™ °·±Ó<×BÑBÓDˆà×(Ñ(¨3Ò.Ü!ŸJ™J }°t×7TÑ7TÑ'TÓUÐX\×XuÑXuÑuˆMäŸ™ }Ó5ˆà-°×0LÑ0L×0OÑ0OÐP^×PdÑPdÓ0eÑeÐÜ#Ÿj™jÐ)=¸t¿z¹zÈrÔRÑˆÐØ(×/Ñ/°Ð4DÓEˆØ)¨O×,?Ñ,?ÀBÐPTÐ,?Ó,UÑUˆØ)×,Ñ,¨]×-@Ñ-@ÓAˆà˜oÐ/?Ð?Ð?r^   )	rh   ri   rj   r„   r�   ÚTensorrš   r¼   r›   rœ   s   @r]   r    r    é   s?   ø„ ôLð
@à—|‘|ð@ð 
ˆu�|‰|˜UŸ\™\¨5¯<©<Ð7Ñ	8÷@r^   r    c                   ó   — e Zd Zy)ÚLagunaExpertsNr}   ra   r^   r]   r¿   r¿     r~   r^   r¿   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚLagunaSparseMoeBlockr�   c                 ó~   •— t         ‰| �  |«       t        ||j                  ¬«      | _        |j
                  | _        y )N©r2   )rƒ   r„   rž   r>   Úshared_expertsrG   Úrouted_scaling_factorr…   s     €r]   r„   zLagunaSparseMoeBlock.__init__	  s3   ø€ Ü‰Ñ˜Ô Ü'¨À&×BhÑBhÔiˆÔØ%+×%EÑ%EˆÕ"r^   r§   rŠ   c                 ó
  — |j                   \  }}}|j                  d|«      }| j                  |«      }| j                  |«      \  }}}| j	                  |||«      }|| j
                  z  }||z   }|j                  |||«      }|S )Nr©   )ÚshapeÚviewrÄ   ÚgateÚexpertsrÅ   r«   )	rZ   r§   Ú
batch_sizeÚsequence_lengthr¬   Úshared_outputr¹   r»   rº   s	            r]   r¼   zLagunaSparseMoeBlock.forward  s“   € Ø2?×2EÑ2EÑ/ˆ
�O ZØ%×*Ñ*¨2¨zÓ:ˆØ×+Ñ+¨MÓ:ˆà/3¯y©y¸Ó/GÑ,ˆˆ?Ð,ØŸ™ ]Ð4DÀoÓVˆà%¨×(BÑ(BÑBˆØ%¨Ñ5ˆà%×-Ñ-¨j¸/È:ÓVˆØÐr^   )	rh   ri   rj   r*   r„   r�   r½   r¼   r›   rœ   s   @r]   rÁ   rÁ     s,   ø„ ðF˜|õ Fð
 U§\¡\ð °e·l±l÷ r^   rÁ   c                   óø   ‡ — e Zd ZdZdededefˆ fd„Z	 ddej                  de	ej                  ej                  f   d	ej                  dz  d
e
dz  dee   de	ej                  ej                  dz  f   fd„Zˆ xZS )ÚLagunaAttentionzSAfmoe-style SWA/GQA attention with Laguna-specific gating and per-layer head count.r�   Ú	layer_idxÚ	num_headsc                 óŒ  •— || _         t        ‰| �	  ||«       | j                   |j                  z  | _        t        j                  |j                  | j                   | j                  z  |j                  ¬«      | _
        t        j                  | j                   | j                  z  |j                  |j                  ¬«      | _        | `|j                  du xs |j                  dk(  | _        | j                  r| j                   n| j                   | j                  z  }t        j                  |j                  |d¬«      | _        y )N)ÚbiasTzper-headF)rÑ   rƒ   r„   r8   Únum_key_value_groupsr   ÚLinearr�   rA   rB   Úq_projÚo_projÚ	gate_projrC   Úgate_per_headÚg_proj)rZ   r�   rÐ   rÑ   Ú
g_proj_dimr†   s        €r]   r„   zLagunaAttention.__init__!  sí   ø€ à"ˆŒä‰Ñ˜ Ô+Ø$(§N¡N°f×6PÑ6PÑ$PˆÔ!ä—i‘i × 2Ñ 2°D·N±NÀTÇ]Á]Ñ4RÐY_×YnÑYnÔoˆŒÜ—i‘i §¡°·±Ñ >À×@RÑ@RÐY_×YnÑYnÔoˆŒàˆNØ#Ÿ]™]¨dÐ2ÒQ°f·m±mÀzÑ6QˆÔØ'+×'9Ò'9�T—^’^¸t¿~¹~ÐPT×P]ÑP]Ñ?]ˆ
Ü—i‘i × 2Ñ 2°JÀUÔKˆ�r^   Nr§   Úposition_embeddingsÚattention_maskÚpast_key_valuesr[   rŠ   c                 óª  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      }| j	                  |«      j                  |«      }	| j                  |«      j                  |«      }
| j                  |«      j                  dd«      }| j                  |	«      j                  dd«      }	|
j                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t         «      } || ||	|
|f| j"                  sdn| j$                  | j&                  | j(                  dœ|¤Ž\  }} |j*                  g |¢d‘­Ž j-                  «       }t/        j0                  | j3                  |«      j5                  «       «      j7                  |j8                  «      }| j:                  rE  |j                  g |¢d‘| j                  ‘­Ž |j=                  d«      z  j                  g |¢d‘­Ž }n||z  }| j?                  |«      }||fS )Nr©   rO   r   rI   )ÚdropoutÚscalingr?   ) rÇ   rA   rÖ   rÈ   Úk_projÚv_projÚq_normÚ	transposeÚk_normr%   ÚupdaterÐ   r   Úget_interfacer�   Ú_attn_implementationr   ÚtrainingÚattention_dropoutrá   r?   r«   Ú
contiguousr­   ÚsoftplusrÚ   rs   r“   rŒ   rÙ   Ú	unsqueezer×   )rZ   r§   rÜ   rÝ   rÞ   r[   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightsrÉ   s                    r]   r¼   zLagunaAttention.forward0  sN  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓDˆØ—[‘[ Ó/×4Ñ4°\ÓBˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓDˆà—{‘{ <Ó0×:Ñ:¸1¸aÓ@ˆØ—[‘[ Ó,×6Ñ6°q¸!Ó<ˆ
Ø#×-Ñ-¨a°Ó3ˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆä�z‰z˜$Ÿ+™+ mÓ4×:Ñ:Ó<Ó=×@Ñ@À×ARÑARÓSˆØ×ÒØgÐ+˜;×+Ñ+ÐL¨[ÐL¸"ÐL¸d¿m¹mÒLÈtÏ~É~Ð^`ÓOaÑa×gÑgð ØðØ ò‰Kð &¨Ñ,ˆKà—k‘k +Ó.ˆØ˜LÐ(Ð(r^   r`   )rh   ri   rj   rk   r*   rn   r„   r�   r½   rš   r   r   r   r¼   r›   rœ   s   @r]   rÏ   rÏ     s¨   ø„ á]ðL˜|ð L¸ð LÈõ Lð( )-ñ3)à—|‘|ð3)ð # 5§<¡<°·±Ð#=Ñ>ð3)ð Ÿ™ tÑ+ð	3)ð
  ™ð3)ð Ð-Ñ.ð3)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷3)r^   rÏ   c                   ó   — e Zd Zdedefd„Zy)ÚLagunaDecoderLayerr�   rÐ   c                 ó¸  — t         j                  j                  | «       |j                  | _        t	        |||j
                  |   «      | _        |j                  |   dk(  rt        |«      | _	        nt        ||j                  ¬«      | _	        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        y )NrN   rÃ   )Úeps)r   ÚModuler„   r�   rÏ   rD   Ú	self_attnrE   rÁ   Úmlprž   r2   r|   Úrms_norm_epsÚinput_layernormÚpost_attention_layernorm)rZ   r�   rÐ   s      r]   r„   zLagunaDecoderLayer.__init__g  s§   € Ü
�	‰	×Ñ˜4Ô Ø!×-Ñ-ˆÔÜ(¨°¸F×<`Ñ<`ÐajÑ<kÓlˆŒØ×!Ñ! )Ñ,°Ò8Ü+¨FÓ3ˆD�Hä  ¸6×;SÑ;SÔTˆDŒHÜ,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔÜ(5°f×6HÑ6HÈf×NaÑNaÔ(bˆÕ%r^   N)rh   ri   rj   r*   rn   r„   ra   r^   r]   rú   rú   f  s   „ ð	c˜|ð 	c¸ô 	cr^   rú   c                   óF   ‡ — e Zd Z ej                  «       ˆ fd„«       Zˆ xZS )ÚLagunaPreTrainedModelc                 ó  •— t         ‰| �  |«       t        |t        «      r4t        j
                  j                  j                  |j                  «       y t        |t        «      r£|j                  D ]“  }|j                  }|j                  |   dk7  rt        |j                  |      } ||j                  |¬«      \  }}t        j                  t!        ||› d�«      |«       t        j                  t!        ||› d�«      |«       Œ• y y )NrP   )r‰   Ú	_inv_freqÚ_original_inv_freq)rƒ   Ú_init_weightsÚ
isinstancer    r�   r   ÚinitÚzeros_r¥   r€   rW   r˜   rQ   r   r�   Úcopy_rŽ   )rZ   Úmoduler‰   Úrope_init_fnÚcurr_inv_freqr¹   r†   s         €r]   r  z#LagunaPreTrainedModel._init_weightst  sß   ø€ ä‰Ñ˜fÔ%Ü�fÔ.Ô/Ü�H‰H�M‰M× Ñ  ×!?Ñ!?Õ@Ü˜Ô 5Ô6Ø$×0Ñ0ò ^�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ò<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ô#UÑ �˜qÜ—
‘
œ7 6¨j¨\¸Ð+CÓDÀmÔTÜ—
‘
œ7 6¨j¨\Ð9KÐ+LÓMÈ}Õ]ñ^ð 7r^   )rh   ri   rj   r�   Úno_gradr  r›   rœ   s   @r]   r  r  s  s   ø„ Ø€U‡]�]ƒ_ó^ó ô^r^   r  c                   ó¼   — e Zd Z	 	 	 	 	 	 ddej                  dz  dej
                  dz  dej                  dz  dedz  dej                  dz  dedz  de	e
   d	efd
„Zy)ÚLagunaModelNÚ	input_idsrÝ   Úposition_idsrÞ   Úinputs_embedsÚ	use_cacher[   rŠ   c           	      ó°  ‡— |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      sP| j                  ||||dœŠˆfd„ˆfd„d	œ}
i }	t        | j                  j                  «      D ]  } |
|   «       |	|<   Œ |}i }t        | j                  j                  «      D ]  }| j                  |||«      ||<   Œ t        | j                   d | j                  j"                   «      D ]G  \  }} ||f|	| j                  j                  |      || j                  j                  |      ||d
œ|¤Ž}ŒI | j%                  |«      }t'        ||r|¬«      S d ¬«      S )Nz:You must specify exactly one of input_ids or inputs_embeds)r�   r   rO   )r‡   )r�   r  rÝ   rÞ   r  c                  ó   •— t        di ‰ ¤ŽS ©Nra   )r   ©Úmask_kwargss   €r]   ú<lambda>z%LagunaModel.forward.<locals>.<lambda>¥  s   ø€ Ô*<Ñ*K¸{Ñ*K€ r^   c                  ó   •— t        di ‰ ¤ŽS r  )r   r  s   €r]   r  z%LagunaModel.forward.<locals>.<lambda>¦  s   ø€ Ô-NÑ-]ÐQ\Ñ-]€ r^   rT   )rÝ   rÜ   r  rÞ   )Úlast_hidden_staterÞ   )rf   Úembed_tokensr   r�   Úget_seq_lengthr�   r‘   rÇ   r‡   rî   r	  ÚdictÚsetrW   Ú
rotary_embÚ	enumerateÚlayersr4   Únormr   )rZ   r  rÝ   r  rÞ   r  r  r[   Úpast_seen_tokensÚcausal_mask_mappingÚmask_creation_functionsr‰   r§   rÜ   ÚiÚdecoder_layerr  s                   @r]   r¼   zLagunaModel.forward„  s  ø€ ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-ÄÔFàŸ+™+Ø!.Ø"0Ø#2Ø ,ñˆKó #LÛ%]ñ'Ð#ð #%ÐÜ! $§+¡+×"9Ñ"9Ó:ò X�
Ø2UÐ2IÈ*Ñ2UÓ2WÐ# JÒ/ðXð &ˆØ ÐÜ˜dŸk™k×5Ñ5Ó6ò 	gˆJØ.2¯o©o¸mÈ\Ð[eÓ.fÐ 
Ò+ð	gô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÓ Wò 	ÑˆAˆ}Ù)Øðà2°4·;±;×3JÑ3JÈ1Ñ3MÑNØ$7¸¿¹×8OÑ8OÐPQÑ8RÑ$SØ)Ø /ñð ñ‰Mð	ð Ÿ	™	 -Ó0ˆä%Ø+Ù/8˜Oô
ð 	
à>Bô
ð 	
r^   )NNNNNN)rh   ri   rj   r�   Ú
LongTensorr½   r   ÚFloatTensorrp   r   r   r   r¼   ra   r^   r]   r  r  ƒ  s¡   „ ð .2Ø.2Ø04Ø(,Ø26Ø!%ñ<
à×#Ñ# dÑ*ð<
ð Ÿ™ tÑ+ð<
ð ×&Ñ&¨Ñ-ð	<
ð
  ™ð<
ð ×(Ñ(¨4Ñ/ð<
ð ˜$‘;ð<
ð Ð+Ñ,ð<
ð 
 ô<
r^   r  c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚLagunaForCausalLMc                 ó"   •— t        ‰| �  di |¤ŽS )a­  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        ra   )rƒ   r¼   )rZ   Úsuper_kwargsr†   s     €r]   r¼   zLagunaForCausalLM.forwardÄ  s   ø€ ô ‰w‰Ñ. Ñ.Ð.r^   )rh   ri   rj   r¼   r›   rœ   s   @r]   r/  r/  Ã  s   ø„ ÷/ð /r^   r/  )r*   r/  r  r  )Ork   Úcollections.abcr   Útypingr   r   r   r�   Útorch.nn.functionalr   Ú
functionalr­   Úhuggingface_hub.dataclassesr   Ú r
   r
  Úcache_utilsr   r   Úconfiguration_utilsr   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_outputsr   Úmodeling_rope_utilsr   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   Úutils.genericr   r   Úafmoe.modeling_afmoer   Úgemma3.modeling_gemma3r   Ú$glm4_moe_lite.modeling_glm4_moe_liter   Úllama.modeling_llamar   r   Ú!qwen2_moe.configuration_qwen2_moer   Úqwen2_moe.modeling_qwen2_moer    r!   r"   r#   Ú qwen3_5_moe.modeling_qwen3_5_moer$   r%   Úqwen3_moe.modeling_qwen3_moer&   r'   Ú
get_loggerrh   Úloggerr*   r|   r€   rž   r    r¿   rÁ   rÏ   rú   r  r  r/  Ú__all__ra   r^   r]   ú<module>rM     sK  ðñ å $ß )Ñ )ã ß Ð Ý .Ý å &ß .Ý 3ß RÝ BÝ 6Ý 6Ý 5Ý &ß ,ß EÝ 1Ý :Ý Jß FÝ >ß uÓ uß Yß Rð 
ˆ×	Ñ	˜HÓ	%€ñ Ð1Ô2ØôB�>ó Bó ó 3ðBôJ	�Oô 	ô'*Ð1ô '*ôT	�ô 	ô@Ð+ô @ô6	�Oô 	ôÐ1ô ð* ôE)�nó E)ó ðE)ôP
cÐ0ô 
cô^Ð3ô ^ô =
�*ô =
ô@/Ð+ô /ò�r^   