
    ^j)Q                        d Z ddlmZ ddlmZmZmZ ddlZddlm	c m
Z ddlmZ ddlm	Z	 ddlmZ dd	lmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddl!m"Z" ddl#m$Z$m%Z% ddl&m'Z'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5m6Z6m7Z7m8Z8 ddl9m:Z:m;Z; ddl<m=Z=m>Z>  e%j~                  e@      ZA e$d      e G d de3                    ZB G d  d!e8      ZC G d" d#e,      ZD G d$ d%e6      ZE G d& d'e:      ZF G d( d)e=      ZG G d* d+e>      ZHe( G d, d-e*             ZI G d. d/e.      ZJ G d0 d1e7      ZK G d2 d3e0      ZL G d4 d5e5      ZMg d6ZNy)7zPyTorch Laguna model.    )Callable)AnyLiteralOptionalN)strict)nn   )initialization)CacheDynamicCache)PreTrainedConfig)create_causal_mask!create_sliding_window_causal_mask)FlashAttentionKwargs)MoeModelOutputWithPast)ROPE_INIT_FUNCTIONS)ALL_ATTENTION_FUNCTIONS)Unpack)auto_docstringlogging)TransformersKwargsno_inherit_decorator   )AfmoeAttention)Gemma3RotaryEmbedding)Glm4MoeLiteDecoderLayer)
LlamaModeleager_attention_forward)Qwen2MoeConfig)Qwen2MoeForCausalLMQwen2MoeMLPQwen2MoePreTrainedModelQwen2MoeRMSNorm)Qwen3_5MoeTopKRouterapply_rotary_pos_emb)Qwen3MoeExpertsQwen3MoeSparseMoeBlockzpoolside/laguna-XS.2)
checkpointc                      e Zd ZU dZdZi dddddddddd	d
ddddddddd	dddd	dddddddd	ZdZeed<   dZ	eed<   dZ
eed<   dZeed<   d Zeed!<   d"Zeed#<   d$Zeed%<   d Zeed&<   d'Zeed(<   d'Zeed)<   d'Zeed*<   d+Zeed,<   d-Zeed.<   d/Zeez  ed0<   d1Zee   d1z  ed2<   d1Zee   d1z  ed3<   d4Zeed5<   d-Zeed6<   d7Zeed8<    e       Z  e       Z! e       Z" e       Z# e       Z$ e       Z%d9 Z&d: Z'd; Z(y1)<LagunaConfigu  
    gating (`bool` or `str`, *optional*, defaults to `True`):
        Softplus output-gate granularity. ``True`` or ``"per-head"`` applies one gate per head,
        broadcast across ``head_dim``; ``"per-element"`` applies one gate per ``(head, head_dim)``
        channel.
    num_attention_heads_per_layer (`list[int]`, *optional*):
        Per-layer override for ``num_attention_heads``. Length must equal ``num_hidden_layers``.
    mlp_layer_types (`list[str]`, *optional*):
        Per-layer MLP type — ``"dense"`` or ``"sparse"``. Length must equal
        ``num_hidden_layers``. Defaults to first layer dense, rest sparse.
    moe_routed_scaling_factor (`float`, *optional*, defaults to 1.0):
        Scalar applied to routed-expert output before combining with the shared-expert output.
    moe_apply_router_weight_on_input (`bool`, *optional*, defaults to `False`):
        Whether to apply router weights to the MoE input rather than the output. Not supported
        in transformers yet; ``True`` will raise a ``NotImplementedError`` for now.
    moe_router_logit_softcapping (`float`, *optional*, defaults to 0.0):
        Scaling factor when applying tanh softcapping on the logits of the MoE router logits.

    Example:

    ```python
    >>> from transformers import LagunaModel, LagunaConfig

    >>> configuration = LagunaConfig()
    >>> model = LagunaModel(configuration)
    >>> configuration = model.config
    ```
    lagunazlayers.*.self_attn.q_projcolwisezlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.g_projzlayers.*.self_attn.o_projrowwisezlayers.*.self_attn.q_normreplicated_with_grad_allreducezlayers.*.self_attn.k_normzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projz!layers.*.mlp.experts.gate_up_projpacked_colwisezlayers.*.mlp.experts.down_projzlayers.*.mlp.expertsmoe_tp_expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_proji  
vocab_sizei    intermediate_size(   num_hidden_layers0   num_attention_heads   num_key_value_headsi   max_position_embeddings   num_expertsnum_experts_per_toki   moe_intermediate_sizeshared_expert_intermediate_sizesliding_window   head_dimFattention_biasTgatingNnum_attention_heads_per_layermlp_layer_types      ?moe_routed_scaling_factor moe_apply_router_weight_on_input        moe_router_logit_softcappingc                 d   | j                   dg| j                  z  | _         | j                  dgdg| j                  dz
  z  z   | _        | j                  | j                  g| j                  z  | _        dddddd	d
dd}| j
                  || _        t        j                  | fi |dddhi y )Nfull_attentiondensesparse   defaultg    Ag      ?)	rope_type
rope_thetapartial_rotary_factorg     @rF   rL   sliding_attentionignore_keys_at_rope_validationrU   )layer_typesr4   rE   rD   r6   rope_parametersr   __post_init__)selfkwargsdefault_rope_paramss      t/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/laguna/modular_laguna.pyrY   zLagunaConfig.__post_init__   s    # 01D4J4JJD'$+9zT=S=SVW=W/X#XD --5262J2J1KdNdNd1dD. -6Xhkl/8jm!ne
 '#6D  	&&	
	
<OQa;b	
    c                     |S N )rZ   r[   s     r]   convert_rope_params_to_dictz(LagunaConfig.convert_rope_params_to_dict   s    r^   c                 0   | j                   rt        d      | j                  Qt        | j                        | j                  k7  r/t        dt        | j                         d| j                   d      t        | j                        | j                  k7  r/t        dt        | j                         d| j                   d      t        | j                        | j                  k7  r/t        dt        | j                         d| j                   d      y)z'Part of ``@strict``-powered validation.zhmoe_apply_router_weight_on_input=True is not yet supported in the transformers implementation of Laguna.Nz&num_attention_heads_per_layer length (z ) must equal num_hidden_layers (z).zlayer_types length (zmlp_layer_types length ()rH   NotImplementedErrorrD   lenr4   
ValueErrorrW   rE   )rZ   s    r]   validate_architecturez"LagunaConfig.validate_architecture   s,   00%9 
 ..:D6674;Q;QQ8T=_=_9`8a b1151G1G0HL  t D$:$::&s4+;+;'<&= >1151G1G0HL  t##$(>(>>*3t/C/C+D*E F1151G1G0HL  ?r^   ))__name__
__module____qualname____doc__
model_typebase_model_tp_planr1   int__annotations__r2   r4   r6   r8   r9   r;   r<   r=   r>   r?   rA   rB   boolrC   strrD   listrE   rG   floatrH   rJ   AttributeErrordecoder_sparse_stepmlp_only_layersqkv_biasnorm_topk_probuse_sliding_windowmax_window_layersrY   rb   rg   ra   r^   r]   r*   r*   0   s   : J#Y#Y 	$Y 	$Y	
 	$Y 	$%E 	$%E 	!) 		 	!) 	,-= 	)) 	 0 	0 	.y  	0!& J!s!s!!  #)S)K  !$3$+.#S.NC Hc ND FD3J6:!49t#3:(,OT#Y%,'*u*-2$d2*- %- )*$&OH#%N')&(
(r^   r*   c                       e Zd Zy)LagunaRMSNormNrh   ri   rj   ra   r^   r]   r|   r|          r^   r|   c                   v     e Zd Zdef fdZe	 	 	 	 ddedz  ded   dedz  dedz  de	d	e
f   f
d
       Z xZS )LagunaRotaryEmbeddingconfigc                 $    t         |   |       y r`   )super__init__rZ   r   	__class__s     r]   r   zLagunaRotaryEmbedding.__init__   s     r^   Ndeviceztorch.deviceseq_len
layer_typereturnztorch.Tensorc                 z   | j                   |   d   }| j                   |   j                  dd      }t        | dd      xs | j                  | j                  z  }t        ||z        }d}d|t        j                  d|dt        j                        j                  |t        j                  	      |z  z  z  }	|	|fS )
a{  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        rR   rS   rF   rA   Nr   r   )dtype)r   r   )rX   getgetattrhidden_sizer6   rn   torcharangeint64tors   )
r   r   r   r   baserS   rA   dimattention_factorinv_freqs
             r]   compute_default_rope_parametersz5LagunaRotaryEmbedding.compute_default_rope_parameters   s    . %%j1,? & 6 6z B F FG^`c d6:t4h8J8JfNhNh8h(223 U\\!S!5;;?BB&X]XcXcBdgjjk
 )))r^   )NNNN)rh   ri   rj   r*   r   staticmethodr   rn   rq   tuplers   r   __classcell__r   s   @r]   r   r      sy    !| ! &*+/"!%	"*t#"*("* t"* $J	"*
 
~u$	%"* "*r^   r   c                       e Zd Zy)	LagunaMLPNr}   ra   r^   r]   r   r      r~   r^   r   c                        e Zd Z fdZdej
                  deej
                  ej
                  ej
                  f   fdZ xZS )LagunaTopKRouterc                     t         |           t        j                  t	        j
                  |j                        d      | _        |j                  | _	        y )NF)requires_grad)
r   r   r   	Parameterr   zerosr;   e_score_correction_biasrJ   router_logit_softcappingr   s     r]   r   zLagunaTopKRouter.__init__   s?    ')||EKK@R@R4Sch'i$(.(K(K%r^   hidden_statesr   c                 f   |j                  d| j                        }t        j                  || j                        j                         }| j                  dkD  r/t        j                  || j                  z        | j                  z  }t        j                  |      }|| j                  j                  |j                        z   }t        j                  || j                  d      \  }}|j                  d|      }||j!                  dd      z  }|j                  |j                        }|||fS )NrI   )r   T)r   keepdim)reshape
hidden_dimFlinearweightrs   r   r   tanhsigmoidr   r   r   topktop_kgathersum)rZ   r   router_logitsrouting_scoresscores_for_selection_selected_expertsrouting_weightss           r]   forwardzLagunaTopKRouter.forward   s
    &--b$//B<BBD((3.!JJ}t7T7T'TUX\XuXuuM}5-0L0L0O0OP^PdPd0ee#jj)=tzzrR(//4DE)O,?,?BPT,?,UU),,]-@-@Ao/???r^   )	rh   ri   rj   r   r   Tensorr   r   r   r   s   @r]   r   r      s?    L
@||@ 
u||U\\5<<7	8@r^   r   c                       e Zd Zy)LagunaExpertsNr}   ra   r^   r]   r   r     r~   r^   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )LagunaSparseMoeBlockr   c                 ~    t         |   |       t        ||j                        | _        |j
                  | _        y )Nr2   )r   r   r   r>   shared_expertsrG   routed_scaling_factorr   s     r]   r   zLagunaSparseMoeBlock.__init__	  s3     '&BhBhi%+%E%E"r^   r   r   c                 
   |j                   \  }}}|j                  d|      }| j                  |      }| j                  |      \  }}}| j	                  |||      }|| j
                  z  }||z   }|j                  |||      }|S )Nr   )shapeviewr   gateexpertsr   r   )	rZ   r   
batch_sizesequence_lengthr   shared_outputr   r   r   s	            r]   r   zLagunaSparseMoeBlock.forward  s    2?2E2E/
OZ%**2z:++M:/3yy/G,?,]4DoV%(B(BB%5%--j/:Vr^   )	rh   ri   rj   r*   r   r   r   r   r   r   s   @r]   r   r     s,    F| F
U\\ ell r^   r   c                        e Zd ZdZdededef fdZ	 ddej                  de	ej                  ej                  f   d	ej                  dz  d
e
dz  dee   de	ej                  ej                  dz  f   fdZ xZS )LagunaAttentionzSAfmoe-style SWA/GQA attention with Laguna-specific gating and per-layer head count.r   	layer_idx	num_headsc                    || _         t        | 	  ||       | j                   |j                  z  | _        t        j                  |j                  | j                   | j                  z  |j                        | _
        t        j                  | j                   | j                  z  |j                  |j                        | _        | `|j                  du xs |j                  dk(  | _        | j                  r| j                   n| j                   | j                  z  }t        j                  |j                  |d      | _        y )N)biasTzper-headF)r   r   r   r8   num_key_value_groupsr   Linearr   rA   rB   q_projo_proj	gate_projrC   gate_per_headg_proj)rZ   r   r   r   
g_proj_dimr   s        r]   r   zLagunaAttention.__init__!  s    "+$(NNf6P6P$P!ii 2 2DNNT]]4RY_YnYnoii >@R@RY_YnYnoN#]]d2Qfmmz6Q'+'9'9T^^t~~PTP]P]?]
ii 2 2JUKr^   Nr   position_embeddingsattention_maskpast_key_valuesr[   r   c                    |j                   d d }g |d| j                  }| j                  |      j                  |      }| j	                  |      j                  |      }	| j                  |      j                  |      }
| j                  |      j                  dd      }| j                  |	      j                  dd      }	|
j                  dd      }
|\  }}t        ||	||      \  }}	| |j                  |	|
| j                        \  }	}
t        j                  | j                  j                  t               } || ||	|
|f| j"                  sdn| j$                  | j&                  | j(                  d|\  }} |j*                  g |d j-                         }t/        j0                  | j3                  |      j5                               j7                  |j8                        }| j:                  rE  |j                  g |d| j                   |j=                  d      z  j                  g |d }n||z  }| j?                  |      }||fS )Nr   rO   r   rI   )dropoutscalingr?   ) r   rA   r   r   k_projv_projq_norm	transposek_normr%   updater   r   get_interfacer   _attn_implementationr   trainingattention_dropoutr   r?   r   
contiguousr   softplusr   rs   r   r   r   	unsqueezer   )rZ   r   r   r   r   r[   input_shapehidden_shapequery_states
key_statesvalue_statescossinattention_interfaceattn_outputattn_weightsr   s                    r]   r   zLagunaAttention.forward0  sN    $))#2.88b8$--8{{=166|D[[/44\B
{{=166|D{{<0::1a@[[,66q!<
#--a3&S#7jRUWZ#[ j&'6'='=j,X\XfXf'g$J(?(M(MKK,,.E)
 %8
%
  $}}C$2H2HLL..
%
 
%
!\ *k));;;;FFHzz$++m4::<=@@ARARSg+;++L[L"LdmmLt~~^`Oaagg  K &,Kkk+.L((r^   r`   )rh   ri   rj   rk   r*   rn   r   r   r   r   r   r   r   r   r   r   s   @r]   r   r     s    ]L| L L L( )-3)||3) #5<<#=>3) t+	3)
 3) -.3) 
u||U\\D00	13)r^   r   c                       e Zd ZdedefdZy)LagunaDecoderLayerr   r   c                    t         j                  j                  |        |j                  | _        t	        |||j
                  |         | _        |j                  |   dk(  rt        |      | _	        nt        ||j                        | _	        t        |j                  |j                        | _        t        |j                  |j                        | _        y )NrN   r   )eps)r   Moduler   r   r   rD   	self_attnrE   r   mlpr   r2   r|   rms_norm_epsinput_layernormpost_attention_layernorm)rZ   r   r   s      r]   r   zLagunaDecoderLayer.__init__g  s    
		4 !--(F<`<`aj<kl!!),8+F3DH 6;S;STDH,V-?-?VEXEXY(5f6H6HfNaNa(b%r^   N)rh   ri   rj   r*   rn   r   ra   r^   r]   r   r   f  s    	c| 	c 	cr^   r   c                   F     e Zd Z ej                          fd       Z xZS )LagunaPreTrainedModelc                    t         |   |       t        |t              r4t        j
                  j                  j                  |j                         y t        |t              r|j                  D ]  }|j                  }|j                  |   dk7  rt        |j                  |      } ||j                  |      \  }}t        j                  t!        || d      |       t        j                  t!        || d      |        y y )NrP   )r   	_inv_freq_original_inv_freq)r   _init_weights
isinstancer   r   r   initzeros_r   r   rW   r   rQ   r   r   copy_r   )rZ   moduler   rope_init_fncurr_inv_freqr   r   s         r]   r  z#LagunaPreTrainedModel._init_weightst  s    f%f./HHMM  !?!?@ 56$00 ^
%EE##J/9<#6v7G7G
7S#TL#/*#U q

76j\+CDmT

76j\9K+LM}]^ 7r^   )rh   ri   rj   r   no_gradr  r   r   s   @r]   r  r  s  s    U]]_^ ^r^   r  c                       e Zd Z	 	 	 	 	 	 ddej                  dz  dej
                  dz  dej                  dz  dedz  dej                  dz  dedz  de	e
   d	efd
Zy)LagunaModelN	input_idsr   position_idsr   inputs_embeds	use_cacher[   r   c           	         |d u |d uz  rt        d      || j                  |      }|r|t        | j                        }|V||j	                         nd}t        j                  |j                  d   |j                        |z   }|j                  d      }t        |x}	t              sP| j                  ||||dfdfdd	}
i }	t        | j                  j                        D ]  } |
|          |	|<    |}i }t        | j                  j                        D ]  }| j                  |||      ||<    t        | j                   d | j                  j"                         D ]G  \  }} ||f|	| j                  j                  |      || j                  j                  |      ||d
|}I | j%                  |      }t'        ||r|      S d       S )Nz:You must specify exactly one of input_ids or inputs_embeds)r   r   rO   )r   )r   r  r   r   r  c                      t        di  S Nra   )r   mask_kwargss   r]   <lambda>z%LagunaModel.forward.<locals>.<lambda>  s    *<*K{*K r^   c                      t        di  S r  )r   r  s   r]   r  z%LagunaModel.forward.<locals>.<lambda>  s    -N-]Q\-] r^   rT   )r   r   r  r   )last_hidden_stater   )rf   embed_tokensr   r   get_seq_lengthr   r   r   r   r   r	  dictsetrW   
rotary_emb	enumeratelayersr4   normr   )rZ   r  r   r  r   r  r  r[   past_seen_tokenscausal_mask_mappingmask_creation_functionsr   r   r   idecoder_layerr  s                   @r]   r   zLagunaModel.forward  s    -t";<YZZ  --i8M0*$++>OCRC^==?de <<(;(;A(>}G[G[\_ooL'11!4L?-F++!."0#2 ,K #L%]'# #%!$++"9"9: X
2U2I*2U2W#J/X & dkk556 	gJ.2oom\[e.f
+	g !*$++6U8U8U*V W 	A})24;;3J3J13MN$78O8OPQ8R$S) / M	 		-0%+/8O
 	
>B
 	
r^   )NNNNNN)rh   ri   rj   r   
LongTensorr   r   FloatTensorrp   r   r   r   r   ra   r^   r]   r  r    s     .2.204(,26!%<
##d*<
 t+<
 &&-	<

 <
 ((4/<
 $;<
 +,<
 
 <
r^   r  c                        e Zd Z fdZ xZS )LagunaForCausalLMc                 "    t        |   di |S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        ra   )r   r   )rZ   super_kwargsr   s     r]   r   zLagunaForCausalLM.forward  s     w...r^   )rh   ri   rj   r   r   r   s   @r]   r/  r/    s    / /r^   r/  )r*   r/  r  r  )Ork   collections.abcr   typingr   r   r   r   torch.nn.functionalr   
functionalr   huggingface_hub.dataclassesr    r
   r
  cache_utilsr   r   configuration_utilsr   masking_utilsr   r   modeling_flash_attention_utilsr   modeling_outputsr   modeling_rope_utilsr   modeling_utilsr   processing_utilsr   utilsr   r   utils.genericr   r   afmoe.modeling_afmoer   gemma3.modeling_gemma3r   $glm4_moe_lite.modeling_glm4_moe_liter   llama.modeling_llamar   r   !qwen2_moe.configuration_qwen2_moer   qwen2_moe.modeling_qwen2_moer    r!   r"   r#    qwen3_5_moe.modeling_qwen3_5_moer$   r%   qwen3_moe.modeling_qwen3_moer&   r'   
get_loggerrh   loggerr*   r|   r   r   r   r   r   r   r   r  r  r/  __all__ra   r^   r]   <module>rM     sK    $ ) )    .  & . 3 R B 6 6 5 & , E 1 : J F > u u Y R 
		H	% 12B> B  3BJ	O 	'*1 '*T	 	@+ @6	O 	1 * E)n E) E)P
c0 
c^3 ^ =
* =
@/+ /r^   