Ë
    ÛÓ^jˆ  ã                   ój  — d dl mZ d dlmZ d dlZd dlmc mZ d dlmZ ddl	m
Z ddlmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZmZ ddlmZ ddlmZ ddlmZmZ ddl m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z' ddl(m)Z)m*Z* ddl+m,Z,m-Z-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3  ed«       G d„ dejh                  «      «       Z5 G d„ dejh                  «      Z6 G d„ dejh                  «      Z7 G d„ dejh                  «      Z8e G d „ d!ejh                  «      «       Z9 G d"„ d#ejh                  «      Z:d$„ Z;dAd%„Z<d&ejz                  d'e>d(ejz                  fd)„Z?	 dBd*ejh                  d+ejz                  d,ejz                  d-ejz                  d.ejz                  dz  d/e@d0e@d1e'e,   fd2„ZA G d3„ d4ejh                  «      ZB G d5„ d6e«      ZCe) G d7„ d8e%«      «       ZDe) G d9„ d:eD«      «       ZE	 	 	 dCd;ejz                  eFejz                     z  dz  d<e>dz  d.ejz                  dz  d(ejz                  e>z  fd=„ZGe) G d>„ d?eDe«      «       ZHg d@¢ZIy)Dé    )ÚCallable)ÚOptionalN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hub)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úauto_docstringÚcan_return_tuple)ÚTransformersKwargsÚmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚLagunaConfigÚRMSNormc                   óh   ‡ — e Zd Zddeddfˆ fd„Zdej                  dej                  fd„Zd„ Zˆ xZ	S )	ÚLagunaRMSNormÚepsÚreturnNc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)z<
        LagunaRMSNorm is equivalent to T5LayerNorm
        N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer%   Ú	__class__s      €úu/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/laguna/modeling_laguna.pyr)   zLagunaRMSNorm.__init__0   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕó    Úhidden_statesc                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S )Né   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtor+   Úfloat32ÚpowÚmeanÚrsqrtr.   r-   )r/   r4   Úinput_dtypeÚvariances       r2   ÚforwardzLagunaRMSNorm.forward8   sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r3   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)Útupler-   Úshaper.   )r/   s    r2   Ú
extra_reprzLagunaRMSNorm.extra_repr?   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr3   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Úfloatr)   r+   ÚTensorrA   rE   Ú__classcell__©r1   s   @r2   r$   r$   .   s7   ø„ ñ$¨ð $¸$õ $ð; U§\¡\ð ;°e·l±ló ;öJr3   r$   c                   óÐ   ‡ — e Zd ZU ej                  ed<   defˆ fd„Ze	 	 	 	 ddedz  de	d   de
dz  dedz  d	ed
ef   f
d„«       Z ej                  «       edd„«       «       Zˆ xZS )ÚLagunaRotaryEmbeddingÚinv_freqÚconfigc                 ót  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        t        t        |j                  «      «      | _        i | _	        | j                  D ]Ç  }| j
                  j                  |   }|€Œ|d   | j                  |<   | j                  }| j                  |   dk7  rt        | j                  |      } || j
                  |¬«      \  }}| j                  |› d�|d¬«       | j                  |› d�|j                  «       d¬«       t        | |› d�|«       ŒÉ y )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqF)Ú
persistentÚ_original_inv_freqÚ_attention_scaling)r(   r)   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrP   ÚlistÚsetÚlayer_typesrR   Úrope_parametersÚcompute_default_rope_parametersr   Úregister_bufferÚcloneÚsetattr)r/   rP   rU   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingr1   s          €r2   r)   zLagunaRotaryEmbedding.__init__F   s6  ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØˆŒÜ¤ F×$6Ñ$6Ó 7Ó8ˆÔØˆŒØ×*Ñ*ò 	UˆJØŸ+™+×5Ñ5°jÑAˆKØÐ"Øà)4°[Ñ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÑ)¨YÒ6Ü2°4·>±>À*Ñ3MÑN�Ù4@ÀÇÁÐYcÔ4dÑ1ˆMÐ1Ø× Ñ  J <¨yÐ!9¸=ÐUZÐ Ô[Ø× Ñ  J <Ð/AÐ!BÀM×DWÑDWÓDYÐfkÐ ÔlÜ�D˜Z˜LÐ(:Ð;Ð=SÕTñ	Ur3   NÚdeviceztorch.deviceÚseq_lenrU   r&   ztorch.Tensorc                 óz  — | j                   |   d   }| j                   |   j                  dd«      }t        | dd«      xs | j                  | j                  z  }t        ||z  «      }d}d|t        j                  d|dt        j                  ¬«      j                  |t        j                  ¬	«      |z  z  z  }	|	|fS )
a{  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
            layer_type (`str`, *optional*):
                The current layer type if the model has different RoPE parameters per type.
                Should not be used unless `config.layer_types is not None`
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimNr   r6   ©r9   )ri   r9   )r`   ÚgetÚgetattrr0   Únum_attention_headsÚintr+   ÚarangeÚint64r:   rI   )
rP   ri   rj   rU   Úbaserm   rn   ÚdimÚattention_factorrO   s
             r2   ra   z5LagunaRotaryEmbedding.compute_default_rope_parameters[   sÅ   € ð. ×%Ñ% jÑ1°,Ñ?ˆà &× 6Ñ 6°zÑ B× FÑ FÐG^Ð`cÓ dÐÜ˜6 :¨tÓ4Òh¸×8JÑ8JÈf×NhÑNhÑ8hˆÜ�(Ð2Ñ2Ó3ˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r3   c                 óN  — t        | |› d�«      }t        | |› d�«      }|d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬	«      5  |j                  «       |j                  «       z  j                  dd
«      }	t        j                  |	|	fd¬«      }
|
j                  «       |z  }|
j                  «       |z  }d d d «       j	                  |j                  ¬«      j	                  |j                  ¬«      fS # 1 sw Y   ŒAxY w)NrV   rY   r   r7   r    ÚmpsÚcpuF)Údevice_typeÚenabledr6   ©rw   ro   )rq   rI   ÚexpandrD   r:   ri   Ú
isinstanceÚtypeÚstrr   Ú	transposer+   ÚcatÚcosÚsinr9   )r/   ÚxÚposition_idsrU   rO   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedr|   ÚfreqsÚembr…   r†   s                r2   rA   zLagunaRotaryEmbedding.forward€   sl  € ô ˜4 J <¨yÐ!9Ó:ˆÜ# D¨Z¨LÐ8JÐ*KÓLÐà$ Tª1¨d ]Ñ3×9Ñ9Ó;×BÑBÀ<×CUÑCUÐVWÑCXÐZ\Ð^_Ó`×cÑcÐde×dlÑdlÓmÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	0Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)Ð/Ñ/ˆCØ—'‘'“)Ð/Ñ/ˆC÷		0ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	0ð 	0ús   Ã*A1FÆF$)NNNN©N)rF   rG   rH   r+   rJ   Ú__annotations__r!   r)   Ústaticmethodr   rs   r‚   rC   rI   ra   Úno_gradr   rA   rK   rL   s   @r2   rN   rN   C   sª   ø… Ø�l‰lÓðU˜|õ Uð* à&*Ø+/Ø"Ø!%ñ	"*Ø˜tÑ#ð"*à˜Ñ(ð"*ð �t‘ð"*ð ˜$‘Jð	"*ð
 
ˆ~˜uÐ$Ñ	%ò"*ó ð"*ðH €U‡]�]ƒ_Øò<ó ó ô<r3   rN   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )Ú	LagunaMLPc                 óÒ  •— t         ‰| �  «        || _        |j                  | _        |€|j                  n|| _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _        t        j                  | j                  | j                  d¬«      | _	        t        |j                     | _        y ©NF©Úbias)r(   r)   rP   r0   Úintermediate_sizer   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn)r/   rP   r˜   r1   s      €r2   r)   zLagunaMLP.__init__”   s®   ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ=NÐ=V ×!9Ò!9Ð\mˆÔÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆŒÜ˜V×.Ñ.Ñ/ˆ�r3   c                 óˆ   — | j                  | j                  | j                  |«      «      | j                  |«      z  «      }|S rŽ   )rœ   rž   rš   r›   )r/   r‡   rœ   s      r2   rA   zLagunaMLP.forwardž   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÑ#SÓTˆ	ØÐr3   rŽ   )rF   rG   rH   r)   rA   rK   rL   s   @r2   r“   r“   “   s   ø„ õ0ör3   r“   c                   óŠ   ‡ — e Zd Zˆ fd„Zdej
                  deej
                  ej
                  ej
                  f   fd„Zˆ xZS )ÚLagunaTopKRouterc                 ó   •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        t        j                  t        j                  | j                  | j                  «      «      | _        t        j                  t        j                  |j                  «      d¬«      | _        |j                  | _        y )NF)Úrequires_grad)r(   r)   Únum_experts_per_tokÚtop_kÚnum_expertsr0   Ú
hidden_dimr   r*   r+   Úzerosr-   Úe_score_correction_biasÚmoe_router_logit_softcappingÚrouter_logit_softcapping©r/   rP   r1   s     €r2   r)   zLagunaTopKRouter.__init__¤   sŒ   ø€ Ü‰ÑÔØ×/Ñ/ˆŒ
Ø!×-Ñ-ˆÔØ ×,Ñ,ˆŒÜ—l‘l¤5§;¡;¨t×/?Ñ/?ÀÇÁÓ#QÓRˆŒÜ')§|¡|´E·K±KÀ×@RÑ@RÓ4SÐchÔ'iˆÔ$Ø(.×(KÑ(KˆÕ%r3   r4   r&   c                 óf  — |j                  d| j                  «      }t        j                  || j                  «      j                  «       }| j                  dkD  r/t        j                  || j                  z  «      | j                  z  }t        j                  |«      }|| j                  j                  |j                  «      z   }t        j                  || j                  d¬«      \  }}|j                  d|«      }||j!                  dd¬«      z  }|j                  |j                  «      }|||fS )Nr7   ç        r~   T)rw   r8   )Úreshaper§   ÚFÚlinearr-   rI   r«   r+   ÚtanhÚsigmoidr©   r:   r9   Útopkr¥   ÚgatherÚsum)r/   r4   Úrouter_logitsÚrouting_scoresÚscores_for_selectionÚ_Úselected_expertsÚrouting_weightss           r2   rA   zLagunaTopKRouter.forward­   s
  € ð &×-Ñ-¨b°$·/±/ÓBˆÜŸ™ °·±Ó<×BÑBÓDˆà×(Ñ(¨3Ò.Ü!ŸJ™J }°t×7TÑ7TÑ'TÓUÐX\×XuÑXuÑuˆMäŸ™ }Ó5ˆà-°×0LÑ0L×0OÑ0OÐP^×PdÑPdÓ0eÑeÐÜ#Ÿj™jÐ)=¸t¿z¹zÈrÔRÑˆÐØ(×/Ñ/°Ð4DÓEˆØ)¨O×,?Ñ,?ÀBÐPTÐ,?Ó,UÑUˆØ)×,Ñ,¨]×-@Ñ-@ÓAˆà˜oÐ/?Ð?Ð?r3   )	rF   rG   rH   r)   r+   rJ   rC   rA   rK   rL   s   @r2   r¡   r¡   £   s?   ø„ ôLð@à—|‘|ð@ð 
ˆu�|‰|˜UŸ\™\¨5¯<©<Ð7Ñ	8÷@r3   r¡   c                   óŠ   ‡ — e Zd ZdZˆ fd„Zdej                  dej                  dej                  dej                  fd„Zˆ xZS )ÚLagunaExpertsz2Collection of expert weights stored as 3D tensors.c                 óò  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        t        j                  t        j                  | j                  d| j                  z  | j                  «      «      | _        t        j                  t        j                  | j                  | j                  | j                  «      «      | _        t        |j                     | _        y )Nr6   )r(   r)   r¦   r0   r§   Úmoe_intermediate_sizeÚintermediate_dimr   r*   r+   ÚemptyÚgate_up_projrœ   r   r�   rž   r¬   s     €r2   r)   zLagunaExperts.__init__Æ   s­   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ ×,Ñ,ˆŒØ &× <Ñ <ˆÔÜŸL™L¬¯©°T×5EÑ5EÀqÈ4×K`ÑK`ÑG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸ™¤e§k¡k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Ñ/ˆ�r3   r4   Útop_k_indexÚtop_k_weightsr&   c                 óf  — t        j                  |«      }t        j                  «       5  t         j                  j                  j                  || j                  ¬«      }|j                  ddd«      }t        j                  |j                  d¬«      d«      j                  «       }d d d «       D ]ð  }|d   }|| j                  k(  rŒt        j                  |   «      \  }}	||	   }
t        j                  j                  |
| j                  |   «      j                  dd¬«      \  }}| j                  |«      |z  }t        j                  j                  || j                   |   «      }|||	|d f   z  }|j#                  d|	|j%                  |j&                  «      «       Œò |S # 1 sw Y   �ŒxY w)N)Únum_classesr6   r    r   )r7   éþÿÿÿr~   r7   )r+   Ú
zeros_liker‘   r   Ú
functionalÚone_hotr¦   ÚpermuteÚgreaterr¶   ÚnonzeroÚwherer±   rÃ   Úchunkrž   rœ   Ú
index_add_r:   r9   )r/   r4   rÄ   rÅ   Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess                 r2   rA   zLagunaExperts.forwardÏ   s™  € ô $×.Ñ.¨}Ó=ÐÜ�]‰]‹_ñ 	SÜŸ(™(×-Ñ-×5Ñ5°kÈt×O_ÑO_Ð5Ó`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸ™ {§¡¸8 Ó'DÀaÓH×PÑPÓRˆJ÷	Sð
 %ò 
	nˆJØ# A™ˆJØ˜T×-Ñ-Ò-ØÜ#(§;¡;¨{¸:Ñ/FÓ#GÑ ˆI�yØ)¨)Ñ4ˆMÜ—}‘}×+Ñ+¨M¸4×;LÑ;LÈZÑ;XÓY×_Ñ_Ð`aÐgiÐ_Ój‰HˆD�"Ø$(§K¡K°Ó$5¸Ñ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9NÐPT×P^ÑP^Ð_iÑPjÓ$kÐ!Ø$9¸MÈ)ÐU^Ð`dÐJdÑ<eÑ$eÐ!Ø×*Ñ*¨1¨iÐ9N×9QÑ9QÐRe×RkÑRkÓ9lÕmð
	nð #Ð"÷#	Sñ 	Sús   ªA=F&Æ&F0)	rF   rG   rH   Ú__doc__r)   r+   rJ   rA   rK   rL   s   @r2   r¾   r¾   Â   sF   ø„ á<ô0ð#à—|‘|ð#ð —\‘\ð#ð —|‘|ð	#ð
 
�‰÷#r3   r¾   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚLagunaSparseMoeBlockrP   c                 ó¼   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        t        ||j                  ¬«      | _        |j                  | _
        y )N©r˜   )r(   r)   r¾   Úexpertsr¡   rÙ   r“   Úshared_expert_intermediate_sizeÚshared_expertsÚmoe_routed_scaling_factorÚrouted_scaling_factorr¬   s     €r2   r)   zLagunaSparseMoeBlock.__init__ë   sI   ø€ Ü‰ÑÔÜ$ VÓ,ˆŒÜ$ VÓ,ˆŒ	Ü'¨À&×BhÑBhÔiˆÔØ%+×%EÑ%EˆÕ"r3   r4   r&   c                 ó
  — |j                   \  }}}|j                  d|«      }| j                  |«      }| j                  |«      \  }}}| j	                  |||«      }|| j
                  z  }||z   }|j                  |||«      }|S )Nr7   )rD   Úviewrã   rÙ   rá   rå   r¯   )	r/   r4   Ú
batch_sizeÚsequence_lengthr§   Úshared_outputrº   r¼   r»   s	            r2   rA   zLagunaSparseMoeBlock.forwardò   s“   € Ø2?×2EÑ2EÑ/ˆ
�O ZØ%×*Ñ*¨2¨zÓ:ˆØ×+Ñ+¨MÓ:ˆà/3¯y©y¸Ó/GÑ,ˆˆ?Ð,ØŸ™ ]Ð4DÀoÓVˆà%¨×(BÑ(BÑBˆØ%¨Ñ5ˆà%×-Ñ-¨j¸/È:ÓVˆØÐr3   )	rF   rG   rH   r!   r)   r+   rJ   rA   rK   rL   s   @r2   rÞ   rÞ   ê   s,   ø„ ðF˜|õ Fð U§\¡\ð °e·l±l÷ r3   rÞ   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Nr7   r6   r~   )rD   r+   r„   )r‡   Úx1Úx2s      r2   Úrotate_halfrî     sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r3   c                 óh  — |j                  |«      }|j                  |«      }|j                  d   }| dd|…f   | d|d…f   }}|dd|…f   |d|d…f   }	}||z  t        |«      |z  z   }
||z  t        |«      |z  z   }t        j                  |
|gd¬«      }
t        j                  ||	gd¬«      }|
|fS )a»  Applies Rotary Position Embedding to the query and key tensors.

    Removes the interleaving of cos and sin from GLM

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    r7   .Nr~   )Ú	unsqueezerD   rî   r+   r„   )ÚqÚkr…   r†   Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds               r2   Úapply_rotary_pos_embrû   	  sÞ   € ð( �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cð —‘˜2‘€JØ�c˜;˜J˜;Ð&Ñ'¨¨3°
±Ð+;Ñ)<ˆ6€EØ�c˜;˜J˜;Ð&Ñ'¨¨3°
±Ð+;Ñ)<ˆ6€Eð �s‰{œ{¨5Ó1°CÑ7Ñ8€GØ�s‰{œ{¨5Ó1°CÑ7Ñ8€Gô �i‰i˜ &Ð)¨rÔ2€GÜ�i‰i˜ &Ð)¨rÔ2€GØ�GÐÐr3   r4   Ún_repr&   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r    N)rD   r   r¯   )r4   rü   ÚbatchÚnum_key_value_headsÚslenrn   s         r2   Ú	repeat_kvr  /  so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr3   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t        || j                  «      }t        || j                  «      }	t        j                  ||j	                  dd«      «      |z  }
|�|
|z   }
t
        j                  j                  |
dt        j                  ¬«      j                  |j                  «      }
t
        j                  j                  |
|| j                  ¬«      }
t        j                  |
|	«      }|j	                  dd«      j                  «       }||
fS )Nr6   r   r7   )rw   r9   )ÚpÚtrainingr    )r  Únum_key_value_groupsr+   Úmatmulrƒ   r   rÊ   Úsoftmaxr;   r:   r9   r  r  Ú
contiguous)r  r  r  r  r  r  r  r	  Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r2   Úeager_attention_forwardr  ;  sà   € ô ˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r3   c                   óø   ‡ — e Zd ZdZdededefˆ fd„Z	 ddej                  de	ej                  ej                  f   d	ej                  dz  d
e
dz  dee   de	ej                  ej                  dz  f   fd„Zˆ xZS )ÚLagunaAttentionzSAfmoe-style SWA/GQA attention with Laguna-specific gating and per-layer head count.rP   Ú	layer_idxÚ	num_headsc                 ón  •— t         ‰| �  «        || _        || _        || _        t        |d|j                  |j                  z  «      | _        | j                  |j                  z  | _
        | j                  dz  | _        |j                  | _        d| _        t        j                  |j                  | j                  | j                  z  |j                   ¬«      | _        t        j                  |j                  |j                  | j                  z  |j                   ¬«      | _        t        j                  |j                  |j                  | j                  z  |j                   ¬«      | _        t        j                  | j                  | j                  z  |j                  |j                   ¬«      | _        |j*                  |   dk(  | _        | j,                  r|j.                  nd | _        t1        | j                  |j2                  ¬«      | _        t1        | j                  |j2                  ¬«      | _        |j8                  du xs |j8                  dk(  | _        | j:                  r| j                  n| j                  | j                  z  }t        j                  |j                  |d¬«      | _        y )	Nrn   g      à¿Tr–   Úsliding_attention©r%   zper-headF)r(   r)   r  rP   r  rq   r0   rr   rn   rÿ   r  r  Úattention_dropoutÚ	is_causalr   r™   Úattention_biasÚq_projÚk_projÚv_projÚo_projr_   Úis_local_attentionÚsliding_windowr$   Úrms_norm_epsÚq_normÚk_normÚgatingÚgate_per_headÚg_proj)r/   rP   r  r  Ú
g_proj_dimr1   s        €r2   r)   zLagunaAttention.__init__W  s  ø€ Ü‰ÑÔà"ˆŒØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$(§N¡N°f×6PÑ6PÑ$PˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i‘i × 2Ñ 2°D·N±NÀTÇ]Á]Ñ4RÐY_×YnÑYnÔoˆŒÜ—i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘i §¡°·±Ñ >À×@RÑ@RÐY_×YnÑYnÔoˆŒð #)×"4Ñ"4°YÑ"?ÐCVÑ"VˆÔØ7;×7NÒ7N˜f×3Ò3ÐTXˆÔä# D§M¡M°v×7JÑ7JÔKˆŒÜ# D§M¡M°v×7JÑ7JÔKˆŒØ#Ÿ]™]¨dÐ2ÒQ°f·m±mÀzÑ6QˆÔØ'+×'9Ò'9�T—^’^¸t¿~¹~ÐPT×P]ÑP]Ñ?]ˆ
Ü—i‘i × 2Ñ 2°JÀUÔKˆ�r3   Nr4   Úposition_embeddingsr  Úpast_key_valuesr	  r&   c                 óª  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      }| j	                  |«      j                  |«      }	| j                  |«      j                  |«      }
| j                  |«      j                  dd«      }| j                  |	«      j                  dd«      }	|
j                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t         «      } || ||	|
|f| j"                  sdn| j$                  | j&                  | j(                  dœ|¤Ž\  }} |j*                  g |¢d‘­Ž j-                  «       }t/        j0                  | j3                  |«      j5                  «       «      j7                  |j8                  «      }| j:                  rE  |j                  g |¢d‘| j                  ‘­Ž |j=                  d«      z  j                  g |¢d‘­Ž }n||z  }| j?                  |«      }||fS )Nr7   r    r6   r®   )r  r  r%  ) rD   rn   r   rç   r!  r"  r'  rƒ   r(  rû   Úupdater  r   Úget_interfacerP   Ú_attn_implementationr  r  r  r  r%  r¯   r  r°   Úsoftplusr+  rI   r:   r9   r*  rð   r#  )r/   r4   r-  r  r.  r	  Úinput_shapeÚhidden_shapeÚquery_statesr  r  r…   r†   Úattention_interfacer  r  rÙ   s                    r2   rA   zLagunaAttention.forwardv  sN  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓDˆØ—[‘[ Ó/×4Ñ4°\ÓBˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓDˆà—{‘{ <Ó0×:Ñ:¸1¸aÓ@ˆØ—[‘[ Ó,×6Ñ6°q¸!Ó<ˆ
Ø#×-Ñ-¨a°Ó3ˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆä�z‰z˜$Ÿ+™+ mÓ4×:Ñ:Ó<Ó=×@Ñ@À×ARÑARÓSˆØ×ÒØgÐ+˜;×+Ñ+ÐL¨[ÐL¸"ÐL¸d¿m¹mÒLÈtÏ~É~Ð^`ÓOaÑa×gÑgð ØðØ ò‰Kð &¨Ñ,ˆKà—k‘k +Ó.ˆØ˜LÐ(Ð(r3   rŽ   )rF   rG   rH   rÜ   r!   rs   r)   r+   rJ   rC   r	   r   r   rA   rK   rL   s   @r2   r  r  T  s©   ø„ Ù]ðL˜|ð L¸ð LÈõ LðH )-ñ3)à—|‘|ð3)ð # 5§<¡<°·±Ð#=Ñ>ð3)ð Ÿ™ tÑ+ð	3)ð
  ™ð3)ð Ð-Ñ.ð3)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷3)r3   r  c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
eej                  ej                  f   dz  dee   dej                  fd„Zˆ xZS )ÚLagunaDecoderLayerrP   r  c                 ó˜  •— t         ‰| �  «        |j                  | _        t        |||j                  |   «      | _        |j                  |   dk(  rt        |«      | _        nt        ||j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        y )NÚsparserà   r  )r(   r)   r0   r  Únum_attention_heads_per_layerÚ	self_attnÚmlp_layer_typesrÞ   Úmlpr“   r˜   r$   r&  Úinput_layernormÚpost_attention_layernorm©r/   rP   r  r1   s      €r2   r)   zLagunaDecoderLayer.__init__­  s¡   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ(¨°¸F×<`Ñ<`ÐajÑ<kÓlˆŒØ×!Ñ! )Ñ,°Ò8Ü+¨FÓ3ˆD�Hä  ¸6×;SÑ;SÔTˆDŒHÜ,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔÜ(5°f×6HÑ6HÈf×NaÑNaÔ(bˆÕ%r3   Nr4   r  rˆ   r.  Ú	use_cacher-  r	  r&   c           
      óÀ   — |}| j                  |«      } | j                  d||||||dœ|¤Ž\  }}	||z   }|}| j                  |«      }| j                  |«      }||z   }|S )N)r4   r  rˆ   r.  rC  r-  © )r@  r=  rA  r?  )
r/   r4   r  rˆ   r.  rC  r-  r	  Úresidualrº   s
             r2   rA   zLagunaDecoderLayer.forward¸  s�   € ð !ˆØ×,Ñ,¨]Ó;ˆà)˜4Ÿ>™>ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
Ñˆ�qð ! =Ñ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ  =Ñ0ˆØÐr3   )NNNFN)rF   rG   rH   r!   rs   r)   r+   rJ   Ú
LongTensorr	   ÚboolrC   r   r   rA   rK   rL   s   @r2   r9  r9  ¬  s¾   ø„ ð	c˜|ð 	c¸õ 	cð /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð+Ñ,ðð 
�‰÷r3   r9  c                   ó–   ‡ — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZ eed¬«      eedœZ ej(                  «       ˆ fd	„«       Zˆ xZS )
ÚLagunaPreTrainedModelrP   ÚmodelTr9  r.  r   )Úindex)r·   r4   Ú
attentionsc                 óL  •— t         ‰| �  |«       | j                  j                  }t	        |t
        «      rEt        j                  |j                  d|¬«       t        j                  |j                  d|¬«       n2t	        |t        «      r"t        j                  |j                  d|¬«       t	        |t        «      r4t        j                  j                  j                  |j                  «       y t	        |t         «      r£|j"                  D ]“  }|j$                  }|j&                  |   dk7  rt(        |j&                  |      } ||j                  |¬«      \  }}t        j*                  t-        ||› d�«      |«       t        j*                  t-        ||› d�«      |«       Œ• y y )Nr®   )r=   ÚstdrS   rT   rV   rX   )r(   Ú_init_weightsrP   Úinitializer_ranger€   r¾   ÚinitÚnormal_rÃ   rœ   r¡   r-   r+   r   Úzeros_r©   rN   r_   ra   rR   r   Úcopy_rq   )r/   r  rO  rU   rf   rg   rº   r1   s          €r2   rP  z#LagunaPreTrainedModel._init_weightsë  sC  ø€ ä‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fœmÔ,Ü�L‰L˜×,Ñ,°3¸CÕ@Ü�L‰L˜×)Ñ)°¸Ö=Ü˜Ô 0Ô1Ü�L‰L˜Ÿ™¨S°cÕ:Ü�fÔ.Ô/Ü�H‰H�M‰M× Ñ  ×!?Ñ!?Õ@Ü˜Ô 5Ô6Ø$×0Ñ0ò ^�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ò<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ô#UÑ �˜qÜ—
‘
œ7 6¨j¨\¸Ð+CÓDÀmÔTÜ—
‘
œ7 6¨j¨\Ð9KÐ+LÓMÈ}Õ]ñ^ð 7r3   )rF   rG   rH   r!   r�   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr   r¡   r9  r  Ú_can_record_outputsr+   r‘   rP  rK   rL   s   @r2   rJ  rJ  Ø  s{   ø… àÓØÐØ&*Ð#Ø-Ð.ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðá'Ð(8ÀÔBØ+Ø%ñÐð €U‡]�]ƒ_ó^ó ô^r3   rJ  c                   óò   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   defd„«       «       «       Zˆ xZS )ÚLagunaModelrP   c           	      ó  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                  ¬«      | _        t#        |¬«      | _        d| _        | j)                  «        y c c}w )Nr  ©rP   F)r(   r)   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr0   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersr9  Úlayersr$   r&  ÚnormrN   Ú
rotary_embÚgradient_checkpointingÚ	post_initrB  s      €r2   r)   zLagunaModel.__init__  sÆ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m‘mÜDIÈ&×JbÑJbÓDcÖd°yÔ ¨	Õ2Òdó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÔNˆŒ	Ü/°vÔ>ˆŒØ&+ˆÔ#ð 	�‰Õùò es   ÂDNÚ	input_idsr  rˆ   r.  Úinputs_embedsrC  r	  r&   c           	      ó°  ‡— |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      sP| j                  ||||dœŠˆfd„ˆfd„d	œ}
i }	t        | j                  j                  «      D ]  } |
|   «       |	|<   Œ |}i }t        | j                  j                  «      D ]  }| j                  |||«      ||<   Œ t        | j                   d | j                  j"                   «      D ]G  \  }} ||f|	| j                  j                  |      || j                  j                  |      ||d
œ|¤Ž}ŒI | j%                  |«      }t'        ||r|¬«      S d ¬«      S )Nz:You must specify exactly one of input_ids or inputs_embedsrc  r   r    )ri   )rP   rr  r  r.  rˆ   c                  ó   •— t        di ‰ ¤ŽS ©NrE  )r   ©Úmask_kwargss   €r2   ú<lambda>z%LagunaModel.forward.<locals>.<lambda>6  s   ø€ Ô*<Ñ*K¸{Ñ*K€ r3   c                  ó   •— t        di ‰ ¤ŽS ru  )r   rv  s   €r2   rx  z%LagunaModel.forward.<locals>.<lambda>7  s   ø€ Ô-NÑ-]ÐQ\Ñ-]€ r3   )Úfull_attentionr  )r  r-  rˆ   r.  )Úlast_hidden_stater.  )Ú
ValueErrorrh  r
   rP   Úget_seq_lengthr+   rt   rD   ri   rð   r€   Údictr^   r_   rn  Ú	enumeraterl  rk  rm  r   )r/   rq  r  rˆ   r.  rr  rC  r	  Úpast_seen_tokensÚcausal_mask_mappingÚmask_creation_functionsrU   r4   r-  ÚiÚdecoder_layerrw  s                   @r2   rA   zLagunaModel.forward  s  ø€ ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-ÄÔFàŸ+™+Ø!.Ø"0Ø#2Ø ,ñˆKó #LÛ%]ñ'Ð#ð #%ÐÜ! $§+¡+×"9Ñ"9Ó:ò X�
Ø2UÐ2IÈ*Ñ2UÓ2WÐ# JÒ/ðXð &ˆØ ÐÜ˜dŸk™k×5Ñ5Ó6ò 	gˆJØ.2¯o©o¸mÈ\Ð[eÓ.fÐ 
Ò+ð	gô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÓ Wò 	ÑˆAˆ}Ù)Øðà2°4·;±;×3JÑ3JÈ1Ñ3MÑNØ$7¸¿¹×8OÑ8OÐPQÑ8RÑ$SØ)Ø /ñð ñ‰Mð	ð Ÿ	™	 -Ó0ˆä%Ø+Ù/8˜Oô
ð 	
à>Bô
ð 	
r3   )NNNNNN)rF   rG   rH   r!   r)   r   r   r   r+   rG  rJ   r	   ÚFloatTensorrH  r   r   r   rA   rK   rL   s   @r2   ra  ra     sÍ   ø„ ð˜|õ ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ<
à×#Ñ# dÑ*ð<
ð Ÿ™ tÑ+ð<
ð ×&Ñ&¨Ñ-ð	<
ð
  ™ð<
ð ×(Ñ(¨4Ñ/ð<
ð ˜$‘;ð<
ð Ð+Ñ,ð<
ð 
 ò<
ó ó ó  ô<
r3   ra  Úgate_logitsr¦   c                 ó¾  — | �t        | t        «      syt        | t        «      rC| d   j                  }t        j                  | D �cg c]  }|j                  |«      ‘Œ c}d¬«      }t        j                  j                  j                  d¬«      }t        j                  ||d¬«      \  }}	t        j                  j                  j                  |	|«      }
|€>t        j                  |
j                  «       d¬«      }t        j                  |d¬«      }�n|j                  \  }}|j                  d   ||z  z  }|ddd…dd…ddf   j                  |||||f«      j                  d||«      j                  «      }t        j                   |
j                  «       |z  d¬«      t        j                   |d¬«      z  }|ddd…dd…df   j                  ||||f«      j                  d|«      j                  |«      }t        j                   ||z  d¬«      t        j                   |d¬«      z  }t        j                   ||j#                  d«      z  «      }||z  S c c}w )aÄ  
    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
    experts is too unbalanced.

    Args:
        gate_logits:
            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
            shape [batch_size X sequence_length, num_experts].
        num_experts:
            Number of experts
        top_k:
            The number of experts to route per-token, can be also interpreted as the `top-k` routing
            parameter.
        attention_mask (`torch.Tensor`, *optional*):
            The attention_mask used in forward function
            shape [batch_size X sequence_length] if not None.

    Returns:
        The auxiliary loss.
    Nr   r~   r7   )r€   rC   ri   r+   r„   r:   r   rÊ   r  r´   rË   r=   rI   rD   r   r¯   r¶   rð   )r†  r¦   r¥   r  Úcompute_deviceÚ
layer_gateÚconcatenated_gate_logitsr¼   rº   r»   rÓ   Útokens_per_expertÚrouter_prob_per_expertrè   ré   rk  Úexpert_attention_maskÚ router_per_expert_attention_maskÚoverall_losss                      r2   Úload_balancing_loss_funcr�  T  s9  € ð: Ð¤*¨[¼%Ô"@Øä�+œuÔ%Ø$ Q™×.Ñ.ˆÜ#(§9¡9Ð^iÖ-jÐPZ¨j¯m©m¸NÕ.KÒ-jÐpqÔ#rÐ ä—h‘h×)Ñ)×1Ñ1Ð2JÐPRÐ1ÓS€OäŸ*™* _°eÀÔDÑ€AÐä—(‘(×%Ñ%×-Ñ-Ð.>ÀÓL€KàÐä!ŸJ™J {×'8Ñ'8Ó':ÀÔBÐô "'§¡¨OÀÔ!CÒà&4×&:Ñ&:Ñ#ˆ
�OØ4×:Ñ:¸1Ñ=À*ÈÑB^Ñ_Ðð ˜4¢¢A t¨TÐ1Ñ2ß‰VÐ&¨
°OÀUÈKÐXÓYß‰W�R˜ Ó,ß‰R�Óð	 	ô "ŸI™I k×&7Ñ&7Ó&9Ð<QÑ&QÐWXÔYÔ\a×\eÑ\eØ! qô]
ñ 
Ðð ˜4¢¢A tÐ+Ñ,ß‰VÐ&¨
°OÀ[ÐQÓRß‰W�R˜Ó%ß‰R�Óð	 	)ô "'§¡¨?Ð=]Ñ+]ÐcdÔ!eÔhm×hqÑhqØ,°!ôi
ñ "
Ðô —9‘9Ð.Ð1G×1QÑ1QÐRSÓ1TÑTÓU€LØ˜+Ñ%Ð%ùò[ .ks   ÁIc                   óN  ‡ — e Zd ZddiZddiZddgdgfiZˆ fd„Zee	 	 	 	 	 	 	 	 	 dd	e	j                  dz  d
e	j                  dz  de	j                  dz  dedz  de	j                  dz  de	j                  dz  dedz  dedz  dee	j                  z  dee   defd„«       «       Zˆ xZS )ÚLagunaForCausalLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr4   Úlogitsc                 óN  •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  |j                  d¬«      | _        |j                  | _	        |j                  | _
        |j                  | _        | j                  «        y r•   )r(   r)   ra  rK  rf  r   r™   r0   r“  Úrouter_aux_loss_coefr¦   r¤   rp  r¬   s     €r2   r)   zLagunaForCausalLM.__init__¬  s   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒØ$*×$?Ñ$?ˆÔ!Ø!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ ð 	�‰Õr3   Nrq  r  rˆ   r.  rr  ÚlabelsrC  Úoutput_router_logitsÚlogits_to_keepr	  r&   c
                 ój  — |�|n| j                   j                  } | j                  d|||||||dœ|
¤Ž}|j                  }t	        |	t
        «      rt        |	 d«      n|	}| j                  |dd…|dd…f   «      }d}|� | j                  ||| j                  fi |
¤Ž}d}|rYt        |j                  | j                  | j                  |«      }|�+|| j                  |j                  |j                   «      z  z  }t#        ||||j$                  |j&                  |j(                  |j                  ¬«      S )a­  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N)rq  r  rˆ   r.  rr  rC  r™  )ÚlossÚaux_lossr•  r.  r4   rM  r·   rE  )rP   r™  rK  r{  r€   rs   Úslicer“  Úloss_functionrf  r�  r·   r¦   r¤   r—  r:   ri   r   r.  r4   rM  )r/   rq  r  rˆ   r.  rr  r˜  rC  r™  rš  r	  Úoutputsr4   Úslice_indicesr•  rœ  r�  s                    r2   rA   zLagunaForCausalLM.forward¸  sT  € ð. %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð
 +5¨$¯*©*ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ñ	+
ð ñ	+
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ% f¨f°d·o±oÑPÈÑPˆDàˆÙÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ð!Ø˜×1Ñ1°H·K±KÀÇÁÓ4LÑLÑL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ô
ð 	
r3   )	NNNNNNNNr   )rF   rG   rH   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr)   r   r   r+   rG  rJ   r	   r…  rH  rs   r   r   r   rA   rK   rL   s   @r2   r’  r’  ¦  s5  ø„ à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€Hô
ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ñ@
à×#Ñ# dÑ*ð@
ð Ÿ™ tÑ+ð@
ð ×&Ñ&¨Ñ-ð	@
ð
  ™ð@
ð ×(Ñ(¨4Ñ/ð@
ð × Ñ  4Ñ'ð@
ð ˜$‘;ð@
ð # T™kð@
ð ˜eŸl™lÑ*ð@
ð Ð+Ñ,ð@
ð 
#ò@
ó ó ô@
r3   r’  )r’  ra  rJ  )r    )r®   )Nr6   N)JÚcollections.abcr   Útypingr   r+   Útorch.nn.functionalr   rÊ   r°   Ú r   rR  Úactivationsr   Úcache_utilsr	   r
   Ú
generationr   Úintegrationsr   r   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.genericr   r   r   Úutils.output_capturingr   r   Úconfiguration_lagunar!   ÚModuler$   rN   r“   r¡   r¾   rÞ   rî   rû   rJ   rs   r  rI   r  r  r9  rJ  ra  rC   r�  r’  Ú__all__rE  r3   r2   ú<module>rº     so  ðõ* %Ý ã ß Ð Ý å &Ý !ß .Ý )ß Sß RÝ BÝ 9ß Qß Kß FÝ &ß 5ß [Ñ [ß EÝ .ñ ˜YÓ'ôJ�B—I‘Ió Jó (ðJô(M<˜BŸI™Iô M<ô`�—	‘	ô ô @�r—y‘yô @ð> ô$#�B—I‘Ió $#ó ð$#ôN˜2Ÿ9™9ô ò.(ó#ðL	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(ó%ô2U)�b—i‘iô U)ôp)Ð3ô )ðX ô$^˜Oó $^ó ð$^ðN ôP
Ð'ó P
ó ðP
ðj #Ø
Ø*.ñ	O&Ø—‘  e§l¡lÑ 3Ñ3°dÑ:ðO&à�t‘ðO&ð —L‘L 4Ñ'ð	O&ð
 ‡\�\�CÑóO&ðd ôS
Ð-¨ó S
ó ðS
òl H�r3   