Ë
    ÜÓ^j0k  ã                   óä  — d dl mZ d dlmZ d dlZd dlmc mZ d dlmZ ddl	m
Z
mZ ddlmZ ddlmZmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZmZ ddlmZmZ ddlm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z' ddl(m)Z)m*Z* ddl+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1  e,«       r	d dl2m3Z3m4Z4 nd\  Z3Z4 ed«       G d„ dejj                  «      «       Z6 G d„ dejj                  «      Z7 G d„ dejj                  «      Z8d„ Z9 ed «      d=d!„«       Z:d"ejv                  d#e<d$ejv                  fd%„Z=	 d>d&ejj                  d'ejv                  d(ejv                  d)ejv                  d*ejv                  dz  d+e>d,e>d-e#e%   fd.„Z? ee:«       G d/„ d0ejj                  «      «       Z@d1„ ZAe3e4fZB eCeB«      ZD G d2„ d3ejj                  «      ZE G d4„ d5e«      ZFe& G d6„ d7e!«      «       ZGe& G d8„ d9eG«      «       ZHe& G d:„ d;eGe«      «       ZIg d<¢ZJy)?é    )ÚCallable)ÚOptionalN)Únné   )ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernel_func_from_hubÚuse_kernelized_func)Úforce_accelerate_hooks)Úcreate_causal_maskÚcreate_recurrent_attention_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Úmaybe_autocastÚmerge_with_config_defaults)Úis_causal_conv1d_availableÚis_torchdynamo_compiling)Úcapture_outputsé   )Ú
Lfm2Config)Úcausal_conv1d_fnÚcausal_conv1d_update)NNÚRMSNormc                   óh   ‡ — e Zd Zddeddfˆ fd„Zdej                  dej                  fd„Zd„ Zˆ xZ	S )	ÚLfm2RMSNormÚepsÚreturnNc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)z:
        Lfm2RMSNorm is equivalent to T5LayerNorm
        N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer'   Ú	__class__s      €úq/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/lfm2/modeling_lfm2.pyr+   zLfm2RMSNorm.__init__4   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕó    Úhidden_statesc                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S )Né   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtor-   Úfloat32ÚpowÚmeanÚrsqrtr0   r/   )r1   r6   Úinput_dtypeÚvariances       r4   ÚforwardzLfm2RMSNorm.forward<   sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r5   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)Útupler/   Úshaper0   )r1   s    r4   Ú
extra_reprzLfm2RMSNorm.extra_reprC   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr5   )g�íµ ÷Æ°>)
Ú__name__Ú
__module__Ú__qualname__Úfloatr+   r-   ÚTensorrC   rG   Ú__classcell__©r3   s   @r4   r&   r&   2   s7   ø„ ñ$¨ð $¸$õ $ð; U§\¡\ð ;°e·l±ló ;öJr5   r&   c                   óÄ   ‡ — e Zd ZU ej                  ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d   de
dz  ded	ef   fd
„«       Z ej                  «       ed„ «       «       Zˆ xZS )ÚLfm2RotaryEmbeddingÚinv_freqNÚconfigc                 ó®  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |«      \  }| _
        | j                  d|d¬«       | j                  d|j                  «       d¬«       y )NÚ	rope_typeÚdefaultrQ   F)Ú
persistentÚoriginal_inv_freq)r*   r+   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrR   Úrope_parametersrT   Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)r1   rR   ÚdeviceÚrope_init_fnrQ   r3   s        €r4   r+   zLfm2RotaryEmbedding.__init__J   s¶   ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àˆŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÒ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆ�$Ô(à×Ñ˜Z¨¸eÐÔDØ×ÑÐ0°(·.±.Ó2BÈuÐÕUr5   r`   ztorch.deviceÚseq_lenr(   ztorch.Tensorc                 ó  — | j                   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNg      ð?r   r8   ©r;   )r`   r;   )	r[   Úgetattrr2   Únum_attention_headsr-   ÚarangeÚint64r<   rK   )rR   r`   rb   ÚbaseÚdimÚattention_factorrQ   s          r4   r\   z3Lfm2RotaryEmbedding.compute_default_rope_parametersZ   s�   € ð& ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r5   c                 óN  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  ||fd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j	                  |j                   ¬
«      	j	                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   r9   r    ÚmpsÚcpuF)Údevice_typeÚenabledr8   ©rl   rf   )rQ   rK   ÚexpandrF   r<   r`   Ú
isinstanceÚtypeÚstrr   Ú	transposer-   ÚcatÚcosr]   Úsinr;   )
r1   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrq   ÚfreqsÚembrz   r{   s
             r4   rC   zLfm2RotaryEmbedding.forwardx   sR  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   ÃBFÆF$©N©NNN)rH   rI   rJ   r-   rL   Ú__annotations__r!   r+   Ústaticmethodr   ÚintrE   rK   r\   Úno_gradr   rC   rM   rN   s   @r4   rP   rP   G   s˜   ø… Ø�l‰lÓñV˜zõ Vð  à$(Ø+/Ø"ñ*Ø˜TÑ!ð*à˜Ñ(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ñ	%ò	*ó ð*ð: €U‡]�]ƒ_Øñ<ó ó ô<r5   rP   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚLfm2MLPrR   c                 óþ  •— t         ‰| �  «        |j                  }|j                  rat	        d|z  dz  «      }|j
                  �Dt	        |j
                  |z  «      }|j                  ||j                  z   dz
  |j                  z  z  }t        j                  |j                  |d¬«      | _
        t        j                  |j                  |d¬«      | _        t        j                  ||j                  d¬«      | _        y )Nr8   r   r    F©Úbias)r*   r+   Úintermediate_sizeÚblock_auto_adjust_ff_dimr†   Úblock_ffn_dim_multiplierÚblock_multiple_ofr   ÚLinearr2   Úw1Úw3Úw2)r1   rR   r�   r3   s      €r4   r+   zLfm2MLP.__init__‰   sÞ   ø€ Ü‰ÑÔØ"×4Ñ4ÐØ×*Ò*Ü # AÐ(9Ñ$9¸AÑ$=Ó >Ðà×.Ñ.Ð:Ü$'¨×(GÑ(GÐJ[Ñ([Ó$\Ð!Ø$*×$<Ñ$<Ø&¨×)AÑ)AÑAÀAÑEÈ&×JbÑJbÑbñ%Ð!ô —)‘)˜F×.Ñ.Ð0AÈÔNˆŒÜ—)‘)˜F×.Ñ.Ð0AÈÔNˆŒÜ—)‘)Ð-¨v×/AÑ/AÈÔNˆ�r5   c                 óŒ   — | j                  t        j                  | j                  |«      «      | j	                  |«      z  «      S r‚   )r”   ÚFÚsilur’   r“   )r1   r|   s     r4   rC   zLfm2MLP.forward˜   s/   € Ø�w‰w”q—v‘v˜dŸg™g a›jÓ)¨D¯G©G°A«JÑ6Ó7Ð7r5   )rH   rI   rJ   r!   r+   rC   rM   rN   s   @r4   r‰   r‰   ˆ   s   ø„ ðO˜zõ Oö8r5   r‰   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Nr9   r8   rs   )rF   r-   ry   )r|   Úx1Úx2s      r4   Úrotate_halfr›   œ   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r5   Úrotary_pos_embc                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer›   )ÚqÚkrz   r{   Úunsqueeze_dimÚq_embedÚk_embeds          r4   Úapply_rotary_pos_embr¤   £   sY   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐr5   r6   Ún_repr(   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r    N)rF   rt   Úreshape)r6   r¥   ÚbatchÚnum_key_value_headsÚslenre   s         r4   Ú	repeat_kvr«   ½   so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr5   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t        || j                  «      }t        || j                  «      }	t        j                  ||j	                  dd«      «      |z  }
|�|
|z   }
t
        j                  j                  |
dt        j                  ¬«      j                  |j                  «      }
t
        j                  j                  |
|| j                  ¬«      }
t        j                  |
|	«      }|j	                  dd«      j                  «       }||
fS )Nr8   r   r9   )rl   r;   )ÚpÚtrainingr    )r«   Únum_key_value_groupsr-   Úmatmulrx   r   Ú
functionalÚsoftmaxr=   r<   r;   r²   r¶   Ú
contiguous)r¬   r­   r®   r¯   r°   r±   r²   r³   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r4   Úeager_attention_forwardrÀ   É   sà   € ô ˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r5   c                   óê   ‡ — e Zd ZdZdedefˆ fd„Z	 ddej                  de	ej                  ej                  f   dej                  dz  d	e
dz  d
e	ej                  ej                  dz  f   f
d„Zˆ xZS )ÚLfm2Attentionz=Multi-headed attention from 'Attention Is All You Need' paperrR   Ú	layer_idxc                 óŠ  •— t         ‰| �  «        || _        || _        t	        |d|j
                  |j                  z  «      | _        |j                  |j                  z  | _	        | j                  dz  | _
        d| _        t        j                  |j
                  |j                  | j                  z  d¬«      | _        t        j                  |j
                  |j                  | j                  z  d¬«      | _        t        j                  |j
                  |j                  | j                  z  d¬«      | _        t        j                  |j                  | j                  z  |j
                  d¬«      | _        t%        | j                  |j&                  ¬«      | _        t%        | j                  |j&                  ¬«      | _        y )Nre   g      à¿TFr‹   ©r'   )r*   r+   rR   rÃ   rg   r2   rh   re   r©   r·   r±   Ú	is_causalr   r‘   Úq_projÚk_projÚv_projÚout_projr&   Únorm_epsÚq_layernormÚk_layernorm©r1   rR   rÃ   r3   s      €r4   r+   zLfm2Attention.__init__æ   sL  ø€ Ü‰ÑÔØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÔkˆŒÜŸ	™	 &×"<Ñ"<¸t¿}¹}Ñ"LÈf×N`ÑN`ÐglÔmˆŒÜ& t§}¡}¸&¿/¹/ÔJˆÔÜ& t§}¡}¸&¿/¹/ÔJˆÕr5   Nr6   Úposition_embeddingsr°   Úpast_key_valuesr(   c                 ó
  — |j                   d d }g |¢d‘| j                  ‘­}| j                   | j                  |«      j                  |Ž «      j                  dd«      }| j                   | j                  |«      j                  |Ž «      j                  dd«      }	 | j                  |«      j                  |Ž j                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t         «      } || ||	|
|fd| j"                  dœ|¤Ž\  }} |j$                  g |¢d‘­Ž j'                  «       }| j)                  |«      }||fS )Nr9   r    r8   ç        )r²   r±   )rF   re   rÌ   rÇ   Úviewrx   rÍ   rÈ   rÉ   r¤   ÚupdaterÃ   r   Úget_interfacerR   Ú_attn_implementationrÀ   r±   r§   r»   rÊ   )r1   r6   rÏ   r°   rÐ   r³   Úinput_shapeÚhidden_shapeÚquery_statesr¼   r½   rz   r{   Úattention_interfacer¿   r¾   Úoutputs                    r4   rC   zLfm2Attention.forwardõ   s¤  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà×'Ñ'Ð(G¨¯©°MÓ(B×(GÑ(GÈÐ(VÓW×aÑaÐbcÐefÓgˆØ×%Ñ%Ð&E d§k¡k°-Ó&@×&EÑ&EÀ|Ð&TÓU×_Ñ_Ð`aÐcdÓeˆ
Ø6�t—{‘{ =Ó1×6Ñ6¸ÐE×OÑOÐPQÐSTÓUˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð Ø—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—‘˜{Ó+ˆØ�|Ð#Ð#r5   r‚   )rH   rI   rJ   Ú__doc__r!   r†   r+   r-   rL   rE   r   rC   rM   rN   s   @r4   rÂ   rÂ   â   s�   ø„ áGðK˜zð K°cõ Kð( )-ñ%$à—|‘|ð%$ð # 5§<¡<°·±Ð#=Ñ>ð%$ð Ÿ™ tÑ+ð	%$ð
  ™ð%$ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷%$r5   rÂ   c                 ó¦   — |�N|j                   d   dkD  r<|j                   d   dkD  r*| j                  }| |dd…dd…df   z  j                  |«      } | S )zm
    Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
    Nr    r   )rF   r;   r<   )r6   r°   r;   s      r4   Úapply_mask_to_padding_statesrÞ     sa   € ð
 Ð! n×&:Ñ&:¸1Ñ&=ÀÒ&AÀn×FZÑFZÐ[\ÑF]Ð`aÒFaØ×#Ñ#ˆØ&¨ºº1¸d¸
Ñ)CÑC×GÑGÈÓNˆàÐr5   c                   óˆ  ‡ — e Zd Zdedefˆ fd„Z	 	 	 ddej                  dedz  dej                  dz  dej                  dz  fd	„Z
	 	 	 ddej                  dedz  dej                  dz  dej                  dz  fd
„Z ed«      	 	 	 ddej                  dedz  dej                  dz  dej                  dz  fd„«       Zˆ xZS )ÚLfm2ShortConvrR   rÃ   c           	      óZ  •— t         ‰| �  «        || _        || _        |j                  | _        |j                  | _        t        j                  |j                  |j                  | j
                  |j                  | j                  | j
                  dz
  ¬«      | _        t        j                  |j                  d|j                  z  | j                  ¬«      | _        t        j                  |j                  |j                  | j                  ¬«      | _        |j                  |   | _        y )Nr    )Úin_channelsÚout_channelsÚkernel_sizeÚgroupsrŒ   Úpaddingr   r‹   )r*   r+   rR   rÃ   Úconv_L_cacheÚL_cacheÚ	conv_biasrŒ   r   ÚConv1dr2   Úconvr‘   Úin_projrÊ   Úlayer_typesÚ
layer_typerÎ   s      €r4   r+   zLfm2ShortConv.__init__.  sá   ø€ ô
 	‰ÑÔØˆŒØ"ˆŒØ×*Ñ*ˆŒØ×$Ñ$ˆŒ	ä—I‘IØ×*Ñ*Ø×+Ñ+ØŸ™Ø×%Ñ%Ø—‘Ø—L‘L 1Ñ$ô
ˆŒ	ô —y‘y ×!3Ñ!3°Q¸×9KÑ9KÑ5KÐRV×R[ÑR[Ô\ˆŒÜŸ	™	 &×"4Ñ"4°f×6HÑ6HÈtÏyÉyÔYˆŒà ×,Ñ,¨YÑ7ˆ�r5   Nr|   rÐ   r°   Úseq_idxc                 ó  — t        ||«      }| j                  |«      j                  dd«      }|j                  dd¬«      \  }}}||z  }| j                  j
                  j                  | j                  j
                  j                  d«      | j                  j
                  j                  d«      «      }	|�ƒ|j                  | j                  «      rht        |j                  d«      |j                  | j                     j                  d   |	| j                  j                  d «      }
|
j                  d«      }
n�|�jt         j"                  j%                  || j&                  |j(                  d   z
  df«      }|j+                  || j                  «      d| j&                   d …f   }t-        ||	| j                  j                  d |¬«      }
||
z  }| j/                  |j                  dd«      j1                  «       «      }|S )	Nr9   éþÿÿÿr   rs   r   r8   .)Ú
activationrï   )rÞ   rì   rx   Úchunkrë   r/   rÓ   ÚsizeÚhas_previous_staterÃ   r#   ÚsqueezeÚlayersÚconv_statesrŒ   rž   r   r¹   Úpadrè   rF   Úupdate_conv_stater"   rÊ   r»   )r1   r|   rÐ   r°   rï   ÚBCxÚBÚCÚBxÚconv_weightsÚconv_outÚ
conv_stateÚys                r4   Úcuda_kernels_forwardz"Lfm2ShortConv.cuda_kernels_forwardF  s­  € ô )¨¨NÓ;ˆØ�l‰l˜1‹o×'Ñ'¨¨BÓ/ˆØ—)‘)˜A 2�)Ó&‰ˆˆ1ˆaà�‰Uˆà—y‘y×'Ñ'×,Ñ,¨T¯Y©Y×-=Ñ-=×-BÑ-BÀ1Ó-EÀtÇyÁy×GWÑGW×G\ÑG\Ð]^ÓG_Ó`ˆØÐ&¨?×+MÑ+MÈdÏnÉnÔ+]Ü+Ø—
‘
˜2“Ø×&Ñ& t§~¡~Ñ6×BÑBÀ1ÑEØØ—	‘	—‘ØóˆHð  ×)Ñ)¨"Ó-‰HàÐ*ÜŸ]™]×.Ñ.¨r°D·L±LÀ2Ç8Á8ÈBÁ<Ñ4OÐQRÐ3SÓT�
Ø,×>Ñ>¸zÈ4Ï>É>ÓZÐ[^Ðae×amÑamÐ`mÑ`oÐ[oÑp�
ô (¨¨L¸$¿)¹)¿.¹.ÐUYÐcjÔkˆHà�‰LˆØ�M‰M˜!Ÿ+™+ b¨"Ó-×8Ñ8Ó:Ó;ˆØˆr5   c           
      óÆ  — |j                   d   }t        ||«      }| j                  |«      j                  dd«      }|j	                  dd¬«      \  }}}||z  }	|�Ò|j                  | j                  «      r·|j                  |	| j                  «      d| j                   d …f   }
t        j                  |
j                  |	j                  «      | j                  j                  d d …dd d …f   z  d¬«      }| j                  r|| j                  j                  z  }|j!                  d«      }�nþ|��w|j                   d   dk(  �rd|�jt"        j$                  j'                  |	| j                  |	j                   d   z
  df«      }
|j                  |
| j                  «      d| j                   d …f   }
|d   }|dd  |d d k7  j)                  d¬	«      d   dz   }t        j*                  |j-                  d«      ||j/                  d
|j1                  «       «      g«      j3                  «       }g }t5        t7        |«      dz
  «      D ]K  }||   ||dz      }}||kD  sŒ|j9                  | j                  |	d d …d d …||…f   «      dd ||z
  …f   «       ŒM t        j*                  |d¬«      }n„|�jt"        j$                  j'                  |	| j                  |	j                   d   z
  df«      }
|j                  |
| j                  «      d| j                   d …f   }
| j                  |	«      dd |…f   }||z  }|j                  dd«      j;                  «       }| j=                  |«      }|S )Nr    r9   rñ   r   rs   .r   T)Úas_tuple©r    )rF   rÞ   rì   rx   ró   rõ   rÃ   rú   rè   r-   Úsumr<   r`   rë   r/   rŒ   rž   r   r¹   rù   Únonzerory   Ú	new_zerosÚnew_fullÚnumelÚtolistÚrangeÚlenÚappendr»   rÊ   )r1   r|   rÐ   r°   rï   Úseqlenrû   rü   rý   rþ   r  r   ÚsiÚchangeÚboundsÚpartsÚiÚsÚer  s                       r4   Úslow_forwardzLfm2ShortConv.slow_forwardi  s  € ð —‘˜‘ˆä(¨¨NÓ;ˆØ�l‰l˜1‹o×'Ñ'¨¨BÓ/ˆØ—)‘)˜A 2�)Ó&‰ˆˆ1ˆaà�‰UˆàÐ&¨?×+MÑ+MÈdÏnÉnÔ+]Ø(×:Ñ:¸2¸t¿~¹~ÓNÈsÐUY×UaÑUaÐTaÑTcÐOcÑdˆJÜ—y‘y §¡¨r¯y©yÓ!9¸D¿I¹I×<LÑ<LÊQÐPQÒSTÈWÑ<UÑ!UÐ[]Ô^ˆHØ�yŠyØ˜DŸI™IŸN™NÑ*�à×)Ñ)¨"Ó-ŠHØÑ  Q§W¡W¨Q¡Z°1£_àÐ*ÜŸ]™]×.Ñ.¨r°D·L±LÀ2Ç8Á8ÈBÁ<Ñ4OÐQRÐ3SÓT�
Ø,×>Ñ>¸zÈ4Ï>É>ÓZÐ[^Ðae×amÑamÐ`mÑ`oÐ[oÑp�
Ø˜‘ˆBØ˜˜�f  3 B Ñ'×0Ñ0¸$Ð0Ó?ÀÑBÀQÑFˆFÜ—Y‘Y × 0Ñ 0°Ó 3°V¸V¿_¹_ÈTÐSU×S[ÑS[ÓS]Ó=^Ð_Ó`×gÑgÓiˆFØˆEÜœ3˜v›;¨™?Ó+ò I�Ø˜a‘y &¨¨Q©¡-�1�Ø�q“5Ø—L‘L §¡¨2ªa²°A°a°C¨i©=Ó!9¸#¸wÀÀQÁ¸w¸,Ñ!GÕHðIô —y‘y ¨BÔ/‰HàÐ*ÜŸ]™]×.Ñ.¨r°D·L±LÀ2Ç8Á8ÈBÁ<Ñ4OÐQRÐ3SÓT�
Ø,×>Ñ>¸zÈ4Ï>É>ÓZÐ[^Ðae×amÑamÐ`mÑ`oÐ[oÑp�
à—y‘y “} S¨'¨6¨' \Ñ2ˆHà�‰LˆØ�K‰K˜˜BÓ×*Ñ*Ó,ˆØ�M‰M˜!ÓˆØˆr5   rë   r6   c                 ó¦   — t         r7d|j                  j                  v rt        «       s| j	                  ||||¬«      S | j                  ||||¬«      S )NÚcuda)rï   )Úis_fast_path_availabler`   rv   r   r  r  )r1   r6   rÐ   r°   rï   s        r4   rC   zLfm2ShortConv.forward™  sV   € õ " f°×0DÑ0D×0IÑ0IÑ&IÔRjÔRlØ×,Ñ,¨]¸OÈ^ÐelÐ,ÓmÐmØ× Ñ  °ÀÐY`Ð ÓaÐar5   rƒ   )rH   rI   rJ   r!   r†   r+   r-   rL   r   Ú	IntTensorr  r  r   rC   rM   rN   s   @r4   rà   rà   -  s&  ø„ ð8àð8ð õ8ð6 )-Ø.2Ø*.ñ!à�<‰<ð!ð  ™ð!ð Ÿ™ tÑ+ð	!ð
 —‘ 4Ñ'ó!ðL )-Ø.2Ø*.ñ.à�<‰<ð.ð  ™ð.ð Ÿ™ tÑ+ð	.ð
 —‘ 4Ñ'ó.ñ` ˜FÓ#ð )-Ø.2Ø*.ñ	bà—|‘|ð	bð  ™ð	bð Ÿ™ tÑ+ð		bð
 —‘ 4Ñ'ò	bó $ô	br5   rà   c                   óì   ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	e
dz  d
ej                  fd„Zˆ xZS )ÚLfm2DecoderLayerrR   rÃ   c                 óf  •— t         ‰| �  «        |j                  |   dk(  | _        | j                  rt	        ||«      | _        nt        ||«      | _        t        |«      | _	        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        y )NÚfull_attentionrÅ   )r*   r+   rí   Úis_attention_layerrÂ   Ú	self_attnrà   rë   r‰   Úfeed_forwardr&   r2   rË   Úoperator_normÚffn_normrÎ   s      €r4   r+   zLfm2DecoderLayer.__init__§  s‰   ø€ Ü‰ÑÔØ"(×"4Ñ"4°YÑ"?ÐCSÑ"SˆÔà×"Ò"Ü*¨6°9Ó=ˆD�Nä% f¨iÓ8ˆDŒIÜ# F›OˆÔÜ(¨×);Ñ);ÀÇÁÔQˆÔÜ# F×$6Ñ$6¸F¿O¹OÔLˆ�r5   Nr6   rÏ   r°   r}   rÐ   r(   c           	      ó.  — |}| j                   r+ | j                  d| j                  |«      ||||dœ|¤Ž\  }}n3| j                  | j                  |«      |||j	                  d«      ¬«      }||z   }|| j                  | j                  |«      «      z   }|S )N)r6   rÏ   r°   r}   rÐ   rï   )r6   rÐ   r°   rï   © )r!  r"  r$  rë   Úgetr#  r%  )	r1   r6   rÏ   r°   r}   rÐ   r³   ÚresidualÚ_s	            r4   rC   zLfm2DecoderLayer.forward³  sµ   € ð !ˆØ×"Ò"Ø-˜tŸ~™~ð  Ø"×0Ñ0°Ó?Ø$7Ø-Ø)Ø /ñ ð ñ ÑˆM™1ð !ŸI™IØ"×0Ñ0°Ó?Ø /Ø-ØŸ
™
 9Ó-ð	 &ó ˆMð &¨Ñ0ˆØ%¨×(9Ñ(9¸$¿-¹-ÈÓ:VÓ(WÑWˆàÐr5   )NNNN)rH   rI   rJ   r!   r†   r+   r-   rL   rE   Ú
LongTensorr   rC   rM   rN   s   @r4   r  r  ¦  sŸ   ø„ ð
M˜zð 
M°cõ 
Mð IMØ.2Ø04Ø(,ñà—|‘|ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ÿ™ tÑ+ð	ð
 ×&Ñ&¨Ñ-ðð  ™ðð 
�‰÷r5   r  c                   óN   — e Zd ZU eed<   dZdZdgZdgZdZ	dZ
dZdZdZeedœZdZy)ÚLfm2PreTrainedModelrR   ÚmodelTr  rÐ   )r6   Ú
attentionsN)rH   rI   rJ   r!   r„   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr  rÂ   Ú_can_record_outputsÚ_is_statefulr'  r5   r4   r-  r-  Ó  sX   … àÓØÐØ&*Ð#Ø+Ð,ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà)Ø#ñÐð �Lr5   r-  c                   óò   ‡ — e Zd Zdefˆ fd„Zeee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  d	edz  d
ee   defd„«       «       «       Zˆ xZS )Ú	Lfm2ModelrR   c           	      ó  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |¬«      | _        d| _        t#        |j                  |j$                  ¬«      | _        | j)                  «        y c c}w )N©rR   FrÅ   )r*   r+   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr2   Úembed_tokensÚ
ModuleListr  Únum_hidden_layersr  r÷   rP   Ú
rotary_embÚgradient_checkpointingr&   rË   Úembedding_normÚ	post_initrÎ   s      €r4   r+   zLfm2Model.__init__é  sÅ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m‘mÜBGÈ×H`ÑH`ÓBaÖb°YÔ˜f iÕ0Òbó
ˆŒô .°VÔ<ˆŒØ&+ˆÔ#Ü)¨&×*<Ñ*<À&Ç/Á/ÔRˆÔð 	�‰Õùò cs   ÂDNÚ	input_idsr°   r}   rÐ   Úinputs_embedsÚ	use_cacher³   r(   c           	      óÈ  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|r|€t        | j                  ¬«      }|€V|�|j	                  «       nd}t        j                  |j                  d   |j                  ¬«      |z   }|j                  d«      }t        |x}	t        «      s)| j                  ||||dœ}
t        di |
¤Žt        di |
¤Ždœ}	|}| j                  ||¬«      }t        | j                   d | j                  j"                   «      D ]-  \  }} ||f|	| j                  j$                  |      |||d	œ|¤Ž}Œ/ | j'                  |«      }t)        ||¬
«      S )Nz:You must specify exactly one of input_ids or inputs_embedsr>  r   r    )r`   )rR   rK  r°   rÐ   r}   )r   rë   )r}   )r°   rÏ   r}   rÐ   )Úlast_hidden_staterÐ   r'  )Ú
ValueErrorrC  r   rR   Úget_seq_lengthr-   ri   rF   r`   rž   ru   Údictr   r   rF  Ú	enumerater÷   rE  rí   rH  r   )r1   rJ  r°   r}   rÐ   rK  rL  r³   Úpast_seen_tokensÚcausal_mask_mappingÚmask_kwargsr6   rÏ   r  Údecoder_layers                  r4   rC   zLfm2Model.forwardù  s˜  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*°$·+±+Ô>ˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-ÄÔFàŸ+™+Ø!.Ø"0Ø#2Ø ,ñˆKô #5Ñ"C°{Ñ"CÜ7ÑF¸+ÑFñ#Ðð
 &ˆØ"Ÿo™o¨mÈ,˜oÓWÐô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÓ Wò 	ÑˆAˆ}Ù)Øðà2°4·;±;×3JÑ3JÈ1Ñ3MÑNØ$7Ø)Ø /ñð ñ‰Mð	ð ×+Ñ+¨MÓ:ˆä&Ø+Ø+ô
ð 	
r5   )NNNNNN)rH   rI   rJ   r!   r+   r   r   r   r-   r+  rL   r   ÚFloatTensorÚboolr   r   r   rC   rM   rN   s   @r4   r<  r<  ç  sÍ   ø„ ð˜zõ ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ8
à×#Ñ# dÑ*ð8
ð Ÿ™ tÑ+ð8
ð ×&Ñ&¨Ñ-ð	8
ð
  ™ð8
ð ×(Ñ(¨4Ñ/ð8
ð ˜$‘;ð8
ð Ð+Ñ,ð8
ð 
!ò8
ó ó ó  ô8
r5   r<  c                   óB  ‡ — e Zd ZddiZddiZddgdgfiZˆ fd„Zee	 	 	 	 	 	 	 	 dd	e	j                  dz  d
e	j                  dz  de	j                  dz  dedz  de	j                  dz  de	j                  dz  dedz  dee	j                  z  dee   defd„«       «       Zˆ xZS )ÚLfm2ForCausalLMzlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr6   Úlogitsc                 óè   •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  |j                  d¬«      | _        | j                  «        y )NFr‹   )
r*   r+   r<  r.  rA  r   r‘   r2   r[  rI  )r1   rR   r3   s     €r4   r+   zLfm2ForCausalLM.__init__=  sU   ø€ Ü‰Ñ˜Ô Ü˜vÓ&ˆŒ
Ø ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Õr5   NrJ  r°   r}   rÐ   rK  ÚlabelsrL  Úlogits_to_keepr³   r(   c	           
      óx  —  | j                   d||||||dœ|	¤Ž}
|
j                  }t        |t        «      rt	        | d«      n|}| j                  |dd…|dd…f   «      }d}|�* | j                  d||| j                  j                  dœ|	¤Ž}t        |||
j                  |
j                  |
j                  ¬«      S )aÉ  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, Lfm2ForCausalLM

        >>> model = Lfm2ForCausalLM.from_pretrained("meta-lfm2/Lfm2-2-7b-hf")
        >>> tokenizer = AutoTokenizer.from_pretrained("meta-lfm2/Lfm2-2-7b-hf")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```)rJ  r°   r}   rÐ   rK  rL  N)r]  r_  rA  )Úlossr]  rÐ   r6   r/  r'  )r.  rN  ru   r†   Úslicer[  Úloss_functionrR   rA  r   rÐ   r6   r/  )r1   rJ  r°   r}   rÐ   rK  r_  rL  r`  r³   Úoutputsr6   Úslice_indicesr]  rb  s                  r4   rC   zLfm2ForCausalLM.forwardF  sÙ   € ð> ,6¨4¯:©:ð ,
ØØ)Ø%Ø+Ø'Øñ,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ%Ðp¨V¸FÈtÏ{É{×OeÑOeÑpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r5   )NNNNNNNr   )rH   rI   rJ   Ú_tied_weights_keysÚ_tp_planÚ_pp_planr+   r   r   r-   r+  rL   r   rW  rX  r†   r   r   r   rC   rM   rN   s   @r4   rZ  rZ  7  s  ø„ à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€Hôð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ6
à×#Ñ# dÑ*ð6
ð Ÿ™ tÑ+ð6
ð ×&Ñ&¨Ñ-ð	6
ð
  ™ð6
ð ×(Ñ(¨4Ñ/ð6
ð × Ñ  4Ñ'ð6
ð ˜$‘;ð6
ð ˜eŸl™lÑ*ð6
ð Ð+Ñ,ð6
ð 
 ò6
ó ó ô6
r5   rZ  )rZ  r<  r-  r  )rÒ   )KÚcollections.abcr   Útypingr   r-   Útorch.nn.functionalr   r¹   r–   Úcache_utilsr   r   Ú
generationr	   Úintegrationsr
   r   r   Úintegrations.accelerater   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   r   Úutils.import_utilsr   r   Úutils.output_capturingr   Úconfiguration_lfm2r!   Úcausal_conv1dr"   r#   ÚModuler&   rP   r‰   r›   r¤   rL   r†   r«   rK   rÀ   rÂ   rÞ   Úkernel_modulesÚallr  rà   r  r-  r<  rZ  Ú__all__r'  r5   r4   ú<module>r�     s3  ðõ( %Ý ã ß Ð Ý ç .Ý )ß fÑ fÝ =ß PÝ 9ß Oß Kß FÝ &ß IÑ Iß Gß VÝ 5Ý *ñ ÔßDÐDà-7Ñ*ÐÐ*ñ ˜YÓ'ôJ�"—)‘)ó Jó (ðJô(><˜"Ÿ)™)ô ><ôB8ˆb�i‰iô 8ò((ñ Ð*Ó+òó ,ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(ó%ñ2 Ð)Ó*ô7$�B—I‘Ió 7$ó +ð7$òt	ð #Ð$8Ð9€Ù˜^Ó,Ð ôvb�B—I‘Iô vbôr*Ð1ô *ðZ ô˜/ó ó ðð& ôL
Ð#ó L
ó ðL
ð^ ôF
Ð)¨?ó F
ó ðF
òR B�r5   