Ë
    ÜÓ^j'¤  ã                   óì  — d dl mZ d dlmZ d dlmZ d dlZd dlmZ d dlm	Z	 ddl
mZ ddlmZmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZmZmZmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z* ddl+m,Z,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2  e)d¬«      e G d„ de«      «       «       Z3 G d„ dejh                  «      Z5 G d„ dejh                  «      Z6 G d„ dejh                  «      Z7d ejp                  d!e9d"ejp                  fd#„Z:	 dEd$ejh                  d%ejp                  d&ejp                  d'ejp                  d(ejp                  dz  d)e;d*e;d+e&e(   fd,„Z<d-„ Z=dFd.„Z> G d/„ d0ejh                  «      Z? G d1„ d2e«      Z@ G d3„ d4e«      ZAe) G d5„ d6e$«      «       ZB G d7„ d8eB«      ZCe) G d9„ d:eB«      «       ZDe) G d;„ d<eB«      «       ZEd=ejp                  d>e9d?e9fd@„ZF e)dA¬«       G dB„ dCeBe«      «       ZGg dD¢ZHy)Gé    )ÚCallable)Ú	dataclass)ÚOptionalN)ÚCrossEntropyLossé   )ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPastÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚMoonshineConfigz™
    Extends [~modeling_outputs.BaseModelOutput] to include the output attention mask since sequence length is not preserved in the model's forward.
    )Úcustom_introc                   ó:   — e Zd ZU dZdZej                  dz  ed<   y)ÚMoonshineEncoderModelOutputa–  
    attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to avoid performing attention on padding token indices after sequence compression. Returned because the
        sequence length may differ from the input sequence length. Mask values selected in `[0, 1]`:

        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.
    NÚattention_mask)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r'   ÚtorchÚTensorÚ__annotations__© ó    ú{/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/moonshine/modeling_moonshine.pyr&   r&   3   s   … ñð +/€N�E—L‘L 4Ñ'Ô.r0   r&   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚMoonshineEncoderMLPc                 ó  •— t         ‰| �  «        || _        t        |   | _        t        j                  |j                  |j                  «      | _	        t        j                  |j                  |j                  «      | _
        y ©N©ÚsuperÚ__init__Úconfigr   Úactivation_fnÚnnÚLinearÚhidden_sizeÚintermediate_sizeÚfc1Úfc2©Úselfr9   Ú
hidden_actÚ	__class__s      €r1   r8   zMoonshineEncoderMLP.__init__G   s^   ø€ Ü‰ÑÔØˆŒÜ# JÑ/ˆÔÜ—9‘9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9‘9˜V×5Ñ5°v×7IÑ7IÓJˆ�r0   Úhidden_statesÚreturnc                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r5   )r?   r:   r@   )rB   rE   s     r1   ÚforwardzMoonshineEncoderMLP.forwardN   s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐr0   ©r(   r)   r*   r8   r,   r-   rH   Ú__classcell__©rD   s   @r1   r3   r3   F   s$   ø„ ôKð U§\¡\ð °e·l±l÷ r0   r3   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚMoonshineDecoderMLPc                 ó  •— t         ‰| �  «        || _        t        |   | _        t        j                  |j                  |j                  dz  «      | _	        t        j                  |j                  |j                  «      | _
        y )Né   r6   rA   s      €r1   r8   zMoonshineDecoderMLP.__init__V   sc   ø€ Ü‰ÑÔØˆŒÜ# JÑ/ˆÔÜ—9‘9˜V×/Ñ/°×1IÑ1IÈAÑ1MÓNˆŒÜ—9‘9˜V×5Ñ5°v×7IÑ7IÓJˆ�r0   rE   rF   c                 óž   — | j                  |«      }|j                  dd¬«      \  }}| j                  |«      |z  }| j                  |«      }|S )NrO   éÿÿÿÿ©Údim)r?   Úchunkr:   r@   )rB   rE   Úgates      r1   rH   zMoonshineDecoderMLP.forward]   sS   € ØŸ™ Ó/ˆØ+×1Ñ1°!¸Ð1Ó<Ñˆ�tØ×*Ñ*¨4Ó0°=Ñ@ˆØŸ™ Ó/ˆØÐr0   rI   rK   s   @r1   rM   rM   U   s$   ø„ ôKð U§\¡\ð °e·l±l÷ r0   rM   c                   óÄ   ‡ — e Zd ZU ej                  ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d   de
dz  ded	ef   fd
„«       Z ej                  «       ed„ «       «       Zˆ xZS )ÚMoonshineRotaryEmbeddingÚinv_freqNr9   c                 ó®  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |«      \  }| _
        | j                  d|d¬«       | j                  d|j                  «       d¬«       y )NÚ	rope_typeÚdefaultrX   F)Ú
persistentÚoriginal_inv_freq)r7   r8   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr9   Úrope_parametersrZ   Úcompute_default_rope_parametersr   Úattention_scalingÚregister_bufferÚclone)rB   r9   ÚdeviceÚrope_init_fnrX   rD   s        €r1   r8   z!MoonshineRotaryEmbedding.__init__h   s¶   ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àˆŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÒ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆ�$Ô(à×Ñ˜Z¨¸eÐÔDØ×ÑÐ0°(·.±.Ó2BÈuÐÕUr0   rf   ztorch.deviceÚseq_lenrF   ztorch.Tensorc                 ón  — | j                   d   }| j                   j                  dd«      }t        | dd«      xs | j                  | j                  z  }t        ||z  «      }d}d|t        j                  d|dt        j                  ¬«      j                  |t        j                  ¬	«      |z  z  z  }||fS )
a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimNr   rO   ©Údtype)rf   rn   )ra   ÚgetÚgetattrr=   Únum_attention_headsÚintr,   ÚarangeÚint64ÚtoÚfloat)	r9   rf   rh   Úbaserk   rl   rS   Úattention_factorrX   s	            r1   rb   z8MoonshineRotaryEmbedding.compute_default_rope_parametersx   s»   € ð& ×%Ñ% lÑ3ˆØ &× 6Ñ 6× :Ñ :Ð;RÐTWÓ XÐÜ˜6 :¨tÓ4Òh¸×8JÑ8JÈf×NhÑNhÑ8hˆÜ�(Ð2Ñ2Ó3ˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)r0   c                 óN  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  ||fd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j	                  |j                   ¬
«      	j	                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   rQ   r"   ÚmpsÚcpuF)Údevice_typeÚenabledrO   rR   rm   )rX   rv   ÚexpandÚshaperu   rf   Ú
isinstanceÚtypeÚstrr   Ú	transposer,   ÚcatÚcosrc   Úsinrn   )
rB   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr|   ÚfreqsÚembr…   r†   s
             r1   rH   z MoonshineRotaryEmbedding.forward˜   sR  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   ÃBFÆF$r5   )NNN)r(   r)   r*   r,   r-   r.   r#   r8   Ústaticmethodr   rr   Útuplerv   rb   Úno_gradr   rH   rJ   rK   s   @r1   rW   rW   e   s˜   ø… Ø�l‰lÓñV˜õ Vð  à)-Ø+/Ø"ñ*Ø $Ñ&ð*à˜Ñ(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ñ	%ò	*ó ð*ð> €U‡]�]ƒ_Øñ<ó ó ô<r0   rW   rE   Ún_reprF   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r"   N)r   r~   Úreshape)rE   r�   ÚbatchÚnum_key_value_headsÚslenrl   s         r1   Ú	repeat_kvr–   ¨   so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr0   ÚmoduleÚqueryÚkeyÚvaluer'   ÚscalingÚdropoutÚkwargsc                 ó  — t        || j                  «      }t        || j                  «      }	t        j                  ||j	                  dd«      «      |z  }
|�|
|z   }
t
        j                  j                  |
dt        j                  ¬«      j                  |j                  «      }
t
        j                  j                  |
|| j                  ¬«      }
t        j                  |
|	«      }|j	                  dd«      j                  «       }||
fS )NrO   r   rQ   )rS   rn   )ÚpÚtrainingr"   )r–   Únum_key_value_groupsr,   Úmatmulrƒ   r;   Ú
functionalÚsoftmaxÚfloat32ru   rn   rœ   r    Ú
contiguous)r—   r˜   r™   rš   r'   r›   rœ   r�   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r1   Úeager_attention_forwardr«   ´   sà   € ô ˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r0   c                 ó|   — | dddd…f   }| dddd…f   }t        j                  | |fd¬«      j                  d«      S )	z*Rotates half the hidden dims of the input..r   NrO   r"   rQ   rR   éþÿÿÿ)r,   ÚstackÚflatten)r‡   Úx1Úx2s      r1   Úrotate_halfr²   Í   sJ   € à	
ˆ3���1�ˆ9‰€BØ	
ˆ3���1�ˆ9‰€BÜ�;‰;˜˜˜R�y bÔ)×1Ñ1°"Ó5Ð5r0   c                 ó  — |j                  |«      }|j                  |«      }|dd|j                  d   dz  …f   j                  dd¬«      }|dd|j                  d   dz  …f   j                  dd¬«      }|j                  d   }| dd|…f   | d|d…f   }}|dd|…f   |d|d…f   }	}||z  t        |«      |z  z   }
||z  t        |«      |z  z   }t	        j
                  |
|gd¬«      }
t	        j
                  ||	gd¬«      }|
|fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    .NrQ   rO   rR   )Ú	unsqueezer   Úrepeat_interleaver²   r,   r„   )ÚqÚkr…   r†   Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds               r1   Úapply_rotary_pos_embrÀ   Ô   sD  € ð$ �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cð ˆcÐ'�S—Y‘Y˜r‘] aÑ'Ð'Ð'Ñ
(×
:Ñ
:¸1À"Ð
:Ó
E€CØ
ˆcÐ'�S—Y‘Y˜r‘] aÑ'Ð'Ð'Ñ
(×
:Ñ
:¸1À"Ð
:Ó
E€Cð —‘˜2‘€JØ�c˜;˜J˜;Ð&Ñ'¨¨3°
±Ð+;Ñ)<ˆ6€EØ�c˜;˜J˜;Ð&Ñ'¨¨3°
±Ð+;Ñ)<ˆ6€Eð �s‰{œ{¨5Ó1°CÑ7Ñ8€GØ�s‰{œ{¨5Ó1°CÑ7Ñ8€Gô �i‰i˜ &Ð)¨rÔ2€GÜ�i‰i˜ &Ð)¨rÔ2€GØ�GÐÐr0   c                   óL  ‡ — e Zd ZdZdededededef
ˆ fd„Z	 	 	 	 dd	ej                  d
e
ej                  ej                  f   dz  dej                  dz  dedz  dej                  dz  dee   de
ej                  ej                  dz  e
ej                     dz  f   fd„Zˆ xZS )ÚMoonshineAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr9   Ú	layer_idxÚ	is_causalrq   r”   c                 ó8  •— t         ‰| �  «        |j                  ||dœ«       || _        || _        t        |d|j                  |j                  z  «      | _        |j                  |j                  z  | _
        | j                  dz  | _        |j                  | _        || _        t        j                  |j                  |j                  | j                  z  |j                   ¬«      | _        t        j                  |j                  |j                  | j                  z  |j                   ¬«      | _        t        j                  |j                  |j                  | j                  z  |j                   ¬«      | _        t        j                  |j                  | j                  z  |j                  d¬«      | _        | j                  j*                  �C| j                  j*                  }|| j                  |z   dz
  |z  z  }|| j                  z
  | _        y d| _        y )N)rq   r”   rl   g      à¿©ÚbiasFr"   r   )r7   r8   Úupdater9   rÃ   rp   r=   rq   rl   r”   r¡   r›   Úattention_dropoutrÄ   r;   r<   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚpad_head_dim_to_multiple_ofÚhead_dim_padding)	rB   r9   rÃ   rÄ   rq   r”   Útarget_multipleÚtarget_head_dimrD   s	           €r1   r8   zMoonshineAttention.__init__ÿ   s²  ø€ ô 	‰ÑÔØ�‰Ð.AÐZmÑnÔoØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØ"ˆŒä—i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^ÐejÔkˆŒð �;‰;×2Ñ2Ð>Ø"Ÿk™k×EÑEˆOØ-°$·-±-À/Ñ2QÐTUÑ2UÐZiÑ1iÑjˆOØ$3°d·m±mÑ$CˆDÕ!à$%ˆDÕ!r0   NrE   Úposition_embeddingsr'   Úpast_key_valuesÚkey_value_statesr�   rF   c                 ó:  — |j                   d d \  }}| j                  |«      j                  ||| j                  j                  | j
                  «      j                  dd«      }	|d u}
|�Y|j                  j                  | j                  «      }|
r&d|j                  | j                  <   |j                  }n|j                  }|�|n|}|
rK|rIrG|j                  | j                     j                  }|j                  | j                     j                  }nÆ| j                  |«      j                  |d| j                  j                  | j
                  «      j                  dd«      }| j!                  |«      j                  |d| j                  j                  | j
                  «      j                  dd«      }|
r"|� |j#                  ||| j                  «      \  }}|
s8|\  }}t%        |	|||«      \  }	}|� |j#                  ||| j                  «      \  }}t'        j(                  | j                  j*                  t,        «      }| j.                  xr |d u xr |dkD  }| j0                  dkD  r¢t2        j4                  j6                  j9                  |	d| j0                  f«      }	t2        j4                  j6                  j9                  |d| j0                  f«      }t2        j4                  j6                  j9                  |d| j0                  f«      } || |	|||f| j:                  sdn| j<                  | j>                  |dœ|¤Ž\  }}| j0                  dkD  r|dd | j0                   …f   }|jA                  ||d«      jC                  «       }| jE                  |«      }||fS )	NrQ   r"   rO   Tr   ç        )rœ   r›   rÄ   .)#r   rË   Úviewr9   r”   rl   rƒ   Ú
is_updatedro   rÃ   Úcross_attention_cacheÚself_attention_cacheÚlayersÚkeysÚvaluesrÌ   rÍ   rÈ   rÀ   r   Úget_interfaceÚ_attn_implementationr«   rÄ   rÐ   r,   r;   r£   Úpadr    rÉ   r›   r’   r¦   rÎ   )rB   rE   rÓ   r'   rÔ   rÕ   r�   ÚbszÚq_lenÚquery_statesÚis_cross_attentionrÙ   Úcurrent_statesr§   r¨   r…   r†   Úattention_interfacerÄ   rª   r©   s                        r1   rH   zMoonshineAttention.forward$  s]  € ð #×(Ñ(¨¨"Ð-‰
ˆˆUð �K‰K˜Ó&×+Ñ+¨C°¸¿¹×8WÑ8WÐY]×YfÑYfÓg×qÑqÐrsÐuvÓwð 	ð .°TÐ9ÐØÐ&Ø(×3Ñ3×7Ñ7¸¿¹ÓGˆJÙ!à=A�×*Ñ*¨4¯>©>Ñ:Ø"1×"GÑ"G‘à"1×"FÑ"F�ð .>Ð-IÑ)È}ˆÙ¡/±jØ(×/Ñ/°·±Ñ?×DÑDˆJØ*×1Ñ1°$·.±.ÑA×HÑH‰Lð —‘˜NÓ+ß‘�c˜2˜tŸ{™{×>Ñ>ÀÇÁÓNß‘˜1˜a“ð ð —‘˜NÓ+ß‘�c˜2˜tŸ{™{×>Ñ>ÀÇÁÓNß‘˜1˜a“ð ñ
 " oÐ&AØ+:×+AÑ+AÀ*ÈlÐ\`×\jÑ\jÓ+kÑ(�
˜Lá!Ø*‰HˆC�Ü';¸LÈ*ÐVYÐ[^Ó'_Ñ$ˆL˜*àÐ*Ø+:×+AÑ+AÀ*ÈlÐ\`×\jÑ\jÓ+kÑ(�
˜Lä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðð —N‘NÒK ~¸Ð'=ÒKÀ%È!Á)ˆ	à× Ñ  1Ò$Ü Ÿ8™8×.Ñ.×2Ñ2°<À!ÀT×EZÑEZÐA[Ó\ˆLÜŸ™×,Ñ,×0Ñ0°¸aÀ×AVÑAVÐ=WÓXˆJÜ Ÿ8™8×.Ñ.×2Ñ2°<À!ÀT×EZÑEZÐA[Ó\ˆLá$7ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LØñ
%
ð ñ
%
Ñ!ˆ�\ð × Ñ  1Ò$Ø% cÐ+C¨d×.CÑ.CÐ-CÐ+CÐ&CÑDˆKà!×)Ñ)¨#¨u°bÓ9×DÑDÓFˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r0   )NNNN)r(   r)   r*   r+   r#   rr   Úboolr8   r,   r-   rŽ   r	   r   r   rH   rJ   rK   s   @r1   rÂ   rÂ   ü   sû   ø„ ÙGð#&àð#&ð ð#&ð ð	#&ð
 !ð#&ð !õ#&ðP IMØ.2Ø(,Ø04ñO)à—|‘|ðO)ð # 5§<¡<°·±Ð#=Ñ>ÀÑEðO)ð Ÿ™ tÑ+ð	O)ð
  ™ðO)ð  Ÿ,™,¨Ñ-ðO)ð Ð-Ñ.ðO)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷O)r0   rÂ   c                   ó  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
eej                  ej                  f   dz  dee   dej                  fd„Zˆ xZS )ÚMoonshineEncoderLayerr9   rÃ   c                 ód  •— t         ‰| �  «        |j                  | _        t        ||d|j                  |j
                  ¬«      | _        t        ||j                  «      | _	        t        j                  |j                  d¬«      | _        t        j                  |j                  d¬«      | _        y )NF©r9   rÃ   rÄ   rq   r”   rÆ   )r7   r8   r=   rÂ   Úencoder_num_attention_headsÚencoder_num_key_value_headsÚ	self_attnr3   Úencoder_hidden_actÚmlpr;   Ú	LayerNormÚinput_layernormÚpost_attention_layernorm©rB   r9   rÃ   rD   s      €r1   r8   zMoonshineEncoderLayer.__init__w  s‹   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä+ØØØØ &× BÑ BØ &× BÑ Bô
ˆŒô ' v¨v×/HÑ/HÓIˆŒÜ!Ÿ|™|¨F×,>Ñ,>ÀUÔKˆÔÜ(*¯©°V×5GÑ5GÈeÔ(TˆÕ%r0   NrE   r'   rˆ   rÔ   Ú	use_cacherÓ   r�   rF   c           
      óÀ   — |}| j                  |«      } | j                  d||||||dœ|¤Ž\  }}	||z   }|}| j                  |«      }| j                  |«      }||z   }|S )N©rE   r'   rˆ   rÔ   rö   rÓ   r/   )ró   rï   rô   rñ   )
rB   rE   r'   rˆ   rÔ   rö   rÓ   r�   ÚresidualÚ_s
             r1   rH   zMoonshineEncoderLayer.forward‡  s�   € ð !ˆØ×,Ñ,¨]Ó;ˆà)˜4Ÿ>™>ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
Ñˆ�qð ! =Ñ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ  =Ñ0ˆØÐr0   )NNNFN)r(   r)   r*   r#   rr   r8   r,   r-   Ú
LongTensorr	   rè   rŽ   r   r   rH   rJ   rK   s   @r1   rê   rê   v  s¾   ø„ ðU˜ð U¸3õ Uð& /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð+Ñ,ðð 
�‰÷r0   rê   c                   óê  ‡ — e Zd Zddededz  fˆ fd„Z	 	 	 	 	 	 	 	 	 ddej                  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  de	dz  de
dz  deej                  ej                  f   dz  deej                  ej                  f   dz  dee   deej                  eej                  ej                  f   dz  f   fd„Zˆ xZS )ÚMoonshineDecoderLayerNr9   rÃ   c                 ó  •— t         ‰| �  «        |j                  | _        t        ||d|j                  |j
                  ¬«      | _        t        ||d|j                  |j
                  ¬«      | _        t        ||j                  «      | _
        t        j                  |j                  d¬«      | _        t        j                  |j                  d¬«      | _        t        j                  |j                  d¬«      | _        y )NTrì   FrÆ   )r7   r8   r=   rÂ   rq   r”   rï   Úencoder_attnrM   rC   rñ   r;   rò   ró   rô   Úfinal_layernormrõ   s      €r1   r8   zMoonshineDecoderLayer.__init__¨  sÎ   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä+ØØØØ &× :Ñ :Ø &× :Ñ :ô
ˆŒô /ØØØØ &× :Ñ :Ø &× :Ñ :ô
ˆÔô ' v¨v×/@Ñ/@ÓAˆŒÜ!Ÿ|™|¨F×,>Ñ,>ÀUÔKˆÔÜ(*¯©°V×5GÑ5GÈeÔ(TˆÔ%Ü!Ÿ|™|¨F×,>Ñ,>ÀUÔKˆÕr0   rE   r'   Úencoder_hidden_statesÚencoder_attention_maskrˆ   Úencoder_position_idsrÔ   rö   rÓ   Úencoder_position_embeddingsr�   rF   c           
      ó&  — |}| j                  |«      } | j                  d||||||	dœ|¤Ž\  }}||z   }|�1|}| j                  |«      }| j                  |||||¬«      \  }}||z   }|}| j	                  |«      }| j                  |«      }||z   }|S )Nrø   )rE   rÕ   r'   rÔ   rö   r/   )ró   rï   rô   rÿ   r   rñ   )rB   rE   r'   r  r  rˆ   r  rÔ   rö   rÓ   r  r�   rù   rú   s                 r1   rH   zMoonshineDecoderLayer.forwardÀ  sÞ   € ð !ˆØ×,Ñ,¨]Ó;ˆà)˜4Ÿ>™>ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
Ñˆ�qð ! =Ñ0ˆà Ð,Ø$ˆHØ ×9Ñ9¸-ÓHˆMØ#×0Ñ0Ø+Ø!6Ø5Ø /Ø#ð  1ó  ÑˆM˜1ð % }Ñ4ˆMà ˆØ×,Ñ,¨]Ó;ˆØŸ™ Ó/ˆØ  =Ñ0ˆØÐr0   r5   )	NNNNNNFNN)r(   r)   r*   r#   rr   r8   r,   r-   rû   r	   rè   rŽ   r   r   ÚFloatTensorrH   rJ   rK   s   @r1   rý   rý   §  sR  ø„ ñL˜ð L¸3À¹:õ Lð6 /3Ø59Ø6:Ø04Ø8<Ø(,Ø!&ØHLØPTñ,à—|‘|ð,ð Ÿ™ tÑ+ð,ð  %Ÿ|™|¨dÑ2ð	,ð
 !&§¡¨tÑ 3ð,ð ×&Ñ&¨Ñ-ð,ð $×.Ñ.°Ñ5ð,ð  ™ð,ð ˜$‘;ð,ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð,ð &+¨5¯<©<¸¿¹Ð+EÑ%FÈÑ%Mð,ð Ð+Ñ,ð,ð 
ˆu× Ñ  %¨×(9Ñ(9¸5×;LÑ;LÐ(LÑ"MÐPTÑ"TÐTÑ	U÷,r0   rý   c                   ó\   — e Zd ZU eed<   dZdZdZdZddgZ	dZ
dZdZdej                  fd	„Zy
)ÚMoonshinePreTrainedModelr9   ÚmodelÚinput_valuesÚaudioTrê   rý   Úinput_lengthsc                 ó~   — t        |dz
  dz  dz   «      }t        |dz
  dz  dz   «      }t        |dz
  dz  dz   «      }|S )zH
        Computes the output length of the convolutional layers
        é   é@   r"   é   r   rO   )rr   )rB   r  Úoutput_conv1_lengthÚoutput_conv2_lengthÚoutput_conv3_lengths        r1   Ú _get_feat_extract_output_lengthsz9MoonshinePreTrainedModel._get_feat_extract_output_lengthsý  sZ   € ô " =°3Ñ#6¸"Ñ"<¸qÑ"@ÓAÐÜ!Ð#6¸Ñ#:¸aÑ"?À!Ñ"CÓDÐÜ!Ð#6¸Ñ#:¸aÑ"?À!Ñ"CÓDÐà"Ð"r0   N)r(   r)   r*   r#   r.   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_can_compile_fullgraphr,   rû   r  r/   r0   r1   r  r  ï  sN   … àÓØÐØ$€OØÐØ&*Ð#Ø0Ð2IÐJÐØÐØ€Nà!Ðð#¸e×>NÑ>Nô #r0   r  c                   óà   ‡ — e Zd ZdZdZeedœZdefˆ fd„Z	de
j                  fd„Zde
j                  fd	„Zee	 ddej"                  dej$                  d
z  dee   deez  fd„«       «       Zˆ xZS )ÚMoonshineEncoderz£
    Transformer encoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MoonshineEncoderLayer`]

    Args:
        config: MoonshineConfig
    r
  )Ú
attentionsrE   r9   c           	      ób  •— t         ‰| �  |«       || _        |j                  }t	        j
                  d|ddd¬«      | _        t	        j
                  |d|z  dd¬	«      | _        t	        j
                  d|z  |dd¬	«      | _        t	        j                  d|d
¬«      | _
        t	        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t	        j                   |d¬«      | _        t%        |¬«      | _        d| _        | j+                  «        y c c}w )Nr"   r  r  F)Úkernel_sizeÚstriderÇ   rO   r  r   )r!  r"  gñhãˆµøä>)Ú
num_groupsÚnum_channelsÚepsrÆ   ©r9   )r7   r8   r9   r=   r;   ÚConv1dÚconv1Úconv2Úconv3Ú	GroupNormÚ	groupnormÚ
ModuleListÚrangeÚencoder_num_hidden_layersrê   rÜ   rò   Ú
layer_normrW   Ú
rotary_embÚgradient_checkpointingÚ	post_init)rB   r9   Ú	embed_dimÚidxrD   s       €r1   r8   zMoonshineEncoder.__init__  sì   ø€ Ü‰Ñ˜Ô ØˆŒØ×&Ñ&ˆ	ä—Y‘Y˜q )¸ÀRÈeÔTˆŒ
Ü—Y‘Y˜y¨!¨i©-ÀQÈqÔQˆŒ
Ü—Y‘Y˜q 9™}¨iÀQÈqÔQˆŒ
ÜŸ™°ÀÐPTÔUˆŒä—m‘mÜ;@À×AaÑAaÓ;bÖc°CÔ" 6¨3Õ/Òcó
ˆŒô Ÿ,™, y°uÔ=ˆŒÜ2¸&ÔAˆŒØ&+ˆÔ#Ø�‰Õùò ds   ÃD,rF   c                 ó   — | j                   S r5   ©r(  ©rB   s    r1   Úget_input_embeddingsz%MoonshineEncoder.get_input_embeddings(  s   € Ø�z‰zÐr0   rš   c                 ó   — || _         y r5   r7  ©rB   rš   s     r1   Úset_input_embeddingsz%MoonshineEncoder.set_input_embeddings+  s	   € Øˆ�
r0   Nr'   r�   c                 óh  — |j                  d«      }t        j                  j                  | j	                  |«      «      }| j                  |«      }t        j                  j                  | j                  |«      «      }t        j                  j                  | j                  |«      «      }|j                  ddd«      }d}|�3| j                  |j                  d   «      }d}|ddd|…f   dd|…f   }|}t        | j                  |||¬«      }t        j                  d|j                  d   |j                   ¬	«      j                  d«      }| j#                  ||¬
«      }	| j$                  D ]  }
 |
|f|||	dœ|¤Ž}Œ | j'                  |«      }t)        ||�|j+                  «       ¬«      S d¬«      S )a.  
        Args:
            input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
                Float values of the raw speech waveform. Raw speech waveform can be
                obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
                `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
                the soundfile library (`pip install soundfile`). To prepare the array into
                `input_values`, the [`AutoFeatureExtractor`] should be used for padding
                and conversion into a tensor of type `torch.FloatTensor`.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding indices in `input_values`. Mask values selected in `[0, 1]`:
                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.
                [What are attention masks?](../glossary#attention-mask)
        r"   r   rO   NrQ   i€  .©r9   Úinputs_embedsr'   r  ©rf   ©rˆ   )r'   rˆ   rÓ   )Úlast_hidden_stater'   )r´   r;   r£   Útanhr(  r,  Úgelur)  r*  Úpermuter  r   r   r9   r,   rs   rf   r1  rÜ   r0  r&   rr   )rB   r
  r'   r�   rE   Úoutput_attention_maskÚmask_lenÚdownsample_striderˆ   rÓ   Úencoder_layers              r1   rH   zMoonshineEncoder.forward.  s»  € ð. $×-Ñ-¨aÓ0ˆÜŸ™×*Ñ*¨4¯:©:°lÓ+CÓDˆØŸ™ }Ó5ˆÜŸ™×*Ñ*¨4¯:©:°mÓ+DÓEˆÜŸ™×*Ñ*¨4¯:©:°mÓ+DÓEˆØ%×-Ñ-¨a°°AÓ6ˆð !%ÐØÐ%Ø×<Ñ<¸^×=QÑ=QÐRTÑ=UÓVˆHØ *ÐØ+¨CÑ1DÐ3DÐ1DÐ,DÑEÀcÈ9ÈHÈ9ÀnÑUˆNØ$2Ð!ä2Ø—;‘;Ø'Ø)Ø"/ô	
ˆô —|‘| A }×':Ñ':¸1Ñ'=Àm×FZÑFZÔ[×eÑeÐfgÓhˆØ"Ÿo™o¨mÈ,˜oÓWÐà!Ÿ[™[ò 	ˆMÙ)Øðà-Ø)Ø$7ñ	ð
 ñ‰Mð	ð Ÿ™¨Ó6ˆä*Ø+Ø:OÐ:[Ð0×4Ñ4Ó6ô
ð 	
àaeô
ð 	
r0   r5   )r(   r)   r*   r+   r  rÂ   rê   Ú_can_record_outputsr#   r8   r;   ÚModuler9  r<  r   r!   r,   r  r-   r   r   rŽ   r   rH   rJ   rK   s   @r1   r  r    s«   ø„ ñð %€Oà(Ø.ñÐð
˜õ ð$ b§i¡ió ð¨"¯)©)ó ð  Øð /3ñ<
à×'Ñ'ð<
ð Ÿ™ tÑ+ð<
ð Ð+Ñ,ð	<
ð
 
Ð(Ñ	(ò<
ó ó  ô<
r0   r  c                   ób  ‡ — e Zd ZdZ eedd¬«      e eedd¬«      dœZdefˆ fd„Z	e
e	 	 	 	 	 	 	 	 ddej                  d	z  d
ej                  d	z  dej                  d	z  ded	z  dej                   d	z  ded	z  dej                   d	z  dej                  d	z  dee   deez  fd„«       «       Zˆ xZS )ÚMoonshineDecoderÚ	input_idsr"   rï   )ÚindexÚ
layer_namerÿ   )r  rE   Úcross_attentionsr9   c           	      ó  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        j                  |j                  d¬«      | _        t!        |¬«      | _        d| _        | j'                  «        y c c}w )NFrÆ   r&  )r7   r8   Úpad_token_idÚpadding_idxÚ
vocab_sizer;   Ú	Embeddingr=   Úembed_tokensr-  r.  Únum_hidden_layersrý   rÜ   rò   ÚnormrW   r1  r2  r3  )rB   r9   r5  rD   s      €r1   r8   zMoonshineDecoder.__init__x  sÁ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m‘mÔSXÐY_×YqÑYqÓSrÖ$sÈCÔ%:¸6À3Õ%GÒ$sÓtˆŒÜ—L‘L ×!3Ñ!3¸%Ô@ˆŒ	Ü2¸&ÔAˆŒØ&+ˆÔ#ð 	�‰Õùò %ts   ÂDNr'   rˆ   rÔ   r?  rö   r  r  r�   rF   c	           
      ó’  — |du |duz  rt        d«      ‚|€| j                  |«      }|r6|€4t        t        | j                  ¬«      t        | j                  ¬«      «      }|€V|�|j                  «       nd}
t        j                  |j                  d   |j                  ¬«      |
z   }|j                  d«      }t        | j                  ||||¬«      }t        | j                  |||¬«      }|}| j                  ||¬	«      }| j                  D ]  } ||||f|||||d
œ|	¤Ž}Œ | j                  |«      }t!        ||r|¬«      S d¬«      S )a¤  
        encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
            of the decoder.
        encoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding indices in `encoder_hidden_states`. Mask values selected in `[0, 1]`:
            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
            [What are attention masks?](../glossary#attention-mask)
        Nz:You must specify exactly one of input_ids or inputs_embedsr&  r   r"   r@  )r9   r?  r'   rÔ   rˆ   r>  rA  )r  rˆ   rÔ   rö   rÓ   )rB  rÔ   )Ú
ValueErrorrW  r   r
   r9   Úget_seq_lengthr,   rs   r   rf   r´   r   r   r1  rÜ   rY  r   )rB   rN  r'   rˆ   rÔ   r?  rö   r  r  r�   Úpast_seen_tokensÚcausal_maskrE   rÓ   Údecoder_layers                  r1   rH   zMoonshineDecoder.forward†  s‚  € ð0 ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü1´,ÀdÇkÁkÔ2RÔT`Ðhl×hsÑhsÔTtÓuˆOàÐØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<™<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ô\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ô
ˆô ";Ø—;‘;Ø'Ø1Ø"7ô	"
Ðð &ˆØ"Ÿo™o¨mÈ,˜oÓWÐà!Ÿ[™[ò 	ˆMÙ)ØØØ%ð
ð (>Ø)Ø /Ø#Ø$7ñ
ð ñ
‰Mð	ð Ÿ	™	 -Ó0ˆä8Ø+Ù/8˜Oô
ð 	
à>Bô
ð 	
r0   )NNNNNNNN)r(   r)   r*   r  r    rÂ   rý   rJ  r#   r8   r   r!   r,   rû   r-   r	   r  rè   r   r   rŽ   r   rH   rJ   rK   s   @r1   rM  rM  o  s0  ø„ à!€Oá$Ð%7¸qÈ[ÔYØ.Ù*Ð+=ÀQÐSaÔbñÐð˜õ ð  Øð .2Ø.2Ø04Ø(,Ø26Ø!%Ø:>Ø6:ñG
à×#Ñ# dÑ*ðG
ð Ÿ™ tÑ+ðG
ð ×&Ñ&¨Ñ-ð	G
ð
  ™ðG
ð ×(Ñ(¨4Ñ/ðG
ð ˜$‘;ðG
ð  %×0Ñ0°4Ñ7ðG
ð !&§¡¨tÑ 3ðG
ð Ð+Ñ,ðG
ð 
Ð(Ñ	(òG
ó ó  ôG
r0   rM  c                   óx  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zd„ Zd„ Ze	e
	 	 	 	 	 	 	 	 	 ddej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  deeej                        dz  dedz  deej                     dz  deej                     dz  dedz  dee   defd„«       «       Zˆ xZS )ÚMoonshineModelr9   c                 ó„   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        | j                  «        y r5   )r7   r8   r  ÚencoderrM  Údecoderr3  ©rB   r9   rD   s     €r1   r8   zMoonshineModel.__init__Ô  s2   ø€ Ü‰Ñ˜Ô ä'¨Ó/ˆŒÜ'¨Ó/ˆŒà�‰Õr0   c                 ó.   — | j                   j                  S r5   ©rd  rW  r8  s    r1   r9  z#MoonshineModel.get_input_embeddingsÜ  s   € Ø�|‰|×(Ñ(Ð(r0   c                 ó&   — || j                   _        y r5   rg  r;  s     r1   r<  z#MoonshineModel.set_input_embeddingsß  s   € Ø$)ˆ�‰Õ!r0   c                 ó8   — | j                   j                  «        y)z«
        Calling this function will disable the gradient computation for the Moonshine encoder so that its parameters will
        not be updated during training.
        N)rc  Ú_freeze_parametersr8  s    r1   Úfreeze_encoderzMoonshineModel.freeze_encoderâ  s   € ð
 	�‰×'Ñ'Õ)r0   c                 ó   — t        d«      ‚)z¢
        Masks extracted features along time axis and/or along feature axis according to
        [SpecAugment](https://huggingface.co/papers/1904.08779).
        zNot needed for Moonshine)ÚAttributeErrorr8  s    r1   Ú_mask_input_featuresz#MoonshineModel._mask_input_featuresé  s   € ô
 Ð7Ó8Ð8r0   Nr
  r'   Údecoder_input_idsÚdecoder_attention_maskÚencoder_outputsrÔ   Údecoder_inputs_embedsÚdecoder_position_idsrö   r�   rF   c
                 óT  — |€ | j                   |fd|i|
¤Ž} | j                  d|||j                  |j                  ||||	dœ|
¤Ž}t	        |j                  |j
                  |j                  |j                  |j                  |j                  |j                  |j                  ¬«      S )a
  
        input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
            Float values of the raw speech waveform. Raw speech waveform can be
            obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
            `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
            the soundfile library (`pip install soundfile`). To prepare the array into
            `input_values`, the [`AutoFeatureExtractor`] should be used for padding
            and conversion into a tensor of type `torch.FloatTensor`.
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`

        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoFeatureExtractor, MoonshineModel
        >>> from datasets import load_dataset

        >>> model = MoonshineModel.from_pretrained("UsefulSensors/moonshine-tiny")
        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("UsefulSensors/moonshine-tiny")
        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="pt")
        >>> input_values = inputs.input_values
        >>> decoder_input_ids = torch.tensor([[1, 1]]) * model.config.decoder_start_token_id
        >>> last_hidden_state = model(input_values, decoder_input_ids=decoder_input_ids).last_hidden_state
        >>> list(last_hidden_state.shape)
        [1, 2, 288]
        ```
        r'   )rN  r'   r  r  rÔ   r?  rˆ   rö   )rB  rÔ   Údecoder_hidden_statesÚdecoder_attentionsrQ  Úencoder_last_hidden_stater  Úencoder_attentionsr/   )	rc  rd  rB  r'   r   rÔ   rE   r  rQ  )rB   r
  r'   ro  rp  rq  rÔ   rr  rs  rö   r�   Údecoder_outputss               r1   rH   zMoonshineModel.forwardð  sÄ   € ðZ Ð"Ø/;¨t¯|©|¸LÑ/rÐYgÐ/rÐkqÑ/rˆOàEQÀTÇ\Á\ð 
F
Ø'Ø1Ø"1×"CÑ"CØ#2×#AÑ#AØ+Ø/Ø-Øñ
F
ð ñ
F
ˆô "Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
r0   )	NNNNNNNNN)r(   r)   r*   r#   r8   r9  r<  rk  rn  r   r   r,   r  rû   rŽ   r   rè   r   r   r   rH   rJ   rK   s   @r1   ra  ra  Ò  sE  ø„ ð˜õ ò)ò*ò*ò9ð Øð 26Ø26Ø59Ø:>ØBFØ6:ØAEØ?CØ!%ñC
à×'Ñ'¨$Ñ.ðC
ð ×(Ñ(¨4Ñ/ðC
ð !×+Ñ+¨dÑ2ð	C
ð
 !&× 0Ñ 0°4Ñ 7ðC
ð ˜u U×%6Ñ%6Ñ7Ñ8¸4Ñ?ðC
ð -¨tÑ3ðC
ð  % U×%6Ñ%6Ñ7¸$Ñ>ðC
ð $ E×$4Ñ$4Ñ5¸Ñ<ðC
ð ˜$‘;ðC
ð Ð+Ñ,ðC
ð 
òC
ó ó ôC
r0   ra  rN  rS  Údecoder_start_token_idc                 óÖ   — | j                  | j                  «      }| dd…dd…f   j                  «       |dd…dd…f<   ||dd…df<   |€t        d«      ‚|j	                  |dk(  |«       |S )z1
    Shift input ids one token to the right.
    NrQ   r"   r   z1self.model.config.pad_token_id has to be defined.iœÿÿÿ)Ú	new_zerosr   re   r[  Úmasked_fill_)rN  rS  rz  Úshifted_input_idss       r1   Úshift_tokens_rightr  8  s}   € ð "×+Ñ+¨I¯O©OÓ<ÐØ(ª¨C¨R¨C¨Ñ0×6Ñ6Ó8Ð’a˜™�eÑØ4Ð’a˜�dÑàÐÜÐLÓMÐMà×"Ñ"Ð#4¸Ñ#<¸lÔKàÐr0   zj
    The Moonshine Model with a language modeling head. Can be used for automatic speech recognition.
    c                   ó´  ‡ — e Zd ZddiZdefˆ fd„Zd„ Zd„ Zdej                  fd„Z
ee	 	 	 	 	 	 	 	 	 	 dd
ej                  d	z  dej                  d	z  dej                  d	z  dej                  d	z  deeej                        d	z  ded	z  deej                     d	z  deej                     d	z  ded	z  dej                  d	z  dee   defd„«       «       Zˆ xZS )Ú!MoonshineForConditionalGenerationzproj_out.weightz!model.decoder.embed_tokens.weightr9   c                 óÆ   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  d¬«      | _        | j                  «        y )NFrÆ   )
r7   r8   ra  r	  r;   r<   r=   rU  Úproj_outr3  re  s     €r1   r8   z*MoonshineForConditionalGeneration.__init__P  sH   ø€ Ü‰Ñ˜Ô Ü# FÓ+ˆŒ
ÜŸ	™	 &×"4Ñ"4°f×6GÑ6GÈeÔTˆŒð 	�‰Õr0   c                 ó   — | j                   S r5   ©rƒ  r8  s    r1   Úget_output_embeddingsz7MoonshineForConditionalGeneration.get_output_embeddingsX  s   € Ø�}‰}Ðr0   c                 ó   — || _         y r5   r…  )rB   Únew_embeddingss     r1   Úset_output_embeddingsz7MoonshineForConditionalGeneration.set_output_embeddings[  s	   € Ø&ˆ�r0   rF   c                 ó6   — | j                   j                  «       S r5   )r	  r9  r8  s    r1   r9  z6MoonshineForConditionalGeneration.get_input_embeddings^  s   € Ø�z‰z×.Ñ.Ó0Ð0r0   Nr
  r'   ro  rp  rq  rÔ   rr  rs  rö   Úlabelsr�   c                 ó.  — |
�9|€7|€5t        |
| j                  j                  | j                  j                  «      } | j                  |f||||||||	dœ|¤Ž}| j                  |j                  «      }d}|
�Ft        «       } ||j                  d| j                  j                  «      |
j                  d«      «      }t        |||j                  |j                  |j                  |j                  |j                  |j                   |j"                  ¬«	      S )a0  
        input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
            Float values of the raw speech waveform. Raw speech waveform can be
            obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
            `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
            the soundfile library (`pip install soundfile`). To prepare the array into
            `input_values`, the [`AutoFeatureExtractor`] should be used for padding
            and conversion into a tensor of type `torch.FloatTensor`.
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`

        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoProcessor, MoonshineForConditionalGeneration
        >>> from datasets import load_dataset

        >>> processor = AutoProcessor.from_pretrained("UsefulSensors/moonshine-tiny")
        >>> model = MoonshineForConditionalGeneration.from_pretrained("UsefulSensors/moonshine-tiny")

        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")

        >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="pt")
        >>> input_values = inputs.input_values

        >>> generated_ids = model.generate(input_values, max_new_tokens=100)

        >>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        >>> transcription
        'Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.'
        ```N)r'   ro  rq  rp  rÔ   rr  rs  rö   rQ   )	ÚlossÚlogitsrÔ   ru  rv  rQ  rw  r  rx  )r  r9   rS  rz  r	  rƒ  rB  r   r’   rU  r   rÔ   ru  rv  rQ  rw  r  rx  )rB   r
  r'   ro  rp  rq  rÔ   rr  rs  rö   r‹  r�   ÚoutputsrŽ  r�  Úloss_fcts                   r1   rH   z)MoonshineForConditionalGeneration.forwarda  s  € ðd ÐØ Ð(Ð-BÐ-JÜ$6Ø˜DŸK™K×4Ñ4°d·k±k×6XÑ6Xó%Ð!ð '1 d§j¡jØð'
à)Ø/Ø+Ø#9Ø+Ø"7Ø!5Øñ'
ð ñ'
ˆð —‘˜w×8Ñ8Ó9ˆàˆØÐÜ'Ó)ˆHÙ˜FŸN™N¨2¨t¯{©{×/EÑ/EÓFÈÏÉÐWYÓHZÓ[ˆDäØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ô

ð 
	
r0   )
NNNNNNNNNN)r(   r)   r*   Ú_tied_weights_keysr#   r8   r†  r‰  r;   rK  r9  r   r   r,   r  rû   rŽ   r   rè   r   r   r   rH   rJ   rK   s   @r1   r�  r�  H  sr  ø„ ð ,Ð-PÐQÐð˜õ òò'ð1 b§i¡ió 1ð Øð 26Ø26Ø59Ø:>ØBFØ6:ØAEØ?CØ!%Ø*.ñS
à×'Ñ'¨$Ñ.ðS
ð ×(Ñ(¨4Ñ/ðS
ð !×+Ñ+¨dÑ2ð	S
ð
 !&× 0Ñ 0°4Ñ 7ðS
ð ˜u U×%6Ñ%6Ñ7Ñ8¸4Ñ?ðS
ð -¨tÑ3ðS
ð  % U×%6Ñ%6Ñ7¸$Ñ>ðS
ð $ E×$4Ñ$4Ñ5¸Ñ<ðS
ð ˜$‘;ðS
ð × Ñ  4Ñ'ðS
ð Ð+Ñ,ðS
ð 
òS
ó ó ôS
r0   r�  )ra  r  r�  )r×   )r"   )IÚcollections.abcr   Údataclassesr   Útypingr   r,   Útorch.nnr;   r   Úactivationsr   Úcache_utilsr	   r
   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   r   Úutils.output_capturingr    r!   Úconfiguration_moonshiner#   r&   rK  r3   rM   rW   r-   rr   r–   rv   r«   r²   rÀ   rÂ   rê   rý   r  r  rM  ra  r  r�  Ú__all__r/   r0   r1   ú<module>r¥     sJ  ðõ* %Ý !Ý ã Ý Ý %å !ß CÑ CÝ )ß JÝ BÝ 9÷õ ÷ Lß FÝ &ß IÑ Iß Gß EÝ 4ñ ðôð
 ô
/ /ó 
/ó óð
/ô˜"Ÿ)™)ô ô˜"Ÿ)™)ô ô @<˜rŸy™yô @<ðF	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(ó%ò26ó%ôPw)˜Ÿ™ô w)ôt.Ð6ô .ôbEÐ6ô EðP ô#˜ó #ó ð#ô0d
Ð/ô d
ðN ô_
Ð/ó _
ó ð_
ðD ôb
Ð-ó b
ó ðb
ðJ %§,¡,ð ¸cð Ð[^ó ñ  ðôô
i
Ð(@À/ó i
óð
i
òX ^�r0   