
    ^j'                        d dl mZ d dlmZ d dlmZ d dlZd dlmZ d dlm	Z	 ddl
mZ ddlmZmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZmZmZmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z* ddl+m,Z,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2  e)d      e G d de                    Z3 G d dejh                        Z5 G d dejh                        Z6 G d dejh                        Z7d ejp                  d!e9d"ejp                  fd#Z:	 dEd$ejh                  d%ejp                  d&ejp                  d'ejp                  d(ejp                  dz  d)e;d*e;d+e&e(   fd,Z<d- Z=dFd.Z> G d/ d0ejh                        Z? G d1 d2e      Z@ G d3 d4e      ZAe) G d5 d6e$             ZB G d7 d8eB      ZCe) G d9 d:eB             ZDe) G d; d<eB             ZEd=ejp                  d>e9d?e9fd@ZF e)dA       G dB dCeBe             ZGg dDZHy)G    )Callable)	dataclass)OptionalN)CrossEntropyLoss   )ACT2FN)CacheDynamicCacheEncoderDecoderCache)GenerationMixin)create_bidirectional_maskcreate_causal_mask)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPast)BaseModelOutputWithPastAndCrossAttentionsSeq2SeqLMOutputSeq2SeqModelOutput)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuple)maybe_autocastmerge_with_config_defaults)OutputRecordercapture_outputs   )MoonshineConfigz
    Extends [~modeling_outputs.BaseModelOutput] to include the output attention mask since sequence length is not preserved in the model's forward.
    )custom_introc                   :    e Zd ZU dZdZej                  dz  ed<   y)MoonshineEncoderModelOutputa  
    attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to avoid performing attention on padding token indices after sequence compression. Returned because the
        sequence length may differ from the input sequence length. Mask values selected in `[0, 1]`:

        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.
    Nattention_mask)__name__
__module____qualname____doc__r'   torchTensor__annotations__     {/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/moonshine/modeling_moonshine.pyr&   r&   3   s     +/NELL4'.r0   r&   c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )MoonshineEncoderMLPc                    t         |           || _        t        |   | _        t        j                  |j                  |j                        | _	        t        j                  |j                  |j                        | _
        y Nsuper__init__configr   activation_fnnnLinearhidden_sizeintermediate_sizefc1fc2selfr9   
hidden_act	__class__s      r1   r8   zMoonshineEncoderMLP.__init__G   s^    #J/99V//1I1IJ99V55v7I7IJr0   hidden_statesreturnc                 l    | j                  |      }| j                  |      }| j                  |      }|S r5   )r?   r:   r@   )rB   rE   s     r1   forwardzMoonshineEncoderMLP.forwardN   s4    /**=9/r0   r(   r)   r*   r8   r,   r-   rH   __classcell__rD   s   @r1   r3   r3   F   s$    KU\\ ell r0   r3   c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )MoonshineDecoderMLPc                    t         |           || _        t        |   | _        t        j                  |j                  |j                  dz        | _	        t        j                  |j                  |j                        | _
        y )N   r6   rA   s      r1   r8   zMoonshineDecoderMLP.__init__V   sc    #J/99V//1I1IA1MN99V55v7I7IJr0   rE   rF   c                     | j                  |      }|j                  dd      \  }}| j                  |      |z  }| j                  |      }|S )NrO   dim)r?   chunkr:   r@   )rB   rE   gates      r1   rH   zMoonshineDecoderMLP.forward]   sS    /+11!1<t**40=@/r0   rI   rK   s   @r1   rM   rM   U   s$    KU\\ ell r0   rM   c                        e Zd ZU ej                  ed<   ddef fdZe	 	 	 ddedz  de	d   de
dz  ded	ef   fd
       Z ej                         ed               Z xZS )MoonshineRotaryEmbeddinginv_freqNr9   c                    t         |           |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |      \  }| _
        | j                  d|d       | j                  d|j                         d       y )N	rope_typedefaultrX   F)
persistentoriginal_inv_freq)r7   r8   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr9   rope_parametersrZ   compute_default_rope_parametersr   attention_scalingregister_bufferclone)rB   r9   devicerope_init_fnrX   rD   s        r1   r8   z!MoonshineRotaryEmbedding.__init__h   s    "("@"@$*$B$B!44[A!%!E!E>>Y&.t~~>L+7V+L($(ZeD0(..2BuUr0   rf   ztorch.deviceseq_lenrF   ztorch.Tensorc                 n   | j                   d   }| j                   j                  dd      }t        | dd      xs | j                  | j                  z  }t        ||z        }d}d|t        j                  d|dt        j                        j                  |t        j                  	      |z  z  z  }||fS )
a  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        
rope_thetapartial_rotary_factorg      ?head_dimNr   rO   dtype)rf   rn   )ra   getgetattrr=   num_attention_headsintr,   arangeint64tofloat)	r9   rf   rh   baserk   rl   rS   attention_factorrX   s	            r1   rb   z8MoonshineRotaryEmbedding.compute_default_rope_parametersx   s    & %%l3 & 6 6 : :;RTW X6:t4h8J8JfNhNh8h(223 U\\!S!5;;?BB&X]XcXcBdgjjk
 )))r0   c                 N   | j                   d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d      5  |j                         |j                         z  j                  dd      }t        j                  ||fd	      }|j                         | j                  z  }|j                         | j                  z  }	d d d        j	                  |j                   
      	j	                  |j                   
      fS # 1 sw Y   AxY w)Nr   rQ   r"   mpscpuF)device_typeenabledrO   rR   rm   )rX   rv   expandshaperu   rf   
isinstancetypestrr   	transposer,   catcosrc   sinrn   )
rB   xposition_idsinv_freq_expandedposition_ids_expandedr|   freqsembr   r   s
             r1   rH   z MoonshineRotaryEmbedding.forward   sR    !MM$4-8>>@GGHZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfkUC 	5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C		5 vvAGGv$cff177f&;;;	5 	5s   BFF$r5   )NNN)r(   r)   r*   r,   r-   r.   r#   r8   staticmethodr   rr   tuplerv   rb   no_gradr   rH   rJ   rK   s   @r1   rW   rW   e   s    llV V  )-+/"*$&*(* t* 
~u$	%	* *> U]]_<  <r0   rW   rE   n_reprF   c                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r"   N)r   r~   reshape)rE   r   batchnum_key_value_headsslenrl   s         r1   	repeat_kvr      so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TTr0   modulequerykeyvaluer'   scalingdropoutkwargsc                    t        || j                        }t        || j                        }	t        j                  ||j	                  dd            |z  }
||
|z   }
t
        j                  j                  |
dt        j                        j                  |j                        }
t
        j                  j                  |
|| j                        }
t        j                  |
|	      }|j	                  dd      j                         }||
fS )NrO   r   rQ   )rS   rn   )ptrainingr"   )r   num_key_value_groupsr,   matmulr   r;   
functionalsoftmaxfloat32ru   rn   r   r   
contiguous)r   r   r   r   r'   r   r   r   
key_statesvalue_statesattn_weightsattn_outputs               r1   eager_attention_forwardr      s     3 ; ;<JUF$?$?@L<<z';';Aq'ABWLL!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|\:K''1-88:K$$r0   c                 |    | ddddf   }| ddddf   }t        j                  | |fd      j                  d      S )	z*Rotates half the hidden dims of the input..r   NrO   r"   rQ   rR   )r,   stackflatten)r   x1x2s      r1   rotate_halfr      sJ    	
319B	
319B;;Ryb)11"55r0   c                    |j                  |      }|j                  |      }|dd|j                  d   dz  f   j                  dd      }|dd|j                  d   dz  f   j                  dd      }|j                  d   }| dd|f   | d|df   }}|dd|f   |d|df   }	}||z  t        |      |z  z   }
||z  t        |      |z  z   }t	        j
                  |
|gd      }
t	        j
                  ||	gd      }|
|fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    .NrQ   rO   rR   )	unsqueezer   repeat_interleaver   r,   r   )qkr   r   unsqueeze_dim
rotary_dimq_rotq_passk_rotk_passq_embedk_embeds               r1   apply_rotary_pos_embr      sD   $ --
&C
--
&C c'SYYr]a'''
(
:
:1"
:
EC
c'SYYr]a'''
(
:
:1"
:
EC 2Jc;J;&'3
+;)<6Ec;J;&'3
+;)<6E s{{51C78Gs{{51C78G ii&)r2Gii&)r2GGr0   c                   L    e Zd ZdZdededededef
 fdZ	 	 	 	 dd	ej                  d
e
ej                  ej                  f   dz  dej                  dz  dedz  dej                  dz  dee   de
ej                  ej                  dz  e
ej                     dz  f   fdZ xZS )MoonshineAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr9   	layer_idx	is_causalrq   r   c                 8   t         |           |j                  ||d       || _        || _        t        |d|j                  |j                  z        | _        |j                  |j                  z  | _
        | j                  dz  | _        |j                  | _        || _        t        j                  |j                  |j                  | j                  z  |j                         | _        t        j                  |j                  |j                  | j                  z  |j                         | _        t        j                  |j                  |j                  | j                  z  |j                         | _        t        j                  |j                  | j                  z  |j                  d      | _        | j                  j*                  C| j                  j*                  }|| j                  |z   dz
  |z  z  }|| j                  z
  | _        y d| _        y )N)rq   r   rl   g      ࿩biasFr"   r   )r7   r8   updater9   r   rp   r=   rq   rl   r   r   r   attention_dropoutr   r;   r<   attention_biasq_projk_projv_projo_projpad_head_dim_to_multiple_ofhead_dim_padding)	rB   r9   r   r   rq   r   target_multipletarget_head_dimrD   s	           r1   r8   zMoonshineAttention.__init__   s    	.AZmno"
F4F4F&JdJd4de$*$>$>&B\B\$\!}}d*!'!9!9"ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii : :T]] JFL^L^ejk ;;22>"kkEEO-$--/2QTU2UZi1ijO$3dmm$CD!$%D!r0   NrE   position_embeddingsr'   past_key_valueskey_value_statesr   rF   c                 :   |j                   d d \  }}| j                  |      j                  ||| j                  j                  | j
                        j                  dd      }	|d u}
|Y|j                  j                  | j                        }|
r&d|j                  | j                  <   |j                  }n|j                  }||n|}|
rK|rIrG|j                  | j                     j                  }|j                  | j                     j                  }n| j                  |      j                  |d| j                  j                  | j
                        j                  dd      }| j!                  |      j                  |d| j                  j                  | j
                        j                  dd      }|
r"| |j#                  ||| j                        \  }}|
s8|\  }}t%        |	|||      \  }	}| |j#                  ||| j                        \  }}t'        j(                  | j                  j*                  t,              }| j.                  xr |d u xr |dkD  }| j0                  dkD  rt2        j4                  j6                  j9                  |	d| j0                  f      }	t2        j4                  j6                  j9                  |d| j0                  f      }t2        j4                  j6                  j9                  |d| j0                  f      } || |	|||f| j:                  sdn| j<                  | j>                  |d|\  }}| j0                  dkD  r|dd | j0                   f   }|jA                  ||d      jC                         }| jE                  |      }||fS )	NrQ   r"   rO   Tr           )r   r   r   .)#r   r   viewr9   r   rl   r   
is_updatedro   r   cross_attention_cacheself_attention_cachelayerskeysvaluesr   r   r   r   r   get_interface_attn_implementationr   r   r   r,   r;   r   padr   r   r   r   r   r   )rB   rE   r   r'   r   r   r   bszq_lenquery_statesis_cross_attentionr   current_statesr   r   r   r   attention_interfacer   r   r   s                        r1   rH   zMoonshineAttention.forward$  s]    #(("-
U KK&++C8W8WY]YfYfgqqrsuvw 	 .T9&(3377GJ!=A**4>>:"1"G"G"1"F"F .>-I)}/j(//?DDJ*11$..AHHL N+c2t{{>>N1a  N+c2t{{>>N1a 
 "o&A+:+A+A*l\`\j\j+k(
L!*HC';L*VY[^'_$L**+:+A+A*l\`\j\j+k(
L(?(M(MKK,,.E)
 NNK~'=K%!)	  1$ 88..22<!TEZEZA[\L,,00aAVAV=WXJ 88..22<!TEZEZA[\L$7
%
  $}}C$2H2HLL
%
 
%
!\   1$%c+Cd.C.C-C+C&CDK!))#ub9DDFkk+.L((r0   )NNNN)r(   r)   r*   r+   r#   rr   boolr8   r,   r-   r   r	   r   r   rH   rJ   rK   s   @r1   r   r      s    G#&#& #& 	#&
 !#& !#&P IM.2(,04O)||O) #5<<#=>EO) t+	O)
 O)  ,,-O) -.O) 
u||U\\D0%2E2LL	MO)r0   r   c                       e Zd Zdedef fdZ	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
eej                  ej                  f   dz  dee   dej                  fdZ xZS )MoonshineEncoderLayerr9   r   c                 d   t         |           |j                  | _        t        ||d|j                  |j
                        | _        t        ||j                        | _	        t        j                  |j                  d      | _        t        j                  |j                  d      | _        y )NFr9   r   r   rq   r   r   )r7   r8   r=   r   encoder_num_attention_headsencoder_num_key_value_heads	self_attnr3   encoder_hidden_actmlpr;   	LayerNorminput_layernormpost_attention_layernormrB   r9   r   rD   s      r1   r8   zMoonshineEncoderLayer.__init__w  s    !--+ & B B & B B
 'vv/H/HI!||F,>,>UK(*V5G5Ge(T%r0   NrE   r'   r   r   	use_cacher   r   rF   c           
          |}| j                  |      } | j                  d||||||d|\  }}	||z   }|}| j                  |      }| j                  |      }||z   }|S )NrE   r'   r   r   r   r   r/   )r   r   r   r   )
rB   rE   r'   r   r   r   r   r   residual_s
             r1   rH   zMoonshineEncoderLayer.forward  s     !,,];)4>> 
')%+ 3
 
q !=0 !55mD/ =0r0   )NNNFN)r(   r)   r*   r#   rr   r8   r,   r-   
LongTensorr	   r   r   r   r   rH   rJ   rK   s   @r1   r   r   v  s    U U3 U& /304(,!&HL|| t+ &&-	
  $; #5<<#=>E +, 
r0   r   c                       e Zd Zddededz  f fdZ	 	 	 	 	 	 	 	 	 ddej                  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  de	dz  de
dz  deej                  ej                  f   dz  deej                  ej                  f   dz  dee   deej                  eej                  ej                  f   dz  f   fdZ xZS )MoonshineDecoderLayerNr9   r   c                    t         |           |j                  | _        t        ||d|j                  |j
                        | _        t        ||d|j                  |j
                        | _        t        ||j                        | _
        t        j                  |j                  d      | _        t        j                  |j                  d      | _        t        j                  |j                  d      | _        y )NTr   Fr   )r7   r8   r=   r   rq   r   r   encoder_attnrM   rC   r   r;   r   r   r   final_layernormr   s      r1   r8   zMoonshineDecoderLayer.__init__  s    !--+ & : : & : :
 / & : : & : :
 'vv/@/@A!||F,>,>UK(*V5G5Ge(T%!||F,>,>UKr0   rE   r'   encoder_hidden_statesencoder_attention_maskr   encoder_position_idsr   r   r   encoder_position_embeddingsr   rF   c           
      &   |}| j                  |      } | j                  d||||||	d|\  }}||z   }|1|}| j                  |      }| j                  |||||      \  }}||z   }|}| j	                  |      }| j                  |      }||z   }|S )Nr   )rE   r   r'   r   r   r/   )r   r   r   r   r   r   )rB   rE   r'   r  r  r   r  r   r   r   r  r   r   r   s                 r1   rH   zMoonshineDecoderLayer.forward  s     !,,];)4>> 
')%+ 3
 
q !=0 ,$H 99-HM#00+!65 /#  1  M1 %}4M ,,];/ =0r0   r5   )	NNNNNNFNN)r(   r)   r*   r#   rr   r8   r,   r-   r   r	   r   r   r   r   FloatTensorrH   rJ   rK   s   @r1   r   r     sR   L L3: L6 /3596:048<(,!&HLPT,||, t+,  %||d2	,
 !&t 3, &&-, $..5, , $;, #5<<#=>E, &+5<<+E%F%M, +,, 
u  %(9(95;L;L(L"MPT"TT	U,r0   r   c                   \    e Zd ZU eed<   dZdZdZdZddgZ	dZ
dZdZdej                  fd	Zy
)MoonshinePreTrainedModelr9   modelinput_valuesaudioTr   r   input_lengthsc                 ~    t        |dz
  dz  dz         }t        |dz
  dz  dz         }t        |dz
  dz  dz         }|S )zH
        Computes the output length of the convolutional layers
           @   r"      r   rO   )rr   )rB   r  output_conv1_lengthoutput_conv2_lengthoutput_conv3_lengths        r1    _get_feat_extract_output_lengthsz9MoonshinePreTrainedModel._get_feat_extract_output_lengths  sZ     "=3#6""<q"@A!#6#:a"?!"CD!#6#:a"?!"CD""r0   N)r(   r)   r*   r#   r.   base_model_prefixmain_input_nameinput_modalitiessupports_gradient_checkpointing_no_split_modules_supports_flash_attn_supports_sdpa_can_compile_fullgraphr,   r   r  r/   r0   r1   r  r    sN    $O&*#02IJN!#e>N>N #r0   r  c                        e Zd ZdZdZeedZdef fdZ	de
j                  fdZde
j                  fd	Zee	 ddej"                  dej$                  d
z  dee   deez  fd              Z xZS )MoonshineEncoderz
    Transformer encoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MoonshineEncoderLayer`]

    Args:
        config: MoonshineConfig
    r
  )
attentionsrE   r9   c           	      b   t         |   |       || _        |j                  }t	        j
                  d|ddd      | _        t	        j
                  |d|z  dd	      | _        t	        j
                  d|z  |dd	      | _        t	        j                  d|d
      | _
        t	        j                  t        |j                        D cg c]  }t        ||       c}      | _        t	        j                   |d      | _        t%        |      | _        d| _        | j+                          y c c}w )Nr"   r  r  F)kernel_sizestrider   rO   r  r   )r!  r"  gh㈵>)
num_groupsnum_channelsepsr   r9   )r7   r8   r9   r=   r;   Conv1dconv1conv2conv3	GroupNorm	groupnorm
ModuleListrangeencoder_num_hidden_layersr   r   r   
layer_normrW   
rotary_embgradient_checkpointing	post_init)rB   r9   	embed_dimidxrD   s       r1   r8   zMoonshineEncoder.__init__  s     &&	YYq)ReT
YYy!i-QqQ
YYq9}iQqQ
PTUmm;@AaAa;bcC"63/c
 ,,yu=2&A&+# ds   D,rF   c                     | j                   S r5   r(  rB   s    r1   get_input_embeddingsz%MoonshineEncoder.get_input_embeddings(  s    zzr0   r   c                     || _         y r5   r7  rB   r   s     r1   set_input_embeddingsz%MoonshineEncoder.set_input_embeddings+  s	    
r0   Nr'   r   c                 h   |j                  d      }t        j                  j                  | j	                  |            }| j                  |      }t        j                  j                  | j                  |            }t        j                  j                  | j                  |            }|j                  ddd      }d}|3| j                  |j                  d         }d}|ddd|f   dd|f   }|}t        | j                  |||      }t        j                  d|j                  d   |j                   	      j                  d      }| j#                  ||
      }	| j$                  D ]  }
 |
|f|||	d|} | j'                  |      }t)        |||j+                               S d      S )a.  
        Args:
            input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
                Float values of the raw speech waveform. Raw speech waveform can be
                obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
                `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
                the soundfile library (`pip install soundfile`). To prepare the array into
                `input_values`, the [`AutoFeatureExtractor`] should be used for padding
                and conversion into a tensor of type `torch.FloatTensor`.
            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
                Mask to avoid performing attention on padding indices in `input_values`. Mask values selected in `[0, 1]`:
                - 1 for tokens that are **not masked**,
                - 0 for tokens that are **masked**.
                [What are attention masks?](../glossary#attention-mask)
        r"   r   rO   NrQ   i  .r9   inputs_embedsr'   r  rf   r   )r'   r   r   )last_hidden_stater'   )r   r;   r   tanhr(  r,  gelur)  r*  permuter  r   r   r9   r,   rs   rf   r1  r   r0  r&   rr   )rB   r
  r'   r   rE   output_attention_maskmask_lendownsample_strider   r   encoder_layers              r1   rH   zMoonshineEncoder.forward.  s   . $--a0**4::l+CD}5**4::m+DE**4::m+DE%--aA6 !%%<<^=Q=QRT=UVH *+C1D3D1D,DEc9H9nUN$2!2;;')"/	
 ||A}':':1'=mFZFZ[eefgh"oom,oW![[ 	M)-)$7	
 M	 6*+:O:[0446
 	
ae
 	
r0   r5   )r(   r)   r*   r+   r  r   r   _can_record_outputsr#   r8   r;   Moduler9  r<  r   r!   r,   r  r-   r   r   r   r   rH   rJ   rK   s   @r1   r  r    s     %O(.
 $bii "))    /3<
''<
 t+<
 +,	<

 
(	(<
   <
r0   r  c                   b    e Zd ZdZ eedd      e eedd      dZdef fdZ	e
e	 	 	 	 	 	 	 	 ddej                  d	z  d
ej                  d	z  dej                  d	z  ded	z  dej                   d	z  ded	z  dej                   d	z  dej                  d	z  dee   deez  fd              Z xZS )MoonshineDecoder	input_idsr"   r   )index
layer_namer   )r  rE   cross_attentionsr9   c           	         t         |   |       |j                  | _        |j                  | _        t        j                  |j                  |j                  | j                        | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        j                  |j                  d      | _        t!        |      | _        d| _        | j'                          y c c}w )NFr   r&  )r7   r8   pad_token_idpadding_idx
vocab_sizer;   	Embeddingr=   embed_tokensr-  r.  num_hidden_layersr   r   r   normrW   r1  r2  r3  )rB   r9   r5  rD   s      r1   r8   zMoonshineDecoder.__init__x  s     !.. ++LL):):F<N<NPTP`P`ammSXY_YqYqSr$sC%:63%G$stLL!3!3%@	2&A&+# 	 %ts   DNr'   r   r   r?  r   r  r  r   rF   c	           
         |du |duz  rt        d      || j                  |      }|r6|4t        t        | j                        t        | j                              }|V||j                         nd}
t        j                  |j                  d   |j                        |
z   }|j                  d      }t        | j                  ||||      }t        | j                  |||      }|}| j                  ||	      }| j                  D ]  } ||||f|||||d
|	} | j                  |      }t!        ||r|      S d      S )a  
        encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
            of the decoder.
        encoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding indices in `encoder_hidden_states`. Mask values selected in `[0, 1]`:
            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.
            [What are attention masks?](../glossary#attention-mask)
        Nz:You must specify exactly one of input_ids or inputs_embedsr&  r   r"   r@  )r9   r?  r'   r   r   r>  rA  )r  r   r   r   r   )rB  r   )
ValueErrorrW  r   r
   r9   get_seq_lengthr,   rs   r   rf   r   r   r   r1  r   rY  r   )rB   rN  r'   r   r   r?  r   r  r  r   past_seen_tokenscausal_maskrE   r   decoder_layers                  r1   rH   zMoonshineDecoder.forward  s   0 -t";<YZZ  --i8M01,dkk2RT`hlhshsTtuOCRC^==?de <<(;(;A(>}G[G[\_ooL'11!4L(;;')+%
 ";;;'1"7	"
 &"oom,oW![[ 	M)%
 (>) /#$7
 
M	 		-08+/8O
 	
>B
 	
r0   )NNNNNNNN)r(   r)   r*   r  r    r   r   rJ  r#   r8   r   r!   r,   r   r-   r	   r  r   r   r   r   r   rH   rJ   rK   s   @r1   rM  rM  o  s0   !O$%7q[Y.*+=QSab    .2.204(,26!%:>6:G
##d*G
 t+G
 &&-	G

 G
 ((4/G
 $;G
  %0047G
 !&t 3G
 +,G
 
(	(G
   G
r0   rM  c                   x    e Zd Zdef fdZd Zd Zd Zd Ze	e
	 	 	 	 	 	 	 	 	 ddej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  deeej                        dz  dedz  deej                     dz  deej                     dz  dedz  dee   defd              Z xZS )MoonshineModelr9   c                     t         |   |       t        |      | _        t	        |      | _        | j                          y r5   )r7   r8   r  encoderrM  decoderr3  rB   r9   rD   s     r1   r8   zMoonshineModel.__init__  s2     '/'/r0   c                 .    | j                   j                  S r5   rd  rW  r8  s    r1   r9  z#MoonshineModel.get_input_embeddings  s    ||(((r0   c                 &    || j                   _        y r5   rg  r;  s     r1   r<  z#MoonshineModel.set_input_embeddings  s    $)!r0   c                 8    | j                   j                          y)z
        Calling this function will disable the gradient computation for the Moonshine encoder so that its parameters will
        not be updated during training.
        N)rc  _freeze_parametersr8  s    r1   freeze_encoderzMoonshineModel.freeze_encoder  s    
 	'')r0   c                     t        d      )z
        Masks extracted features along time axis and/or along feature axis according to
        [SpecAugment](https://huggingface.co/papers/1904.08779).
        zNot needed for Moonshine)AttributeErrorr8  s    r1   _mask_input_featuresz#MoonshineModel._mask_input_features  s    
 788r0   Nr
  r'   decoder_input_idsdecoder_attention_maskencoder_outputsr   decoder_inputs_embedsdecoder_position_idsr   r   rF   c
                 T   | | j                   |fd|i|
} | j                  d|||j                  |j                  ||||	d|
}t	        |j                  |j
                  |j                  |j                  |j                  |j                  |j                  |j                        S )a
  
        input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
            Float values of the raw speech waveform. Raw speech waveform can be
            obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
            `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
            the soundfile library (`pip install soundfile`). To prepare the array into
            `input_values`, the [`AutoFeatureExtractor`] should be used for padding
            and conversion into a tensor of type `torch.FloatTensor`.
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`

        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoFeatureExtractor, MoonshineModel
        >>> from datasets import load_dataset

        >>> model = MoonshineModel.from_pretrained("UsefulSensors/moonshine-tiny")
        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("UsefulSensors/moonshine-tiny")
        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="pt")
        >>> input_values = inputs.input_values
        >>> decoder_input_ids = torch.tensor([[1, 1]]) * model.config.decoder_start_token_id
        >>> last_hidden_state = model(input_values, decoder_input_ids=decoder_input_ids).last_hidden_state
        >>> list(last_hidden_state.shape)
        [1, 2, 288]
        ```
        r'   )rN  r'   r  r  r   r?  r   r   )rB  r   decoder_hidden_statesdecoder_attentionsrQ  encoder_last_hidden_stater  encoder_attentionsr/   )	rc  rd  rB  r'   r   r   rE   r  rQ  )rB   r
  r'   ro  rp  rq  r   rr  rs  r   r   decoder_outputss               r1   rH   zMoonshineModel.forward  s    Z "/;t||L/rYg/rkq/rOEQT\\ 
F
'1"1"C"C#2#A#A+/-
F
 
F
 "-??+;;"1"?"?.99,==&5&G&G"1"?"?.99	
 		
r0   )	NNNNNNNNN)r(   r)   r*   r#   r8   r9  r<  rk  rn  r   r   r,   r  r   r   r   r   r   r   r   rH   rJ   rK   s   @r1   ra  ra    sE    )**9  262659:>BF6:AE?C!%C
''$.C
 ((4/C
 !++d2	C

 !& 0 04 7C
 uU%6%6784?C
 -t3C
  %U%6%67$>C
 $E$4$45<C
 $;C
 +,C
 
C
  C
r0   ra  rN  rS  decoder_start_token_idc                     | j                  | j                        }| ddddf   j                         |ddddf<   ||dddf<   |t        d      |j	                  |dk(  |       |S )z1
    Shift input ids one token to the right.
    NrQ   r"   r   z1self.model.config.pad_token_id has to be defined.i)	new_zerosr   re   r[  masked_fill_)rN  rS  rz  shifted_input_idss       r1   shift_tokens_rightr  8  s}     "++IOO<(CRC0668ae4adLMM""#4#<lKr0   zj
    The Moonshine Model with a language modeling head. Can be used for automatic speech recognition.
    c                       e Zd ZddiZdef fdZd Zd Zdej                  fdZ
ee	 	 	 	 	 	 	 	 	 	 dd
ej                  d	z  dej                  d	z  dej                  d	z  dej                  d	z  deeej                        d	z  ded	z  deej                     d	z  deej                     d	z  ded	z  dej                  d	z  dee   defd              Z xZS )!MoonshineForConditionalGenerationzproj_out.weightz!model.decoder.embed_tokens.weightr9   c                     t         |   |       t        |      | _        t	        j
                  |j                  |j                  d      | _        | j                          y )NFr   )
r7   r8   ra  r	  r;   r<   r=   rU  proj_outr3  re  s     r1   r8   z*MoonshineForConditionalGeneration.__init__P  sH     #F+
		&"4"4f6G6GeT 	r0   c                     | j                   S r5   r  r8  s    r1   get_output_embeddingsz7MoonshineForConditionalGeneration.get_output_embeddingsX  s    }}r0   c                     || _         y r5   r  )rB   new_embeddingss     r1   set_output_embeddingsz7MoonshineForConditionalGeneration.set_output_embeddings[  s	    &r0   rF   c                 6    | j                   j                         S r5   )r	  r9  r8  s    r1   r9  z6MoonshineForConditionalGeneration.get_input_embeddings^  s    zz..00r0   Nr
  r'   ro  rp  rq  r   rr  rs  r   labelsr   c                 .   |
9|7|5t        |
| j                  j                  | j                  j                        } | j                  |f||||||||	d|}| j                  |j                        }d}|
Ft               } ||j                  d| j                  j                        |
j                  d            }t        |||j                  |j                  |j                  |j                  |j                  |j                   |j"                  	      S )a0  
        input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
            Float values of the raw speech waveform. Raw speech waveform can be
            obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
            `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
            the soundfile library (`pip install soundfile`). To prepare the array into
            `input_values`, the [`AutoFeatureExtractor`] should be used for padding
            and conversion into a tensor of type `torch.FloatTensor`.
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`

        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoProcessor, MoonshineForConditionalGeneration
        >>> from datasets import load_dataset

        >>> processor = AutoProcessor.from_pretrained("UsefulSensors/moonshine-tiny")
        >>> model = MoonshineForConditionalGeneration.from_pretrained("UsefulSensors/moonshine-tiny")

        >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")

        >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="pt")
        >>> input_values = inputs.input_values

        >>> generated_ids = model.generate(input_values, max_new_tokens=100)

        >>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
        >>> transcription
        'Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.'
        ```N)r'   ro  rq  rp  r   rr  rs  r   rQ   )	losslogitsr   ru  rv  rQ  rw  r  rx  )r  r9   rS  rz  r	  r  rB  r   r   rU  r   r   ru  rv  rQ  rw  r  rx  )rB   r
  r'   ro  rp  rq  r   rr  rs  r   r  r   outputsr  r  loss_fcts                   r1   rH   z)MoonshineForConditionalGeneration.forwarda  s   d  (-B-J$6DKK44dkk6X6X%! '1djj'
)/+#9+"7!5'
 '
 w889')HFNN2t{{/E/EFWYHZ[D#33")"?"?&99$55&-&G&G")"?"?&99

 
	
r0   )
NNNNNNNNNN)r(   r)   r*   _tied_weights_keysr#   r8   r  r  r;   rK  r9  r   r   r,   r  r   r   r   r   r   r   r   rH   rJ   rK   s   @r1   r  r  H  sr    ,-PQ '1bii 1  262659:>BF6:AE?C!%*.S
''$.S
 ((4/S
 !++d2	S

 !& 0 04 7S
 uU%6%6784?S
 -t3S
  %U%6%67$>S
 $E$4$45<S
 $;S
   4'S
 +,S
 
S
  S
r0   r  )ra  r  r  )r   )r"   )Icollections.abcr   dataclassesr   typingr   r,   torch.nnr;   r   activationsr   cache_utilsr	   r
   r   
generationr   masking_utilsr   r   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   r   r   r   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   utils.genericr   r   utils.output_capturingr    r!   configuration_moonshiner#   r&   rK  r3   rM   rW   r-   rr   r   rv   r   r   r   r   r   r   r  r  rM  ra  r  r  __all__r/   r0   r1   <module>r     sJ  * % !    % ! C C ) J B 9  L F & I I G E 4 
 
// 
/ 
/")) "))  @<ryy @<F	UU\\ 	U# 	U%,, 	U& %II%<<% 
% <<	%
 LL4'% % % '(%26%Pw) w)t.6 .bE6 EP # # #0d
/ d
N _
/ _
 _
D b
- b
 b
J%,, c [^   
i
(@/ i

i
X ^r0   