
    ^jD                        d dl mZ d dlmZ d dlmZ d dlmZmZ ddl	m
Z ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZmZmZmZ ddlmZ ddlm Z m!Z!m"Z"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z)  ejT                  e+      Z, e       rd dlZ ed      e G d de                    Z-e G d de%             Z.d Z/d Z0 G d de)      Z1 G d de#      Z2e G d  d!e"             Z3 G d" d#e!      Z4 ed$%       G d& d'e              Z5g d(Z6y))    )	dataclass)pi)strict)Tensorbroadcast_tensors   )initialization)Cache)PreTrainedConfig)BaseModelOutputWithPooling)PreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tupleis_torch_availableloggingtorch_compilable_check   )AudioFlamingo3Config)&AudioFlamingo3ForConditionalGenerationAudioFlamingo3Model!AudioFlamingo3ModelOutputWithPastAudioFlamingo3PreTrainedModel)AudioFlamingo3Processor)CONFIG_MAPPING)MoonshineRotaryEmbeddingNznvidia/music-flamingo-2601-hf)
checkpointc                   V    e Zd ZU dZdZeed<   dZeed<   dZe	ed<   dZ
edz  ed	<   d
 Zy)MusicFlamingoConfiga  
    audio_bos_token_id (`int`, *optional*, defaults to 151670):
        The beginning-of-audio token index used to mark the start of audio spans.
    audio_eos_token_id (`int`, *optional*, defaults to 151671):
        The end-of-audio token index used to mark the end of audio spans.
    audio_frame_step (`float`, *optional*, defaults to 0.01):
        Duration in seconds of one input mel frame (trained with hop_length 160 at sampling_rate 16000).

    Example:

    ```python
    >>> from transformers import MusicFlamingoForConditionalGeneration, MusicFlamingoConfig, AudioFlamingo3EncoderConfig, Qwen2Config

    >>> # Initializing an MusicFlamingoEncoder config
    >>> audio_config = AudioFlamingo3EncoderConfig()

    >>> # Initializing a Qwen2 config
    >>> text_config = Qwen2Config()

    >>> # Initializing an MusicFlamingo configuration
    >>> configuration = MusicFlamingoConfig(audio_config, text_config)

    >>> # Initializing a model from the musicflamingo style configuration
    >>> model = MusicFlamingoForConditionalGeneration(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```ivP audio_bos_token_idiwP audio_eos_token_idg{Gz?audio_frame_stepNrope_parametersc                    | j                   dddd| _         t        | j                  t              rK| j                  d   dv rd| j                  d<   t	        | j                  d      d
i | j                  | _        n| j                  t	        d          | _        t        | j
                  t              rT| j
                  j                  dd      | j
                  d<   t	        | j
                  d      d
i | j
                  | _        n| j
                  t	        d          | _        | j                   d	   | _        | j                  j                  | _	        t        j                  | fi | y )Ndefaultg     @g?)	rope_type
rope_thetapartial_rotary_factor
model_type)Nmusicflamingo_encoderaudioflamingo3_encoderqwen2r(    )r$   
isinstanceaudio_configdictr   text_configgetmax_position_embeddingshidden_sizehead_dimr   __post_init__)selfkwargss     /var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/musicflamingo/modular_musicflamingo.pyr7   z!MusicFlamingoConfig.__post_init__\   sK   '&$),$D 
 d''.  .2QQ2J!!,/ .t/@/@/N O dRVRcRc dD& ./G H JDd&&--1-=-=-A-A,PW-XD\*-d.>.>|.LMaPTP`P`aD%-g68D'+';';L'I$))55&&t6v6    )__name__
__module____qualname____doc__r!   int__annotations__r"   r#   floatr$   r1   r7   r.   r;   r:   r    r    7   s=    : %$$$"e"#'OTD['7r;   r    c                   v     e Zd Z	 	 	 	 	 d fd	ZdededefdZed        Z	ed        Z
d Zd	 Zd
 Zd Z xZS )MusicFlamingoProcessorc                     t         |   |||||       | `|| _        || _        |j                  |      | _        |j                  |      | _        y)as  
        audio_token (`Optional[str]`, *optional*, defaults to `"<sound>"`):
            Special token used to represent audio inputs in the chat template.
        audio_bos_token (`Optional[str]`, *optional*, defaults to `"<|sound_bos|>"`):
            Special token used to represent the beginning of audio.
        audio_eos_token (`Optional[str]`, *optional*, defaults to `"<|sound_eos|>"`):
            Special token used to represent the end of audio.
        max_audio_len (`int`, *optional*, defaults to 1200):
            Maximum length of audio sequences in seconds. Audio longer than this will be truncated.
        )chat_templateaudio_tokenmax_audio_lenN)super__init__default_transcription_promptaudio_bos_tokenaudio_eos_tokenconvert_tokens_to_idsr!   r"   )	r8   feature_extractor	tokenizerrF   rG   rL   rM   rH   	__class__s	           r:   rJ   zMusicFlamingoProcessor.__init__x   sd    ( 	'#' 	 	
 -.."+"A"A/"R"+"A"A/"Rr;   audio_inputs	audio_idxreturnc                 d    |d   |   }| j                   | j                  |z  z   | j                  z   S )Nnum_audio_tokens)rL   rG   rM   )r8   rR   rS   rV   s       r:   replace_audio_tokenz*MusicFlamingoProcessor.replace_audio_token   s;    '(:;IF##d&6&69I&IIDL`L```r;   c                 H    | j                   | j                  | j                  gS N)audio_token_idr!   r"   r8   s    r:   audio_token_idsz&MusicFlamingoProcessor.audio_token_ids   s!    ##T%<%<d>U>UVVr;   c                 D    t         j                  d       | j                  S )zLDeprecated alias for `audio_token_ids`; will be removed in a future release.z@`audio_ids` is deprecated; please use `audio_token_ids` instead.)loggerwarning_oncer\   r[   s    r:   	audio_idsz MusicFlamingoProcessor.audio_ids   s     	^_###r;   c                     t        d      Nz/This method is not supported for MusicFlamingo.NotImplementedErrorr8   argsr9   s      r:   apply_transcription_requestz2MusicFlamingoProcessor.apply_transcription_request       !"STTr;   c                     t        d      Nz5MusicFlamingo does not need to overwrite this method.rc   re   s      r:   decodezMusicFlamingoProcessor.decode       !"YZZr;   c                     t        d      rj   rc   re   s      r:   batch_decodez#MusicFlamingoProcessor.batch_decode   rl   r;   c                     t        d      rb   rc   re   s      r:   "_strip_assistant_prefix_and_quotesz9MusicFlamingoProcessor._strip_assistant_prefix_and_quotes   rh   r;   )Nz<sound>z<|sound_bos|>z<|sound_eos|>i  )r<   r=   r>   rJ   r1   r@   strrW   propertyr\   r`   rg   rk   rn   rp   __classcell__rQ   s   @r:   rD   rD   v   s{     ''SBa a a a W W $ $
U[[Ur;   rD   c                      | j                   g | j                  d d dd } | j                  d      \  }}t        j                  | |fd      } | j                  d      S )Nr   dim)reshapeshapeunbindtorchstackflatten)xx1x2s      r:   rotate_halfr      sa    		'1773B<''Q'AXX"XFBbS"I2&A99R=r;   c                 V   | j                   }| j                  t        j                        } |j                  |       }|j                  |       }|j                  d   }| d|d f   }| dd |f   }||z  t        |      |z  z   }t        j                  ||fd      j                  |      S )Nrv   .rw   )dtypetor}   float64r{   r   cat)hidden_statescossinoriginal_dtyperot_dimpassthroughrotateds          r:   apply_rotary_time_embr      s    "((N!$$U]]3M
&&
C
&&
CiimGWX.KC'M*G}W!5!;<G99g{+477GGr;   c            	       v     e Zd ZdZd	def fdZd Z ej                         de	de
dee	e	f   fd       Z xZS )
MusicFlamingoRotaryEmbeddinga  Rotary time embedding module used by MusicFlamingo checkpoints.

    This is a checkpoint-faithful integration, not a direct implementation of the RoTE formulation described in
    (Goel et al., 2024): https://arxiv.org/abs/2410.12109. It applies axial rotary embeddings over the window index
    within each audio sample and the encoder time index within each window, then modulates both axes with absolute
    timestamps in seconds.
    configc                     t         |   ||       | j                  | j                        }| j	                  d|d       y )Ndeviceposition_anglesF)
persistent)rI   rJ   _compute_position_anglesinv_freqregister_buffer)r8   r   r   r   rQ   s       r:   rJ   z%MusicFlamingoRotaryEmbedding.__init__   s?    /77F.ERr;   c                 B   t        j                  t        | j                        |j                  |j
                        }|| j                  z  dt        z  z  }|j                  d      |z  }t        j                  |dd      }|j                  |j
                        S )Nr   r   r   rv   rw   )r   )
r}   aranger@   max_seq_len_cachedr   r   r   	unsqueezerepeat_interleaver   )r8   r   	positionsr   s       r:   r   z5MusicFlamingoRotaryEmbedding._compute_position_angles   s    LLT%<%<!=hoo]e]k]kl	 7 771r6B	#--b1H<11/1"M!!!77r;   
timestampsseq_lenrT   c                    |dddf   j                  | j                  j                  | j                  j                        }| j                  j
                  dz  |z  }t        j                  ||z        | j                  z  }|j                  d      | j                  z  }t        j                  |dd      }|dddddf   }| j                  d| dddddf   }t        ||      \  }}t        j                  ||fd      }| dz  t        z  j                  |      }	||	j                  d      z  }|j                         |j!                         fS )zBCompute 2D axial rotary embeddings for window and time dimensions.Nr   r      rv   r   rw   )r   r   r   r   r   r#   r}   roundr   r   r   r   r   r   r   r   r   )
r8   r   r   window_startswindow_durationwindow_positionswindow_freqs
time_freqsfreqsangles
             r:   forwardz$MusicFlamingoRotaryEmbedding.forward   s5   
 #1a4(++4==3G3Gt}}ObOb+c++66:WD ;;}'FG$JaJaa'11"5E..|QBG $AtQJ/))(73D!QJ?
#4\:#N j		<4"=q2%))%0++yy{EIIK''r;   rY   )r<   r=   r>   r?   r    rJ   r   r}   no_gradr   r@   tupler   rs   rt   s   @r:   r   r      sV    S2 S
8 U]]_(& (3 (5;P ( (r;   r   c                   >    e Zd ZdZ ej
                         d        Zy)MusicFlamingoPreTrainedModelNc                     t        j                  | |       t        |t              r<|j	                  |j
                        }t        j                  |j                  |       y y rY   )	r   _init_weightsr/   r   r   r   initcopy_r   )r8   modulebuffer_values      r:   r   z*MusicFlamingoPreTrainedModel._init_weights   sJ    %%dF3f:;!::6??KLJJv--|< <r;   )r<   r=   r>   _no_split_modulesr}   r   r   r.   r;   r:   r   r      s"    U]]_= =r;   r   c                       e Zd Zy) MusicFlamingoModelOutputWithPastN)r<   r=   r>   r.   r;   r:   r   r      s    r;   r   c                       e Zd Zdef fdZdej                  dej                  dedej                  fdZ	e
 ed	      d
ej                  dej                  dej                  dee   deez  f
d              Ze
e	 	 	 	 	 	 	 	 ddej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dej                  dz  dedz  dee   fd              Z xZS )MusicFlamingoModelr   c                 D    t         |   |       t        |      | _        y rY   )rI   rJ   r   pos_embr8   r   rQ   s     r:   rJ   zMusicFlamingoModel.__init__  s     3F;r;   	input_idspost_lengthsmax_post_lengthrT   c                    || j                   j                  k(  }t        j                  t        j                  j
                  j                  |j                         dd      d      }t        j                  |dk(        \  }}t        j                  |dk(        \  }}||z
  j                  t        j                        }	|j                         }
|j                         }t        |
|k(  d|
 d|        | j                   j                  d	z  }t        j                  ||j                  t        j                   
      |z  }t        j"                  t        j$                  d|j                        t        j&                  |d      d d g      }t        j&                  |	d      }t        j(                  ||d      }t        j(                  |t        j                  |	j*                  d   |j                              }t        j                  |j*                  d   |j                        ||   z
  }|j-                  d      |z  |z  |z   S )N)   r   r   )valuer   rw   rv   z6Audio features and audio tokens do not match, tokens: z, features: r   r   r   T)right)r   rZ   r}   diffnn
functionalpadr@   wherer   longsumr   r#   r   r   float32r   zeroscumsumsearchsortedr{   r   )r8   r   r   r   audio_token_maskr   _startsendssample_lengthsn_audio_tokensn_audio_featuresaudio_embed_frame_stepframe_offsetscumsum_postcumsum_samplessample_indicessample_start_rowswindow_indicess                      r:   _build_audio_timestampsz*MusicFlamingoModel._build_audio_timestamps  s    %(B(BBzz%((--112B2F2F2H&XY1Z`abKK	*	6++dbj)4-++EJJ7)--/'++-..D^DTT`aq`rs	
 "&!=!=!ALL1D1DEMMZ]ss 	
 iiQ|7J7J!KU\\ZflmMnorprMs tun!<++NKtT "..ELL)=)=a)@I\I\]
 LL++A.|7J7JKN_`nNoo 	
 ''*_<?UUXeeer;   zThis method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector.custom_introinput_featuresinput_features_maskr9   c                     | j                   |f|dd|}|j                  }| j                   j                  |j                  d      j	                  t
        j                              \  }}| j                  |||j                  d         }	| j                  |	j	                  |j                        |j                  d         \  }
}t        ||
|      }| j                  |      }t        j                  |j                  d   |j                        dddf   |dddf   k  }||j	                  |j                           |_        |S )	az  
        input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`):
            Mask to avoid performing attention on padded feature indices.
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Token ids containing the audio token ID placeholders, for reconstructing rotary time embedding timestamps.
        T)r   return_dictrv   ry   )r   r   r   N)audio_towerlast_hidden_state _get_feat_extract_output_lengthsr   r   r}   r   r   r{   r   r   r   multi_modal_projectorr   pooler_output)r8   r   r   r   r9   audio_outputr   r   r   audio_timestampsr   r   audio_embeds
valid_masks                 r:   get_audio_featuresz%MusicFlamingoModel.get_audio_features1  sL   " (t''
 3
 	
 %66**KKL_LcLcdfLgLjLjkpkukuLvw<77	<Q^QdQdegQhi<< 0 3 3M4H4H IS`SfSfgiSj<kS-mS#F11-@ \\,"4"4Q"7@S@STUY[\U\]`lmnptmt`uu
%1*--@S@S2T%U"r;   Nattention_maskposition_idspast_key_valuesinputs_embeds	use_cachec	           	         | | j                         |      }d}
|`|^| j                  |||d      j                  }
| j                  |||
      }|j	                  ||
j                  |j                              } | j                  d|||||d|	}t        |j                  |j                  |j                  |j                  |
      S )z
        input_features_mask (`torch.Tensor` of shape `(batch_size, feature_sequence_length)`):
            Mask to avoid performing attention on padding feature indices.
        NT)r   r   )r   audio_features)r   r   r   r   r   )r   r   r   
attentionsaudio_hidden_statesr.   )get_input_embeddingsr   r   get_placeholder_maskmasked_scatterr   r   language_modelr   r   r   r   r   )r8   r   r   r   r   r   r   r   r   r9   r   special_audio_maskoutputss                r:   r   zMusicFlamingoModel.forwardU  s   $  7D557	BM%)*?22 3yVZ 3 m 
 "&!:!:| "; " *889K\__]j]q]qMrsM%$%% 
')%+
 
 0%77#33!//)) ,
 	
r;   )NNNNNNNN)r<   r=   r>   r    rJ   r}   
LongTensorr@   FloatTensorr   r   r   r   r   r   r   r   r   r
   boolr   rs   rt   s   @r:   r   r     s   <2 <'f##'f &&'f 	'f
 
		'fR  w)) #\\ ##	
 +, 
+	+ @  .23737.204(,26!%.
##d*.
 ))D0.
 #\\D0	.

 t+.
 &&-.
 .
 ((4/.
 $;.
 +,.
  .
r;   r   z
    The MusicFlamingo model which consists of a fine-tuned Whisper encoder, rotary time embedding, a multi-modal projector, and a Qwen2 language model.
    r   c                   *     e Zd Zdef fdZd Z xZS )%MusicFlamingoForConditionalGenerationr   c                 d    t         |   |       t        |      | _        | j	                          y rY   )rI   rJ   r   model	post_initr   s     r:   rJ   z.MusicFlamingoForConditionalGeneration.__init__  s&     '/
r;   c                 @     | j                   j                  |||fi |S rY   )r  r   )r8   r   r   r   r9   s        r:   r   z8MusicFlamingoForConditionalGeneration.get_audio_features  s%    ,tzz,,^=PR[f_effr;   )r<   r=   r>   r    rJ   r   rs   rt   s   @r:   r  r    s    2 
gr;   r  )r    rD   r  r   r   )7dataclassesr   mathr   huggingface_hub.dataclassesr   r}   r   r    r	   r   cache_utilsr
   configuration_utilsr   modeling_outputsr   modeling_utilsr   processing_utilsr   utilsr   r   r   r   r   r   +audioflamingo3.configuration_audioflamingo3r   &audioflamingo3.modeling_audioflamingo3r   r   r   r   (audioflamingo3.processing_audioflamingo3r   autor   moonshine.modeling_moonshiner   
get_loggerr<   r^   r    rD   r   r   r   r   r   r   r  __all__r.   r;   r:   <module>r     s7    "  . + &   3 : - &  O  O ! C 
		H	%  :;:7. :7  <:7z ;U4 ;U ;U|
H'(#; '(T=#@ = 	'H 	 	B
, B
J 
g,R g
gr;   