
    ^j                     .   d dl mZ d dlmZ d dlZd dlZd dlmZ d dl	mc m
Z d dlmZ ddlmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZmZmZ ddlm Z  ddl!m"Z"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z* ddl+m,Z, ddl-m.Z.m/Z/m0Z0m1Z1m2Z2 ddl3m4Z4m5Z5m6Z6m7Z7m8Z8 ddl9m:Z:  ed      e G d de,                    Z;ee G d de                    Z<ee G d de                    Z=ee G d de                    Z> G d d e0      Z? G d! d"e&      Z@ G d# d$e.      ZA G d% d&e/      ZB G d' d(e6      ZC G d) d*e5      ZD G d+ d,e7      ZE G d- d.e4      ZF G d/ d0e*      ZG G d1 d2e)      ZH G d3 d4e(      ZI G d5 d6ej                        ZK G d7 d8ej                        ZL G d9 d:ej                        ZM G d; d<ej                        ZN G d= d>ej                        ZO G d? d@ej                        ZP G dA dBe:      ZQ edCD       G dE dFeQ             ZRg dGZSy)H    )Callable)	dataclassN)strict   )initialization)Cache)PreTrainedConfig)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)ModelOutputTransformersKwargsauto_docstringcan_return_tuple)maybe_autocast   )CONFIG_MAPPING
AutoConfig	AutoModel)CLIPMLP)
DacEncoderDacEncoderBlockDacResidualUnit)LlamaConfig)LlamaAttentionLlamaDecoderLayerLlamaRotaryEmbeddingapply_rotary_pos_embeager_attention_forward)"Qwen2_5OmniAntiAliasedActivation1dQwen2_5OmniDownSample1dQwen2_5OmniSnakeBetaQwen2_5OmniUpSample1dkaiser_sinc_filter1d)VoxtralPreTrainedModelzHKUSTAudio/xcodec2-hf)
checkpointc                       e Zd ZU dZdZdeiZdZee	d<   dZ
ee   eedf   z  e	d<   d	Zeez  d	z  e	d<   d
Zee	d<   dZee	d<   dZee	d<   dZee   eedf   z  e	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<   dZee	d<    e       Z e       Z e       Z e       Z  e       Z! e       Z" e       Z# e       Z$ fdZ%e&defd        Z'e&defd!       Z( xZ)S )"Xcodec2Configa  
    downsampling_ratios (`list[int]`, *optional*, defaults to `[2, 2, 4, 4, 5]`):
        Ratios for downsampling in the encoder.
    semantic_model_config (`Union[Dict, Wav2Vec2BertConfig]`, *optional*):
        An instance of the configuration object for the semantic (Wav2Vec2BertConfig) model.
    quantization_dim (`int`, *optional*, defaults to 2048):
        Dimension for the vector quantization codebook.
    quantization_levels (`list[int]`, *optional*, defaults to `[4, 4, 4, 4, 4, 4, 4, 4]`):
        Levels for the vector quantization codebook.

    Example:

    ```python
    >>> from transformers import Xcodec2Config, Xcodec2Model

    >>> # Initializing configuration
    >>> configuration = Xcodec2Config()

    >>> # Initializing a model (with random weights) from the configuration
    >>> model = Xcodec2Model(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```xcodec2semantic_model_config0   encoder_hidden_size)r   r      r-      .downsampling_ratiosNi>  sampling_rateg?activation_dropouti   quantization_dim)r-   r-   r-   r-   r-   r-   r-   r-   quantization_levelsi   hidden_sizei   intermediate_size   num_attention_headsnum_key_value_heads   num_hidden_layers@   head_dimmax_position_embeddingsc                 @   t        | j                  t              rT| j                  j                  dd      | j                  d<   t	        | j                  d      di | j                  | _        n | j                  t	        d   d      | _        t        |   di | y )N
model_typezwav2vec2-bertr6   )r:    )
isinstancer*   dictgetr   super__post_init__)selfkwargs	__class__s     v/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/xcodec2/modular_xcodec2.pyrE   zXcodec2Config.__post_init__n   s    d00$77;7Q7Q7U7UVbds7tD&&|4)78R8RS_8`)a *,,*D& ''/)7)H[])^D&''    returnc                 R    t        t        j                  | j                              S N)intnpprodr/   rF   s    rI   
hop_lengthzXcodec2Config.hop_lengthy   s    277433455rJ   c                      | j                   dz  S )Nr-   )rR   rQ   s    rI   n_fftzXcodec2Config.n_fft}   s    ""rJ   )*__name__
__module____qualname____doc__r?   r   sub_configsr,   rN   __annotations__r/   listtupler*   rB   r	   r0   r1   floatr2   r3   r4   r5   r7   r8   r:   r<   r=   AttributeError
vocab_sizebos_token_ideos_token_idpretraining_tpmlp_bias	use_cachebase_model_tp_planbase_model_pp_planrE   propertyrR   rT   __classcell__rH   s   @rI   r(   r(   8   sZ   2 J*J7K!!7FcU38_4F<@4"22T9@M3 ## c 7OcU38_4OK!s!!!!!sHc#'S'!J!#L!#L#%NH I')')	( 6C 6 6 #s # #rJ   r(   c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ
ej                  dz  ed<   dZej                  dz  ed<   y)Xcodec2OutputaL  
    audio_values (`torch.FloatTensor` of shape `(batch_size, 1, sequence_length)`, *optional*):
        Decoded audio waveform values in the time domain, obtained using the decoder
        part of Xcodec2. These represent the reconstructed audio signal.
    audio_codes (`torch.LongTensor` of shape `(batch_size, 1, codes_length)`, *optional*):
        Discrete code embeddings computed using `model.encode`. These are the quantized
        representations of the input audio used for further processing or generation.
    latents (`torch.Tensor` of shape `(batch_size, dimension, time_steps)`):
        Quantized continuous representation of input's embedding.
    audio_codes_mask (`torch.int32` of shape `(batch_size, 1, codes_length)`, *optional*):
        Downsampled `padding_mask` for indicating valid audio codes in `audio_codes`.
    Naudio_valuesaudio_codeslatentsaudio_codes_mask)rU   rV   rW   rX   rl   torchFloatTensorrZ   rm   
LongTensorrn   Tensorro   r@   rJ   rI   rk   rk      sZ     .2L%##d*1+/K!!D(/#'GU\\D ',0ellT)0rJ   rk   c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ
ej                  dz  ed<   y)Xcodec2EncoderOutputat  
    audio_codes (`torch.LongTensor` of shape `(batch_size, 1, codes_length)`, *optional*):
        Discrete code embeddings computed using `model.encode`. These represent
        the compressed, quantized form of the input audio signal that can be
        used for storage, transmission, or generation.
    latents (`torch.Tensor` of shape `(batch_size, dimension, time_steps)`):
        Quantized continuous representation of input's embedding.
    audio_codes_mask (`torch.int32` of shape `(batch_size, 1, codes_length)`, *optional*):
        Downsampled `padding_mask` for indicating valid audio codes in `audio_codes`.
    Nrm   rn   ro   )rU   rV   rW   rX   rm   rp   rr   rZ   rn   rs   ro   r@   rJ   rI   ru   ru      sE    	 ,0K!!D(/#'GU\\D ',0ellT)0rJ   ru   c                   :    e Zd ZU dZdZej                  dz  ed<   y)Xcodec2DecoderOutputa=  
    audio_values (`torch.FloatTensor` of shape `(batch_size, 1, segment_length)`, *optional*):
        Decoded audio waveform values in the time domain, obtained by converting
        the discrete codes back into continuous audio signals. This represents
        the reconstructed audio that can be played back.
    Nrl   )rU   rV   rW   rX   rl   rp   rq   rZ   r@   rJ   rI   rw   rw      s     .2L%##d*1rJ   rw   c                       e Zd Zy)Xcodec2RotaryEmbeddingNrU   rV   rW   r@   rJ   rI   ry   ry          rJ   ry   c                   $     e Zd Zdef fdZ xZS )
Xcodec2MLPconfigc                     t         |   |       t        j                  |j                  |j
                  d      | _        t        j                  |j
                  |j                  d      | _        y )NF)bias)rD   __init__nnLinearr4   r5   fc1fc2rF   r~   rH   s     rI   r   zXcodec2MLP.__init__   sR     99V//1I1IPUV99V55v7I7IPUVrJ   rU   rV   rW   r(   r   rh   ri   s   @rI   r}   r}      s    W} W WrJ   r}   c                        e Zd Zdedef fdZ	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  de	dz  d	e
e   d
eej                  ej                  f   fdZ xZS )Xcodec2Attentionr~   	layer_idxc                 4    t         |   ||       d| _        y )NF)rD   r   	is_causalrF   r~   r   rH   s      rI   r   zXcodec2Attention.__init__   s    +rJ   Nhidden_statesposition_embeddingsattention_maskpast_key_valuesrG   rK   c                    |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }	| j                  |      j                  |      j	                  dd      }
|\  }}t        ||	||d      \  }}	| |j                  |	|
| j                        \  }	}
t        j                  | j                  j                  t              } || ||	|
|f| j                  sdn| j                   | j"                  d|\  }} |j$                  g |d j'                         }| j)                  |      }||fS )N   r   )unsqueeze_dim        )dropoutscaling)shaper<   q_projview	transposek_projv_projr   updater   r
   get_interfacer~   _attn_implementationr   trainingattention_dropoutr   reshape
contiguouso_proj)rF   r   r   r   r   rG   input_shapehidden_shapequery_states
key_statesvalue_statescossinattention_interfaceattn_outputattn_weightss                   rI   forwardzXcodec2Attention.forward   s    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST&S $8jRUWZjk#l j&'6'='=j,X\XfXf'g$J(?(M(MKK,,.E)
 %8	%
  $}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((rJ   )NNN)rU   rV   rW   r(   rN   r   rp   rs   r\   r   r   r   r   rh   ri   s   @rI   r   r      s    }   IM.2(,))||)) #5<<#=>E)) t+	))
 )) +,)) 
u||U\\)	*))rJ   r   c                       e Zd Zy)Xcodec2DecoderLayerNrz   r@   rJ   rI   r   r      r{   rJ   r   c                       e Zd Zy)Xcodec2SnakeBetaNrz   r@   rJ   rI   r   r      r{   rJ   r   c                       e Zd Zd Zy)Xcodec2DownSample1dc                 .   |j                   d   }t        j                  || j                  | j                  fd      }t        j
                  || j                  j                  |j                        j                  |dd      | j                  |      }|S )Nr   	replicatemoder   stridegroups)r   Fpadpad_left	pad_rightconv1dfiltertodtypeexpandr   )rF   r   channelsouts       rI   r   zXcodec2DownSample1d.forward   sy     &&q)mdmmT^^-LS^_hhKKNN=../66xRH;;
 
rJ   NrU   rV   rW   r   r@   rJ   rI   r   r      s    
rJ   r   c                       e Zd Zd Zy)Xcodec2UpSample1dc           	         |j                   d   }t        j                  || j                  | j                  fd      }| j                  t        j                  || j
                  j                  |j                        j                  |dd      | j                  |      z  }|d| j                  | j                   f   }|S )Nr   r   r   r   r   .)r   r   r   ratioconv_transpose1dr   r   r   r   r   r   r   )rF   r   r   s      rI   r   zXcodec2UpSample1d.forward  s     &&q)mdhh-AT

Q%7%7KKNN=../66xRH;;&
 
 &c4==DNN?+J&JKrJ   Nr   r@   rJ   rI   r   r   
  s    rJ   r   c            	       :     e Zd Z	 	 	 	 ddedededef fdZ xZS )Xcodec2AntiAliasedActivation1dup_ratio
down_ratioup_kernel_sizedown_kernel_sizec                 r    t         |   |||||       t        ||      | _        t	        ||      | _        y )N)
activationr   r   r   r   )rD   r   r   upsampler   
downsample)rF   r   r   r   r   r   rH   s         rI   r   z'Xcodec2AntiAliasedActivation1d.__init__  sE     	!!)- 	 	
 *(NC-j:JKrJ   )r   r   r9   r9   )rU   rV   rW   rN   r   rh   ri   s   @rI   r   r     sI       "L L 	L
 L L LrJ   r   c                        e Zd Z fdZ xZS )Xcodec2ResidualUnitc                     t         |   ||       t        t        |            | _        t        t        |            | _        y )Nr   )rD   r   r   r   snake1snake2)rF   	dimensiondilationrH   s      rI   r   zXcodec2ResidualUnit.__init__.  s8    H-4@PQZ@[\4@PQZ@[\rJ   )rU   rV   rW   r   rh   ri   s   @rI   r   r   -  s    ] ]rJ   r   c                   .     e Zd Zddededef fdZ xZS )Xcodec2EncoderBlockr~   r   stride_indexc                     t         |   |||       |j                  d|z  z  }t        t	        |dz              | _        y Nr   r   )rD   r   r,   r   r   r   )rF   r~   r   r   r   rH   s        rI   r   zXcodec2EncoderBlock.__init__5  sA    6..L@	4@PQZ^_Q_@`arJ   )r   r   )rU   rV   rW   r(   rN   r   rh   ri   s   @rI   r   r   4  s'    b} bc bS b brJ   r   c                   $     e Zd Zdef fdZ xZS )Xcodec2Encoderr~   c                     t         |   |       |j                  dt        |j                        z  z  }t        t        |            | _        y r   )rD   r   r,   lenr/   r   r   r   )rF   r~   d_modelrH   s      rI   r   zXcodec2Encoder.__init__<  sC     ,,qC8R8R4S/SS4@PQX@YZrJ   r   ri   s   @rI   r   r   ;  s    [} [ [rJ   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )Xcodec2ResNetBlockr~   c                    t         |           t        j                  d|j                  dd      | _        t        j                         | _        t        j                  |j                  |j                  ddd      | _	        t        j                  d|j                  dd      | _
        t        j                         | _        |j                  | _        t        j                  |j                  |j                  ddd      | _        y )N    ư>T)
num_groupsnum_channelsepsaffiner   r   )kernel_sizer   padding)rD   r   r   	GroupNormr4   norm1SiLUactivation1Conv1dconv1norm2activation2r1   conv2r   s     rI   r   zXcodec2ResNetBlock.__init__C  s    \\Rf>P>PVZcgh
779YYv1163E3EST]^hij
\\Rf>P>PVZcgh
779"(";";YYv1163E3EST]^hij
rJ   r   rK   c                    |j                  dd      }|}| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }t        j                  j                  || j                  | j                        }| j                  |      }||z   j                  dd      S )Nr   r   )pr   )r   r   r   r   r   r   r   
functionalr   r1   r   r   rF   r   residuals      rI   r   zXcodec2ResNetBlock.forwardM  s    %//15 

=1((7

=1

=1((7--mt?V?Vaeanan-o

=1(33Aq99rJ   	rU   rV   rW   r(   r   rp   rs   r   rh   ri   s   @rI   r   r   B  s,    k} k
:U\\ 
:ell 
:rJ   r   c                        e Zd ZdZdef fdZddZdej                  dej                  fdZ	ddej                  d	e
dej                  fd
Zdej                  deej                  ej                  f   fdZ xZS )Xcodec2FiniteScalarQuantizationa!  
    Finite Scalar Quantization (FSQ) module that quantizes continuous latent representations into discrete codes.
    Original code: https://github.com/lucidrains/vector-quantize-pytorch/blob/353d46027888dfb140c3c65a67a7356f1492d71d/vector_quantize_pytorch/finite_scalar_quantization.py#L64

    Original modeling uses `ResidualFSQ` with a single quantizer: https://huggingface.co/HKUSTAudio/xcodec2/blob/main/vq/codec_decoder_vocos.py#L389
    But we can directly use FSQ since a main feature of Xcodec2 is that it uses a single codebook.
    r~   c                     t         |           t        |j                        | _        | j	                         \  }}}| j                  d|d       | j                  d|d       | j                  d|d       y )NlevelsF
persistentbasiscodebook)rD   r   r[   r3   _compute_buffersregister_buffer)rF   r~   r	  r  r  rH   s        rI   r   z(Xcodec2FiniteScalarQuantization.__init__c  su    #'(B(B#C "&"7"7"9xXv%@We>ZeDrJ   c                    t        j                  | j                  t         j                  |      }t        j                  t        j                  dg| j                  dd z   |      dt         j                        }t        j
                  t        t        j                  | j                              |      j                  d      }||z  |z  }|dz  }||z
  |z  }|||fS )	zFCompute the levels, basis, and codebook buffers for the FSQ quantizer.)r   devicer   Nr   r  r   )dimr   r   )
rp   tensorr3   int32cumprodarangerN   rO   rP   	unsqueeze)rF   r  r	  r  indiceslevel_indices
half_widthr  s           rI   r  z0Xcodec2FiniteScalarQuantization._compute_buffersk  s    d66ekkRXYLL!t77<<VLRS[`[f[f
 ,,s2774+C+C#DEfU__`bc E)V3q[
!J.*<uh&&rJ   r  rK   c                     |j                  d      }|| j                  z  | j                  z  }| j                  dz  }||z
  |z  }|S )z`
        Convert integer codebook indices to normalized per-dimension codes in [-1, 1].
        r   r   )r  r  r	  )rF   r  r  r  codess        rI   _indices_to_codesz1Xcodec2FiniteScalarQuantization._indices_to_codesw  sM     ##B' DJJ.$++=[[A%
+z9rJ   r   r   c                     | j                   dz
  d|z   z  dz  }t        j                  | j                   dz  dk(  dd      }||z  j                         }||z   j	                         |z  |z
  S )a  
        Constrain `hidden_states` to the valid quantization range for each dimension.

        Uses a scaled tanh to soft-clip values into the interval
        $[-(L-1)/2, (L-1)/2]$ (offset by 0.5 for even-level dimensions), where $L$ is
        the number of quantization levels. The small `eps` margin prevents values from
        saturating exactly at the boundary, which would zero out gradients.

        Args:
            hidden_states (`torch.Tensor`): Continuous input to be bounded.
            eps (`float`, *optional*, defaults to `1e-3`):
                Small margin added to the level range to avoid gradient saturation at boundaries.

        Returns:
            `torch.Tensor`: Bounded values in the valid quantization range.
        r   r   r         ?r   )r	  rp   whereatanhtanh)rF   r   r   
half_rangeoffsetshifts         rI   boundz%Xcodec2FiniteScalarQuantization.bound  so    " kkAo!c'2Q6
T[[1_13<*$++-%++-
:VCCrJ   c                 l   |j                   }t        |j                  j                  t              r/|j                  j                  dk7  r|j                  j                  nd}t        |d      5  |j                         }| j                  dz  }| j                  |      }|j                         }|||z
  j                         z   }||z  }||z  |z   }|| j                  z  j                  d      j                  t        j                        }d d d        j                  |      fS # 1 sw Y   xY w)NmpscpuF)device_typeenabledr   r   r  )r   rA   r  typestrr   r]   r	  r'  rounddetachr  sumr   rp   r  )	rF   r   original_dtyper+  r  roundedr  code_scaledr  s	            rI   r   z'Xcodec2FiniteScalarQuantization.forward  s!   &,, -..33S9m>R>R>W>W[`>`   %% 	
 UC 
	M)//1M)J JJ}5M#))+G!W}%<$D$D$FFEJ&E :-;K"TZZ/444<??LG
	M xx'00
	M 
	Ms   /B D**D3rM   )gMbP?)rU   rV   rW   rX   r(   r   r  rp   rs   r  r]   r'  r\   r   rh   ri   s   @rI   r  r  Z  s    E} E
' %,, D5<< De Du|| D,1U\\ 1eELL%,,<V6W 1rJ   r  c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ xZ	S )Xcodec2ISTFTHeadz
    Head for converting decoder outputs to waveform via STFT projection and ISTFT.

    Uses custom "same" padding ISTFT from Vocos:
    https://github.com/gemelo-ai/vocos/blob/c859e3b7b534f3776a357983029d34170ddd6fc3/vocos/spectral_ops.py#L47
    r~   c                 r   t         |           t        j                  |j                  |j
                  dz         | _        |j
                  | _        |j                  | _        | j
                  | j                  z
  dz  | _        t        j                  |j
                        }| j                  d|d       y )Nr   windowFr
  )rD   r   r   r   r4   rT   linearrR   r   rp   hann_windowr  )rF   r~   r9  rH   s      rI   r   zXcodec2ISTFTHead.__init__  s    ii 2 2FLL14DE\\
 ++

T__4:""6<<0Xv%@rJ   r   rK   c                 .   | j                  |      j                  dd      }|j                  dd      \  }}|j                         }|j                         }t	        j
                  |      j                  d      }|t	        j
                  d|z        z  }t        j                  j                  || j                  dd      }|| j                  d d d d f   z  }|j                  d	   }|dz
  | j                  z  | j                  z   }t        j                  |d|fd| j                  fd| j                  f
      d d dd| j                  | j                   f   }	t        j                  | j                  j!                         j#                  d|d	      j                  dd      d|fd| j                  fd| j                  f
      j%                         | j                  | j                    }
|
j                  d      }
|	|
z  }	|	j'                  d      S )Nr   r   r-  g      Y@)maxy              ?backward)r  normr   )output_sizer   r   r   gdy=)min)r:  r   chunkr]   rp   expclampfftirfftrT   r9  r   rR   r   foldr   squarer   squeezer  )rF   r   	stft_pred	magnitudephasespectrogram_complextime_frames
num_framesr@  audiowindow_envelopes              rI   r   zXcodec2ISTFTHead.forward  s   KK.88A>	$??1!?4	5OO%	IIi(..3.7	'%))BJ*?? iioo&94::1S]o^!DKKa$>>(..r2
!A~84::EK(DJJt'	

 Q4<<4<<-/
/1 &&KK '':r:DDQJK(DJJt'	

 ')DLLDLL=2 *//E/:'q!!rJ   
rU   rV   rW   rX   r(   r   rp   rs   r   rh   ri   s   @rI   r7  r7    s1    A} A!"U\\ !"ell !"rJ   r7  c                        e Zd Zdef fdZdej                  dej                  fdZdej                  deej                  ej                  f   fdZ	 xZ
S )Xcodec2Quantizerr~   c                 "   t         |           t        |      | _        t	        j
                  |j                  t        |j                              | _	        t	        j
                  t        |j                        |j                        | _
        y rM   )rD   r   r  	quantizerr   r   r2   r   r3   
project_inproject_outr   s     rI   r   zXcodec2Quantizer.__init__  sa    8@))F$;$;SA[A[=\]99S)C)C%DfF]F]^rJ   r  rK   c                 x    |j                  d      }| j                  j                  |   }| j                  |      S Nr   )rI  rV  r  rX  )rF   r  r  s      rI   
from_codeszXcodec2Quantizer.from_codes  s5    //"%''0&&rJ   r   c                    | j                  |      }|j                  }| j                  j                  |      }| j                  |      \  }}| j	                  |j                  |            }|j                  d      }||fS rZ  )rW  r   rV  r'  rX  r   r  )rF   r   r3  quantized_outr  s        rI   r   zXcodec2Quantizer.forward  sx    6&,,,,];!%!>w(()9)9.)IJ##B'g%%rJ   )rU   rV   rW   r(   r   rp   rs   r[  r\   r   rh   ri   s   @rI   rT  rT    sU    _} _'%,, '5<< '
&U\\ &eELL%,,<V6W &rJ   rT  c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ xZ	S )Xcodec2DecoderzVVocos-based decoder with ResNet, Transformer, and ISTFT head for audio reconstruction.r~   c           	         t         |           t        j                  |j                  |j
                  j                  z   |j                        | _        t        j                  |j                  |j                  dd      | _        t        j                  t        |      t        |      g      | _        |j                  | _        t        |      | _        t        j                  t        |j                         D cg c]  }t#        ||       c}      | _        t        j                  t        |      t        |      g      | _        t        j(                  |j                  d      | _        t-        |      | _        y c c}w )N   r   )r   r   )r~   r   )r   )rD   r   r   r   r4   r*   fcr   embed
ModuleListr   	prior_netr7   ry   
rotary_embranger:   r   layerspost_net	LayerNormr?  r7  headr   s      rI   r   zXcodec2Decoder.__init__  s   ))F..1M1M1Y1YY[a[m[mnYYv1163E3EST^_`
(:6(BDVW]D^'_`#)#=#= 0?mmEJ6KcKcEde	 3e
 '9&'ACUV\C]&^_LL!3!3>	$V,		 fs   >Fr   rK   c                    | j                  |      }|j                  dd      }| j                  |      }|j                  dd      }| j                  D ]
  } ||      } t	        j
                  | j                  |j                        j                  d      }| j                  ||      }| j                  D ]  } ||fd|i|} | j                  D ]
  } ||      } | j                  | j                  |            S )Nr   r   r  r   r   )rb  r   rc  re  rp   r  r7   r  r  rf  rh  ri  rk  r?  )rF   r   rG   layerposition_idsr   s         rI   r   zXcodec2Decoder.forward	  s    .%//15

=1%//15 ^^ 	1E!-0M	1 ||D$<$<]EYEYZddefg"oom\J[[ 	dE!-cEXc\bcM	d ]] 	1E!-0M	1 yy=122rJ   rR  ri   s   @rI   r_  r_    s-    `-} -3U\\ 3 3rJ   r_  c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )Xcodec2SemanticAdapterr~   c                    t         |           t        j                  |j                  j
                  |j                  j
                  ddd      | _        t        j                         | _        t        j                  |j                  j
                  |j                  j
                  ddd      | _	        t        j                         | _
        t        j                  |j                  j
                  |j                  j
                  ddd      | _        t        j                  |j                  j
                  |j                  j
                  ddd      | _        y )Nr   r   F)in_channelsout_channelsr   r   r   T)r   r   r   )rD   r   r   r   r*   r4   r   ReLUact1r   act2conv3conv4r   s     rI   r   zXcodec2SemanticAdapter.__init__%  s
   YY44@@55AA

 GGI	YY((44((44

 GGI	YY((44((44

 YY44@@55AA

rJ   r   rK   c                     | j                  |      }| j                  |      }|}| j                  |      }| j                  |      }| j	                  |      }||z   }| j                  |      }|S rM   )r   ru  r   rv  rw  rx  r  s      rI   r   zXcodec2SemanticAdapter.forwardF  sk    

=1		-0 

=1		-0

=1%0

=1rJ   r  ri   s   @rI   rp  rp  $  s+    
} 
B	U\\ 	ell 	rJ   rp  c                   (    e Zd ZdZdZdZeedZd Zy)Xcodec2PreTrainedModelr)   input_values)rP  )r   
attentionsc                    t        j                  |       t        |t              r?t	        j
                  |j                         t	        j
                  |j                         y t        |t              r@t        j                  |j                        }t	        j                  |j                  |       y t        |t              r|j                  |j                   j"                        \  }}}t	        j                  |j                   |       t	        j                  |j$                  |       t	        j                  |j&                  |       y t        |t(              rRt+        d|j,                  z  d|j,                  z  |j.                        }t	        j                  |j0                  |       y t        |t2              rLt+        |j4                  |j6                  |j.                        }t	        j                  |j0                  |       y y )Nr  r   g333333?)r   _init_weightsrA   r   initzeros_alphabetar7  rp   r;  rT   copy_r9  r  r  r	  r  r  r  r   r$   r   r   r   r   cutoffr  )rF   moduler9  r	  r  r  filter_tensors          rI   r  z$Xcodec2PreTrainedModel._init_weights[  sV   %%f-f./KK%KK$ 01&&v||4FJJv}}f- ?@&,&=&=V]]EYEY&=&Z#FE8JJv}}f-JJv||U+JJv1 120v||1CS6<<EWY_YkYklMJJv}}m4 340@Q@QSYSeSefMJJv}}m4 5rJ   N)	rU   rV   rW   base_model_prefixmain_input_nameinput_modalitiesr   _can_record_outputsr  r@   rJ   rI   r{  r{  R  s&    !$O!,)
5rJ   r{  z!Xcodec2 neural audio codec model.)custom_introc                       e Zd ZeZdef fdZee	 	 	 ddej                  dej                  dej                  dz  dej                  dz  de
d	ee   d
eez  fd              Zee	 	 ddej                  dz  dej                  dz  d	ee   d
eez  fd              Zee	 	 	 ddej                  dej                  dej                  dz  dej                  dz  de
d	ee   d
eez  fd              Z xZS )Xcodec2Modelr~   c                    t         |   |       |j                  | _        t        j                  |j
                        | _        t        |      | _        t        |      | _
        t        j                  |j                  |j
                  j                  z   |j                  |j
                  j                  z         | _        t        |      | _        t#        |      | _        | j'                          y rM   )rD   r   rR   r   from_configr*   semantic_encoderrp  semantic_adapterr   acoustic_encoderr   r   r4   
fc_encoderrT  rV  r_  acoustic_decoder	post_initr   s     rI   r   zXcodec2Model.__init__t  s      ++ ) 5 5f6R6R S 6v > .v 6))!=!=!I!II!=!=!I!II
 *&1 .v 6rJ   Nr|  input_featurespadding_maskinput_features_maskoutput_latentsrG   rK   c                    t        j                         5  | j                  ||      }ddd       j                  j	                  dd      }| j                  |      }| j                  |      }	t        j                  ||	gd      }
| j                  |
j	                  dd            }
| j                  |
      \  }}|j	                  dd      }|j	                  dd      }d}|~|j                  dd      }|| j                  z  }t        j                  |j                  d   |j                  	      j                  dd      }||k  j!                  |j"                        }t%        ||r||
      S d|
      S # 1 sw Y   TxY w)a  
        input_values (`torch.Tensor` of shape `(batch_size, 1, sequence_length)`):
            Input audio waveform.
        input_features (`torch.Tensor` of shape `(batch_size, mel_bins, time_steps)`):
            Input audio mel spectrogram for semantic encoding.
        padding_mask (`torch.Tensor` of shape `(batch_size, 1, sequence_length)`):
            Padding mask used to pad `input_values`.
        input_features_mask (`torch.Tensor` of shape `(batch_size, time_steps)`, *optional*):
            Attention mask for the spectrogram input to the semantic encoder. `1` for valid frames, `0` for padding.
        output_latents (`bool`, *optional*, defaults to `False`):
            Whether to return the continuous latent representation from the quantizer.
        )r   Nr   r   r-  r   T)r  keepdimr  )rm   rn   ro   )rp   no_gradr  last_hidden_stater   r  r  catr  rV  r2  rR   r  r   r  r   r   r   ru   )rF   r|  r  r  r  r  rG   semantic_outputsemantic_hidden_statesacoustic_hidden_statesr   rn   rm   ro   audio_lengthtoken_lengthidxs                    rI   encodezXcodec2Model.encode  s   2 ]]_ 	h"33NSf3gO	h!0!B!B!L!LQPQ!R!%!6!67M!N "&!6!6|!D		#9;Q"RXYZ(?(?1(EF  $~~m<##Aq)!++Aq1  #'++D+AL'4??:L,,{004\=P=PQVVWXZ\]C #l 266|7I7IJ##-G-
 	
37-
 	
/	h 	hs   E;;Frm   rn   c                     ||t        d      |,| j                  j                  |j                  dd            }n|j                  dd      } | j                  |fi |}t        |      S )a3  
        audio_codes (`torch.LongTensor`  of shape `(batch_size, 1, codes_length)`):
            Discrete code indices computed using `model.encode`.
        latents (torch.Tensor of shape `(batch_size, dimension, time_steps)`, *optional*):
            Quantized continuous representation of input.
        z3Either `latents` or `audio_codes` must be provided.r   r   )rl   )
ValueErrorrV  r[  r   r  rw   )rF   rm   rn   rG   recon_audios        rI   decodezXcodec2Model.decode  su     ?{2RSS"nn//0E0Ea0KLG''1-G+d++G>v>#==rJ   c                    |j                   d   }| j                  ||||dd      } | j                  d	|j                  dd|d   dd|f   }	t	        |	|j
                  |r|j                  nd|j                        S )
a  
        input_values (`torch.Tensor` of shape `(batch_size, 1, sequence_length)`):
            Input audio waveform.
        input_features (`torch.Tensor` of shape `(batch_size, mel_bins, time_steps)`):
            Input audio mel spectrogram for semantic encoding.
        padding_mask (`torch.Tensor` of shape `(batch_size, 1, sequence_length)`):
            Padding mask used to pad `input_values`.
        input_features_mask (`torch.Tensor` of shape `(batch_size, time_steps)`, *optional*):
            Attention mask for the spectrogram input to the semantic encoder. `1` for valid frames, `0` for padding.
        output_latents (`bool`, *optional*, defaults to `False`):
            Whether to return the continuous latent representation from the quantizer.

        Examples:

        ```python
        >>> from datasets import load_dataset
        >>> from transformers import AutoFeatureExtractor, Xcodec2Model

        >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> audio = dataset["train"]["audio"][0]["array"]

        >>> model_id = "HKUSTAudio/xcodec2-hf"
        >>> model = Xcodec2Model.from_pretrained(model_id)
        >>> feature_extractor = AutoFeatureExtractor.from_pretrained(model_id)

        >>> inputs = feature_extractor(audio=audio, sampling_rate=feature_extractor.sampling_rate, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> audio_codes = outputs.audio_codes
        >>> audio_values = outputs.audio_values
        ```r   T)r  r  r  r  return_dict)rn   r  r   .N)rl   rm   rn   ro   r@   )r   r  r  rn   rk   rm   ro   )
rF   r|  r  r  r  r  rG   lengthencoder_outputsrl   s
             rI   r   zXcodec2Model.forward  s    V ##B'++)% 3 & 
 #t{{_?+B+BPT_X^_`abcfhoiohocop%'33/=O++4,==	
 	
rJ   )NNF)NN)rU   rV   rW   r(   config_classr   r   r   rp   rs   boolr   r   r\   ru   r  rw   r  rk   r   rh   ri   s   @rI   r  r  p  s    L}   
 -137$2
ll2
 2
 llT)	2

 #\\D02
 2
 +,2
 
%	%2
  2
h  ,0'+>\\D(> $> +,	>
 
%	%>  >. 
 -137$:
ll:
 :
 llT)	:

 #\\D0:
 :
 +,:
 
	:
  :
rJ   r  )r(   r  r{  )Tcollections.abcr   dataclassesr   numpyrO   rp   torch.nnr   torch.nn.functionalr  r   huggingface_hub.dataclassesr    r   r  cache_utilsr   configuration_utilsr	   modeling_utilsr
   r   processing_utilsr   utilsr   r   r   r   utils.genericr   autor   r   r   clip.modeling_clipr   dac.modeling_dacr   r   r   llama.configuration_llamar   llama.modeling_llamar   r   r   r   r   "qwen2_5_omni.modeling_qwen2_5_omnir    r!   r"   r#   r$   voxtral.modeling_voxtralr%   r(   rk   ru   rw   ry   r}   r   r   r   r   r   r   r   r   r   Moduler   r  r7  rT  r_  rp  r{  r  __all__r@   rJ   rI   <module>r     s5   % !      . &   3 F &  , 8 8 ( K K 3   > 23E#K E#  4E#P 
1K 1  1( 
1; 1  1" 
2; 2  2	1 	W W.)~ .)b	+ 		+ 	1 - L%G L(]/ ]b/ b[Z [: :0P1bii P1f2"ryy 2"j&ryy &,)3RYY )3X+RYY +\53 5< @A^
) ^
 B^
B FrJ   