
    ^j                        d dl Z d dlmZ d dlmZ d dlmZ d dlZd dl	Z	d dl
mZ d dlmc mZ d dl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZ ddlmZ ddlmZmZ ddl m!Z!m"Z" ddl#m$Z$ ddl%m&Z&m'Z'm(Z(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/ e(e G d de&                    Z0e(e G d de&                    Z1e(e G d de&                    Z2 G d dejf                        Z4 G d dejf                        Z5d Z6 ed       dWd!       Z7d"e	jp                  d#e9d$e	jp                  fd%Z:	 dXd&ejf                  d'e	jp                  d(e	jp                  d)e	jp                  d*e	jp                  dz  d+e;d,e;d-e$e'   fd.Z< ee7       G d/ d0ejf                               Z= ed1       G d2 d3ejf                               Z> G d4 d5e      Z? G d6 d7ejf                        Z@d8 ZA G d9 d:ejf                        ZB G d; d<ejf                        ZC G d= d>ejf                        ZD G d? d@ejf                        ZE G dA dBejf                        ZF G dC dDejf                        ZG G dE dFejf                        ZH G dG dHejf                        ZI G dI dJejf                        ZJ G dK dLejf                        ZK G dM dNejf                        ZL G dO dPejf                        ZMe( G dQ dRe"             ZN e(dST       G dU dVeN             ZOdVdRgZPy)Y    N)Callable)	dataclass)Optional)	Parameter   )initialization)ACT2FN)Cache)use_kernel_forward_from_hubuse_kernel_func_from_hubuse_kernelized_func)GradientCheckpointingLayer)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)ModelOutputTransformersKwargsauto_docstringcan_return_tuple)maybe_autocast   )	AutoModel   )Xcodec2Configc                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ
ej                  dz  ed<   dZej                  dz  ed<   y)Xcodec2OutputaL  
    audio_values (`torch.FloatTensor` of shape `(batch_size, 1, sequence_length)`, *optional*):
        Decoded audio waveform values in the time domain, obtained using the decoder
        part of Xcodec2. These represent the reconstructed audio signal.
    audio_codes (`torch.LongTensor` of shape `(batch_size, 1, codes_length)`, *optional*):
        Discrete code embeddings computed using `model.encode`. These are the quantized
        representations of the input audio used for further processing or generation.
    latents (`torch.Tensor` of shape `(batch_size, dimension, time_steps)`):
        Quantized continuous representation of input's embedding.
    audio_codes_mask (`torch.int32` of shape `(batch_size, 1, codes_length)`, *optional*):
        Downsampled `padding_mask` for indicating valid audio codes in `audio_codes`.
    Naudio_valuesaudio_codeslatentsaudio_codes_mask)__name__
__module____qualname____doc__r   torchFloatTensor__annotations__r    
LongTensorr!   Tensorr"        w/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/xcodec2/modeling_xcodec2.pyr   r   -   sZ     .2L%##d*1+/K!!D(/#'GU\\D ',0ellT)0r-   r   c                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ
ej                  dz  ed<   y)Xcodec2EncoderOutputat  
    audio_codes (`torch.LongTensor` of shape `(batch_size, 1, codes_length)`, *optional*):
        Discrete code embeddings computed using `model.encode`. These represent
        the compressed, quantized form of the input audio signal that can be
        used for storage, transmission, or generation.
    latents (`torch.Tensor` of shape `(batch_size, dimension, time_steps)`):
        Quantized continuous representation of input's embedding.
    audio_codes_mask (`torch.int32` of shape `(batch_size, 1, codes_length)`, *optional*):
        Downsampled `padding_mask` for indicating valid audio codes in `audio_codes`.
    Nr    r!   r"   )r#   r$   r%   r&   r    r'   r*   r)   r!   r+   r"   r,   r-   r.   r0   r0   C   sE    	 ,0K!!D(/#'GU\\D ',0ellT)0r-   r0   c                   :    e Zd ZU dZdZej                  dz  ed<   y)Xcodec2DecoderOutputa=  
    audio_values (`torch.FloatTensor` of shape `(batch_size, 1, segment_length)`, *optional*):
        Decoded audio waveform values in the time domain, obtained by converting
        the discrete codes back into continuous audio signals. This represents
        the reconstructed audio that can be played back.
    Nr   )r#   r$   r%   r&   r   r'   r(   r)   r,   r-   r.   r2   r2   V   s     .2L%##d*1r-   r2   c                        e Zd ZU ej                  ed<   ddef fdZe	 	 	 ddedz  de	d   de
dz  ded	ef   fd
       Z ej                         ed               Z xZS )Xcodec2RotaryEmbeddinginv_freqNconfigc                    t         |           |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |      \  }| _
        | j                  d|d       | j                  d|j                         d       y )N	rope_typedefaultr5   F
persistentoriginal_inv_freq)super__init__max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenr6   rope_parametersr8   compute_default_rope_parametersr   attention_scalingregister_bufferclone)selfr6   devicerope_init_fnr5   	__class__s        r.   r>   zXcodec2RotaryEmbedding.__init__f   s    "("@"@$*$B$B!44[A!%!E!E>>Y&.t~~>L+7V+L($(ZeD0(..2BuUr-   rH   ztorch.deviceseq_lenreturnztorch.Tensorc                    | j                   d   }t        | dd      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                        j                  |t        j                        |z  z  z  }||fS )	a  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        
rope_thetahead_dimN      ?r   r   dtype)rH   rR   )	rB   getattrhidden_sizenum_attention_headsr'   arangeint64tofloat)r6   rH   rK   basedimattention_factorr5   s          r.   rC   z6Xcodec2RotaryEmbedding.compute_default_rope_parametersv   s    & %%l3fj$/c63E3EIcIc3c U\\!S!5;;?BB&X]XcXcBdgjjk
 )))r-   c                 N   | j                   d d d d f   j                         j                  |j                  d   dd      j	                  |j
                        }|d d d d d f   j                         }t        |j
                  j                  t              r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d      5  |j                         |j                         z  j                  dd      }t        j                  ||fd	      }|j                         | j                  z  }|j                         | j                  z  }	d d d        j	                  |j                   
      	j	                  |j                   
      fS # 1 sw Y   AxY w)Nr   r   mpscpuFdevice_typeenabledr   r[   rQ   )r5   rY   expandshaperX   rH   
isinstancetypestrr   	transposer'   catcosrD   sinrR   )
rG   xposition_idsinv_freq_expandedposition_ids_expandedrb   freqsembrl   rm   s
             r.   forwardzXcodec2RotaryEmbedding.forward   sR    !MM$4-8>>@GGHZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfkUC 	5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C		5 vvAGGv$cff177f&;;;	5 	5s   BFF$NNNN)r#   r$   r%   r'   r+   r)   r   r>   staticmethodr   inttuplerY   rC   no_gradr   rt   __classcell__rJ   s   @r.   r4   r4   c   s    llV} V  '++/"*$*(* t* 
~u$	%	* *: U]]_<  <r-   r4   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )
Xcodec2MLPr6   c                 $   t         |           || _        t        |j                     | _        t        j                  |j                  |j                  d      | _
        t        j                  |j                  |j                  d      | _        y )NFbias)r=   r>   r6   r	   
hidden_actactivation_fnnnLinearrT   intermediate_sizefc1fc2rG   r6   rJ   s     r.   r>   zXcodec2MLP.__init__   sj    #F$5$5699V//1I1IPUV99V55v7I7IPUVr-   hidden_statesrL   c                 l    | j                  |      }| j                  |      }| j                  |      }|S ru   )r   r   r   rG   r   s     r.   rt   zXcodec2MLP.forward   s4    /**=9/r-   	r#   r$   r%   r   r>   r'   r+   rt   r{   r|   s   @r.   r~   r~      s,    W} WU\\ ell r-   r~   c                     | dd| j                   d   dz  f   }| d| j                   d   dz  df   }t        j                  | |fd      S )z*Rotates half the hidden dims of the input..Nr^   r   rd   )rf   r'   rk   )rn   x1x2s      r.   rotate_halfr      sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r-   rotary_pos_embc                     |j                  |      }|j                  |      }| |z  t        |       |z  z   }||z  t        |      |z  z   }||fS )a  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )	unsqueezer   )qkrl   rm   unsqueeze_dimq_embedk_embeds          r.   apply_rotary_pos_embr      sY    & --
&C
--
&C3w;q>C/0G3w;q>C/0GGr-   r   n_reprL   c                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)rf   re   reshape)r   r   batchnum_key_value_headsslenrO   s         r.   	repeat_kvr      so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TTr-   modulequerykeyvalueattention_maskscalingdropoutkwargsc                    t        || j                        }t        || j                        }	t        j                  ||j	                  dd            |z  }
||
|z   }
t
        j                  j                  |
dt        j                        j                  |j                        }
t
        j                  j                  |
|| j                        }
t        j                  |
|	      }|j	                  dd      j                         }||
fS )Nr   r   r^   r[   rR   ptrainingr   )r   num_key_value_groupsr'   matmulrj   r   
functionalsoftmaxfloat32rX   rR   r   r   
contiguous)r   r   r   r   r   r   r   r   
key_statesvalue_statesattn_weightsattn_outputs               r.   eager_attention_forwardr      s     3 ; ;<JUF$?$?@L<<z';';Aq'ABWLL!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|\:K''1-88:K$$r-   c                        e Zd ZdZdedef fdZ	 	 	 ddej                  de	ej                  ej                  f   dz  dej                  dz  d	e
dz  d
ee   de	ej                  ej                  f   fdZ xZS )Xcodec2Attentionz=Multi-headed attention from 'Attention Is All You Need' paperr6   	layer_idxc                 d   t         |           || _        || _        t	        |d|j
                  |j                  z        | _        |j                  |j                  z  | _	        | j                  dz  | _
        |j                  | _        d| _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j                  | j                  z  |j
                  |j                        | _        y )NrO   g      Fr   )r=   r>   r6   r   rS   rT   rU   rO   r   r   r   attention_dropout	is_causalr   r   attention_biasq_projk_projv_projo_projrG   r6   r   rJ   s      r.   r>   zXcodec2Attention.__init__   sM   "
F4F4F&JdJd4de$*$>$>&B\B\$\!}}d*!'!9!9ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii&&68J8JQWQfQf
r-   Nr   position_embeddingsr   past_key_valuesr   rL   c                    |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }	| j                  |      j                  |      j	                  dd      }
|\  }}t        ||	||d      \  }}	| |j                  |	|
| j                        \  }	}
t        j                  | j                  j                  t              } || ||	|
|f| j                  sdn| j                   | j"                  d|\  }} |j$                  g |d j'                         }| j)                  |      }||fS )Nr^   r   r   )r           )r   r   )rf   rO   r   viewrj   r   r   r   updater   r   get_interfacer6   _attn_implementationr   r   r   r   r   r   r   )rG   r   r   r   r   r   input_shapehidden_shapequery_statesr   r   rl   rm   attention_interfacer   r   s                   r.   rt   zXcodec2Attention.forward  s    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST&S $8jRUWZjk#l j&'6'='=j,X\XfXf'g$J(?(M(MKK,,.E)
 %8	%
  $}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((r-   rv   )r#   r$   r%   r&   r   rx   r>   r'   r+   ry   r
   r   r   rt   r{   r|   s   @r.   r   r      s    G
} 
 
4 IM.2(,))||)) #5<<#=>E)) t+	))
 )) +,)) 
u||U\\)	*))r-   r   RMSNormc                   h     e Zd Zddeddf fdZdej                  dej                  fdZd Z xZ	S )	Xcodec2RMSNormepsrL   Nc                     t         |           t        j                  t	        j
                  |            | _        || _        y)z=
        Xcodec2RMSNorm is equivalent to T5LayerNorm
        N)r=   r>   r   r   r'   onesweightvariance_epsilon)rG   rT   r   rJ   s      r.   r>   zXcodec2RMSNorm.__init__B  s1     	ll5::k#:; #r-   r   c                 "   |j                   }|j                  t        j                        }|j	                  d      j                  dd      }|t        j                  || j                  z         z  }| j                  |j                  |      z  S )Nr   r^   T)keepdim)	rR   rX   r'   r   powmeanrsqrtr   r   )rG   r   input_dtypevariances       r.   rt   zXcodec2RMSNorm.forwardJ  sy    #))%((7 $$Q',,R,>%Ht?T?T4T(UU{{]--k:::r-   c                 ^    t        | j                  j                         d| j                   S )Nz, eps=)ry   r   rf   r   )rG   s    r.   
extra_reprzXcodec2RMSNorm.extra_reprQ  s*    ))*+6$2G2G1HIIr-   )ư>)
r#   r$   r%   rY   r>   r'   r+   rt   r   r{   r|   s   @r.   r   r   @  s7    $ $$ $;U\\ ;ell ;Jr-   r   c                       e Zd Zdedef fdZ	 	 	 	 	 ddej                  dej                  dz  dej                  dz  de	dz  d	e
dz  d
eej                  ej                  f   dz  dee   dej                  fdZ xZS )Xcodec2DecoderLayerr6   r   c                     t         |           |j                  | _        t        ||      | _        t        |      | _        t        |j                  |j                        | _	        t        |j                  |j                        | _
        y )N)r6   r   r   )r=   r>   rT   r   	self_attnr~   mlpr   rms_norm_epsinput_layernormpost_attention_layernormr   s      r.   r>   zXcodec2DecoderLayer.__init__V  sl    !--)9Mf%-f.@.@fFYFYZ(6v7I7IvObOb(c%r-   Nr   r   ro   r   	use_cacher   r   rL   c           
          |}| j                  |      } | j                  d||||||d|\  }}	||z   }|}| j                  |      }| j                  |      }||z   }|S )N)r   r   ro   r   r   r   r,   )r   r   r   r   )
rG   r   r   ro   r   r   r   r   residual_s
             r.   rt   zXcodec2DecoderLayer.forward`  s     !,,];)4>> 
')%+ 3
 
q !=0 !55mD/ =0r-   )NNNFN)r#   r$   r%   r   rx   r>   r'   r+   r*   r
   boolry   r   r   rt   r{   r|   s   @r.   r   r   U  s    d} d d /304(,!&HL|| t+ &&-	
  $; #5<<#=>E +, 
r-   r   c                   *     e Zd ZdZd fd	Zd Z xZS )Xcodec2SnakeBetaa  
    A modified Snake function which uses separate parameters for the magnitude of the periodic components
    Shape:
        - Input: (B, C, T)
        - Output: (B, C, T), same shape as the input
    Parameters:
        - alpha - trainable parameter that controls frequency
        - beta - trainable parameter that controls magnitude
    References:
        - This activation function is a modified version based on this paper by Liu Ziyin, Tilman Hartwig, Masahito Ueda:
        https://huggingface.co/papers/2006.08195
    c                     t         |           || _        t        t	        j
                  |      |z        | _        t        t	        j
                  |      |z        | _        d| _        y )Ng&.>)	r=   r>   in_featuresr   r'   zerosalphabetano_div_by_zero)rG   r   r   rJ   s      r.   r>   zXcodec2SnakeBeta.__init__  sT    & u{{;7%?@
ekk+6>?	)r-   c                    | j                   j                  d      j                  d      }| j                  j                  d      j                  d      }t        j                  |      }t        j                  |      }|d|| j
                  z   z  t        j                  t        j                  ||z        d      z  z   }|S )u   
        Forward pass of the function.
        Applies the function to the input elementwise.
        SnakeBeta ∶= x + 1/b * sin^2 (xa)
        r   r^   rP   r   )r   r   r   r'   expr   r   rm   )rG   r   r   r   s       r.   rt   zXcodec2SnakeBeta.forward  s     

$$Q'11"5yy""1%//3		% yy%t7J7J0J)KuyyIIme+,aP
 )
 
 r-   )rP   r#   r$   r%   r&   r>   rt   r{   r|   s   @r.   r   r     s    *r-   r   c                 b   |dz  dk(  }|dz  }d|z  }d|dz
  z  t         j                  z  |z  dz   }|dkD  r	d|d	z
  z  }n|d
k\  rd|dz
  dz  z  d|d
z
  z  z   }nd}t        j                  ||dt        j                        }|rt        j
                  | |      dz   }	nt        j
                  |      |z
  }	| dk(  r(t        j                  dd|ft        j                        S t        j                  d| z  |	z        }
d| z  |z  |
z  }||j                         z  }|j                  dd|      S )aB  Generates a 1D Kaiser-windowed sinc filter.

    Args:
        cutoff (float): Normalized cutoff frequency (0 to 0.5).
        half_width (float): Transition bandwidth.
        kernel_size (int): Number of filter taps.

    Returns:
        torch.Tensor: A tensor of shape (1, 1, kernel_size) representing the filter.
    r   r      gHzG@r   g@g      I@gK46?gffffff!@g      5@gW2ı?   g?gUj@+0?r   F)r   periodicrR         ?rQ   )
mathpir'   kaiser_windowr   rV   r   sincsumr   )cutoff
half_widthkernel_sizeis_even	half_sizedelta_fattenuationr   r  time_indicessinc_filternormalized_filters               r.   kaiser_sinc_filter1dr    sW    Ao"Gq I *nG9q=)DGG3g=DKTs*+		r)c11G{T?Q4RR''$V[VcVcdM ||YJ	:S@||K09< {{{Aq+.emmDD**QZ,67KF
]2[@ *..00!!!Q44r-   c                   &     e Zd Zd fd	Zd Z xZS )Xcodec2DownSample1dc                 ^   t         |           d|z  }d|z  }|| _        || _        || _        |dk  rt        d      |dkD  rt        d      |dz  dk(  | _        |dz  t        | j                        z
  | _        |dz  | _	        || _
        t        |||      }| j                  d|d	
       y )Nr  333333?r   z(Minimum cutoff must be larger than zero.z'A cutoff above 0.5 does not make sense.r   r   filterFr:   )r=   r>   r  r  r	  
ValueErrorevenrx   pad_left	pad_rightstrider  rE   )rG   ratior	  r  r  r  rJ   s         r.   r>   zXcodec2DownSample1d.__init__  s    u5[
$&C<GHHC<FGG!Oq(	#q(3tyy>9$)%fj+FXv%@r-   c                 .   |j                   d   }t        j                  || j                  | j                  fd      }t        j
                  || j                  j                  |j                        j                  |dd      | j                  |      }|S )Nr   	replicatemoder^   r  groups)rf   Fpadr  r  conv1dr  rX   rR   re   r  )rG   r   channelsouts       r.   rt   zXcodec2DownSample1d.forward  sy     &&q)mdmmT^^-LS^_hhKKNN=../66xRH;;
 
r-   r   Nr#   r$   r%   r>   rt   r{   r|   s   @r.   r  r    s    A(
r-   r  c                   &     e Zd Zd fd	Zd Z xZS )Xcodec2UpSample1dc                    t         |           || _        |t        d|z  dz        dz  n|| _        || _        | j                  |z  dz
  | _        | j                  | j
                  z  | j                  | j
                  z
  dz  z   | _        | j                  | j
                  z  | j                  | j
                  z
  dz   dz  z   | _        t        d|z  d|z  | j                        }| j                  d|d	       y )
N   r   r   r  r  )r  r  r	  r  Fr:   )r=   r>   r  rx   r	  r  r$  r  r  r  rE   )rG   r  r	  r  rJ   s       r.   r>   zXcodec2UpSample1d.__init__  s    
6A6I3q5yA~.2{##u,q04;;.$2B2BT[[2PUV1VVDKK/43C3Cdkk3QTU3UZ[2[[%S5[S5[^b^n^noXv%@r-   c           	         |j                   d   }t        j                  || j                  | j                  fd      }| j                  t        j                  || j
                  j                  |j                        j                  |dd      | j                  |      z  }|d| j                  | j                   f   }|S )Nr   r  r  r^   r!  .)rf   r#  r$  r  conv_transpose1dr  rX   rR   re   r  r  r  )rG   r   r&  s      r.   rt   zXcodec2UpSample1d.forward  s     &&q)mdhh-AT

Q%7%7KKNN=../66xRH;;&
 
 &c4==DNN?+J&JKr-   r(  r)  r|   s   @r.   r+  r+    s    
Ar-   r+  c            	       @     e Zd Z	 	 	 	 ddedededef fdZd Z xZS )Xcodec2AntiAliasedActivation1dup_ratio
down_ratioup_kernel_sizedown_kernel_sizec                     t         |           t        |      st        d      || _        t        ||      | _        t        ||      | _        y )Nz$Activation function must be callable)	r=   r>   callable	TypeErroractr+  upsampler  
downsample)rG   
activationr2  r3  r4  r5  rJ   s         r.   r>   z'Xcodec2AntiAliasedActivation1d.__init__  sH     	
#BCC)(NC-j:JKr-   c                 l    | j                  |      }| j                  |      }| j                  |      }|S ru   )r:  r9  r;  r   s     r.   rt   z&Xcodec2AntiAliasedActivation1d.forward$  s2    m4/6r-   )r   r      r>  )r#   r$   r%   rx   r>   rt   r{   r|   s   @r.   r1  r1    sH       "L L 	L
 L Lr-   r1  c                   (     e Zd ZdZ fdZd Z xZS )Xcodec2ResidualUnitza
    A residual unit composed of Snake1d and weight-normalized Conv1d layers with dilations.
    c                    t         |           d|z  dz  }t        t        |            | _        t        j                  ||d||      | _        t        t        |            | _        t        j                  ||d      | _	        y )Nr-  r   r<     )r	  dilationpaddingr   )r	  )
r=   r>   r1  r   snake1r   Conv1dconv1snake2conv2)rG   	dimensionrD  r$  rJ   s       r.   r>   zXcodec2ResidualUnit.__init__1  sp    !a'4@PQZ@[\YYy)X_bc
4@PQZ@[\YYy)C
r-   c                     |}| j                  | j                  |            }| j                  | j                  |            }|j                  d   |j                  d   z
  dz  }|dkD  r
|d|| f   }||z   }|S )ar  
        Forward pass through the residual unit.

        Args:
            hidden_state (`torch.Tensor` of shape `(batch_size, channels, time_steps)`):
                Input tensor .

        Returns:
            output_tensor (`torch.Tensor` of shape `(batch_size, channels, time_steps)`):
                Input tensor after passing through the residual unit.
        r^   r   r   .)rH  rF  rJ  rI  rf   )rG   hidden_stateoutput_tensorrE  s       r.   rt   zXcodec2ResidualUnit.forward9  s     %

4;;}#=>

4;;}#=>%%b)M,?,?,CCIQ;'WgX-=(=>L$}4r-   r   r|   s   @r.   r@  r@  ,  s    Dr-   r@  c                   8     e Zd ZdZddededef fdZd Z xZS )Xcodec2EncoderBlockz&Encoder block used in XCODEC2 encoder.r6   r  stride_indexc           
      t   t         |           |j                  d|z  z  }t        |dz  d      | _        t        |dz  d      | _        t        |dz  d      | _        t        t        |dz              | _	        t        j                  |dz  |d|z  |t        j                  |dz              | _        y )Nr   r   )rD  r   	   rB  r	  r  rE  )r=   r>   encoder_hidden_sizer@  	res_unit1	res_unit2	res_unit3r1  r   rF  r   rG  r  ceilrH  )rG   r6   r  rQ  rK  rJ   s        r.   r>   zXcodec2EncoderBlock.__init__S  s    ..L@	,Y!^aH,Y!^aH,Y!^aH4@PQZ^_Q_@`aYYNI1v:fVZV_V_`fij`jVk

r-   c                     | j                  |      }| j                  |      }| j                  | j                  |            }| j	                  |      }|S ru   )rV  rW  rF  rX  rH  )rG   rM  s     r.   rt   zXcodec2EncoderBlock.forward^  sI    ~~l3~~l3{{4>>,#?@zz,/r-   )r   r   )	r#   r$   r%   r&   r   rx   r>   rt   r{   r|   s   @r.   rP  rP  P  s%    0	
} 	
c 	
S 	
r-   rP  c                   .     e Zd ZdZdef fdZd Z xZS )Xcodec2EncoderzXCODEC2 Encoderr6   c                    t         |           t        j                  d|j                  dd      | _        g | _        t        |j                        D ],  \  }}|dz   }| xj                  t        |||      gz  c_        . t        j                  | j                        | _        |j                  dt        |j                        z  z  }t        t        |            | _        t        j                  ||j                  dd      | _        y )Nr   rC  r   r	  rE  )r  rQ  r   rB  )r=   r>   r   rG  rU  rH  block	enumeratedownsampling_ratiosrP  
ModuleListlenr1  r   rF  rT   rJ  )rG   r6   rQ  r  d_modelrJ   s        r.   r>   zXcodec2Encoder.__init__j  s     YYq&"<"<!UVW

$-f.H.H$I 	b L&'!+LJJ.vfS_`aaJ	b ]]4::.
,,qC8R8R4S/SS4@PQX@YZYYw(:(:STU
r-   c                     | j                  |      }| j                  D ]
  } ||      } | j                  |      }| j                  |      }|S ru   )rH  r_  rF  rJ  )rG   rM  r   s      r.   rt   zXcodec2Encoder.forward{  sQ    zz,/jj 	0F!,/L	0 {{<0zz,/r-   )r#   r$   r%   r&   r   r>   rt   r{   r|   s   @r.   r\  r\  g  s    V} V"	r-   r\  c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )Xcodec2ResNetBlockr6   c                    t         |           t        j                  d|j                  dd      | _        t        j                         | _        t        j                  |j                  |j                  ddd      | _	        t        j                  d|j                  dd      | _
        t        j                         | _        |j                  | _        t        j                  |j                  |j                  ddd      | _        y )N    r   T)
num_groupsnum_channelsr   affiner   r   rT  )r=   r>   r   	GroupNormrT   norm1SiLUactivation1rG  rH  norm2activation2activation_dropoutrJ  r   s     r.   r>   zXcodec2ResNetBlock.__init__  s    \\Rf>P>PVZcgh
779YYv1163E3EST]^hij
\\Rf>P>PVZcgh
779"(";";YYv1163E3EST]^hij
r-   r   rL   c                    |j                  dd      }|}| j                  |      }| j                  |      }| j                  |      }| j	                  |      }| j                  |      }t        j                  j                  || j                  | j                        }| j                  |      }||z   j                  dd      S )Nr   r   r   )rj   rn  rp  rH  rq  rr  r   r   r   rs  r   rJ  rG   r   r   s      r.   rt   zXcodec2ResNetBlock.forward  s    %//15 

=1((7

=1

=1((7--mt?V?Vaeanan-o

=1(33Aq99r-   r   r|   s   @r.   rg  rg    s,    k} k
:U\\ 
:ell 
:r-   rg  c                        e Zd ZdZdef fdZddZdej                  dej                  fdZ	ddej                  d	e
dej                  fd
Zdej                  deej                  ej                  f   fdZ xZS )Xcodec2FiniteScalarQuantizationa!  
    Finite Scalar Quantization (FSQ) module that quantizes continuous latent representations into discrete codes.
    Original code: https://github.com/lucidrains/vector-quantize-pytorch/blob/353d46027888dfb140c3c65a67a7356f1492d71d/vector_quantize_pytorch/finite_scalar_quantization.py#L64

    Original modeling uses `ResidualFSQ` with a single quantizer: https://huggingface.co/HKUSTAudio/xcodec2/blob/main/vq/codec_decoder_vocos.py#L389
    But we can directly use FSQ since a main feature of Xcodec2 is that it uses a single codebook.
    r6   c                     t         |           t        |j                        | _        | j	                         \  }}}| j                  d|d       | j                  d|d       | j                  d|d       y )NlevelsFr:   basiscodebook)r=   r>   listquantization_levels_compute_buffersrE   )rG   r6   ry  rz  r{  rJ   s        r.   r>   z(Xcodec2FiniteScalarQuantization.__init__  su    #'(B(B#C "&"7"7"9xXv%@We>ZeDr-   c                    t        j                  | j                  t         j                  |      }t        j                  t        j                  dg| j                  dd z   |      dt         j                        }t        j
                  t        t        j                  | j                              |      j                  d      }||z  |z  }|dz  }||z
  |z  }|||fS )	zFCompute the levels, basis, and codebook buffers for the FSQ quantizer.)rR   rH   r   Nr^   rH   r   r   r   )
r'   tensorr}  int32cumprodrV   rx   npprodr   )rG   rH   ry  rz  indiceslevel_indicesr  r{  s           r.   r~  z0Xcodec2FiniteScalarQuantization._compute_buffers  s    d66ekkRXYLL!t77<<VLRS[`[f[f
 ,,s2774+C+C#DEfU__`bc E)V3q[
!J.*<uh&&r-   r  rL   c                     |j                  d      }|| j                  z  | j                  z  }| j                  dz  }||z
  |z  }|S )z`
        Convert integer codebook indices to normalized per-dimension codes in [-1, 1].
        r^   r   )r   rz  ry  )rG   r  r  r  codess        r.   _indices_to_codesz1Xcodec2FiniteScalarQuantization._indices_to_codes  sM     ##B' DJJ.$++=[[A%
+z9r-   r   r   c                     | j                   dz
  d|z   z  dz  }t        j                  | j                   dz  dk(  dd      }||z  j                         }||z   j	                         |z  |z
  S )a  
        Constrain `hidden_states` to the valid quantization range for each dimension.

        Uses a scaled tanh to soft-clip values into the interval
        $[-(L-1)/2, (L-1)/2]$ (offset by 0.5 for even-level dimensions), where $L$ is
        the number of quantization levels. The small `eps` margin prevents values from
        saturating exactly at the boundary, which would zero out gradients.

        Args:
            hidden_states (`torch.Tensor`): Continuous input to be bounded.
            eps (`float`, *optional*, defaults to `1e-3`):
                Small margin added to the level range to avoid gradient saturation at boundaries.

        Returns:
            `torch.Tensor`: Bounded values in the valid quantization range.
        r   r   r   r  r   )ry  r'   whereatanhtanh)rG   r   r   
half_rangeoffsetshifts         r.   boundz%Xcodec2FiniteScalarQuantization.bound  so    " kkAo!c'2Q6
T[[1_13<*$++-%++-
:VCCr-   c                 l   |j                   }t        |j                  j                  t              r/|j                  j                  dk7  r|j                  j                  nd}t        |d      5  |j                         }| j                  dz  }| j                  |      }|j                         }|||z
  j                         z   }||z  }||z  |z   }|| j                  z  j                  d      j                  t        j                        }d d d        j                  |      fS # 1 sw Y   xY w)Nr_   r`   Fra   r   r^   rd   )rR   rg   rH   rh   ri   r   rY   ry  r  rounddetachrz  r  rX   r'   r  )	rG   r   original_dtyperb   r  roundedr  code_scaledr  s	            r.   rt   z'Xcodec2FiniteScalarQuantization.forward  s!   &,, -..33S9m>R>R>W>W[`>`   %% 	
 UC 
	M)//1M)J JJ}5M#))+G!W}%<$D$D$FFEJ&E :-;K"TZZ/444<??LG
	M xx'00
	M 
	Ms   /B D**D3ru   )gMbP?)r#   r$   r%   r&   r   r>   r~  r'   r+   r  rY   r  ry   rt   r{   r|   s   @r.   rw  rw    s    E} E
' %,, D5<< De Du|| D,1U\\ 1eELL%,,<V6W 1r-   rw  c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ xZ	S )Xcodec2ISTFTHeadz
    Head for converting decoder outputs to waveform via STFT projection and ISTFT.

    Uses custom "same" padding ISTFT from Vocos:
    https://github.com/gemelo-ai/vocos/blob/c859e3b7b534f3776a357983029d34170ddd6fc3/vocos/spectral_ops.py#L47
    r6   c                 r   t         |           t        j                  |j                  |j
                  dz         | _        |j
                  | _        |j                  | _        | j
                  | j                  z
  dz  | _        t        j                  |j
                        }| j                  d|d       y )Nr   windowFr:   )r=   r>   r   r   rT   n_fftlinear
hop_lengthrE  r'   hann_windowrE   )rG   r6   r  rJ   s      r.   r>   zXcodec2ISTFTHead.__init__  s    ii 2 2FLL14DE\\
 ++

T__4:""6<<0Xv%@r-   r   rL   c                 .   | j                  |      j                  dd      }|j                  dd      \  }}|j                         }|j                         }t	        j
                  |      j                  d      }|t	        j
                  d|z        z  }t        j                  j                  || j                  dd      }|| j                  d d d d f   z  }|j                  d	   }|dz
  | j                  z  | j                  z   }t        j                  |d|fd| j                  fd| j                  f
      d d dd| j                  | j                   f   }	t        j                  | j                  j!                         j#                  d|d	      j                  dd      d|fd| j                  fd| j                  f
      j%                         | j                  | j                    }
|
j                  d      }
|	|
z  }	|	j'                  d      S )Nr   r   rd   g      Y@)maxy              ?backward)r[   normr^   )output_sizer	  r  r   gdy=)min)r  rj   chunkrY   r'   r   clampfftirfftr  r  rf   r  r#  foldrE  squarere   squeezer   )rG   r   	stft_pred	magnitudephasespectrogram_complextime_frames
num_framesr  audiowindow_envelopes              r.   rt   zXcodec2ISTFTHead.forward  s   KK.88A>	$??1!?4	5OO%	IIi(..3.7	'%))BJ*?? iioo&94::1S]o^!DKKa$>>(..r2
!A~84::EK(DJJt'	

 Q4<<4<<-/
/1 &&KK '':r:DDQJK(DJJt'	

 ')DLLDLL=2 *//E/:'q!!r-   
r#   r$   r%   r&   r   r>   r'   r+   rt   r{   r|   s   @r.   r  r    s1    A} A!"U\\ !"ell !"r-   r  c                        e Zd Zdef fdZdej                  dej                  fdZdej                  deej                  ej                  f   fdZ	 xZ
S )Xcodec2Quantizerr6   c                 "   t         |           t        |      | _        t	        j
                  |j                  t        |j                              | _	        t	        j
                  t        |j                        |j                        | _
        y ru   )r=   r>   rw  	quantizerr   r   quantization_dimrc  r}  
project_inproject_outr   s     r.   r>   zXcodec2Quantizer.__init__(  sa    8@))F$;$;SA[A[=\]99S)C)C%DfF]F]^r-   r  rL   c                 x    |j                  d      }| j                  j                  |   }| j                  |      S Nr^   )r  r  r{  r  )rG   r  r  s      r.   
from_codeszXcodec2Quantizer.from_codes.  s5    //"%''0&&r-   r   c                    | j                  |      }|j                  }| j                  j                  |      }| j                  |      \  }}| j	                  |j                  |            }|j                  d      }||fS r  )r  rR   r  r  r  rX   r   )rG   r   r  quantized_outr  s        r.   rt   zXcodec2Quantizer.forward3  sx    6&,,,,];!%!>w(()9)9.)IJ##B'g%%r-   )r#   r$   r%   r   r>   r'   r+   r  ry   rt   r{   r|   s   @r.   r  r  '  sU    _} _'%,, '5<< '
&U\\ &eELL%,,<V6W &r-   r  c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ xZ	S )Xcodec2DecoderzVVocos-based decoder with ResNet, Transformer, and ISTFT head for audio reconstruction.r6   c           	         t         |           t        j                  |j                  |j
                  j                  z   |j                        | _        t        j                  |j                  |j                  dd      | _        t        j                  t        |      t        |      g      | _        |j                  | _        t        |      | _        t        j                  t        |j                         D cg c]  }t#        ||       c}      | _        t        j                  t        |      t        |      g      | _        t        j(                  |j                  d      | _        t-        |      | _        y c c}w )NrC  r   r^  )r6   r   r   )r=   r>   r   r   rT   semantic_model_configfcrG  embedrb  rg  	prior_netrU   r4   
rotary_embrangenum_hidden_layersr   layerspost_net	LayerNormr  r  headr   s      r.   r>   zXcodec2Decoder.__init__@  s   ))F..1M1M1Y1YY[a[m[mnYYv1163E3EST^_`
(:6(BDVW]D^'_`#)#=#= 0?mmEJ6KcKcEde	 3e
 '9&'ACUV\C]&^_LL!3!3>	$V,		 fs   >Fr   rL   c                    | j                  |      }|j                  dd      }| j                  |      }|j                  dd      }| j                  D ]
  } ||      } t	        j
                  | j                  |j                        j                  d      }| j                  ||      }| j                  D ]  } ||fd|i|} | j                  D ]
  } ||      } | j                  | j                  |            S )Nr   r   r  r   r   )r  rj   r  r  r'   rV   rU   rH   r   r  r  r  r  r  )rG   r   r   layerro   r   s         r.   rt   zXcodec2Decoder.forwardN  s    .%//15

=1%//15 ^^ 	1E!-0M	1 ||D$<$<]EYEYZddefg"oom\J[[ 	dE!-cEXc\bcM	d ]] 	1E!-0M	1 yy=122r-   r  r|   s   @r.   r  r  =  s-    `-} -3U\\ 3 3r-   r  c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )Xcodec2SemanticAdapterr6   c                    t         |           t        j                  |j                  j
                  |j                  j
                  ddd      | _        t        j                         | _        t        j                  |j                  j
                  |j                  j
                  ddd      | _	        t        j                         | _
        t        j                  |j                  j
                  |j                  j
                  ddd      | _        t        j                  |j                  j
                  |j                  j
                  ddd      | _        y )Nr   r   F)in_channelsout_channelsr	  rE  r   T)r	  rE  r   )r=   r>   r   rG  r  rT   rH  ReLUact1rJ  act2conv3conv4r   s     r.   r>   zXcodec2SemanticAdapter.__init__j  s
   YY44@@55AA

 GGI	YY((44((44

 GGI	YY((44((44

 YY44@@55AA

r-   r   rL   c                     | j                  |      }| j                  |      }|}| j                  |      }| j                  |      }| j	                  |      }||z   }| j                  |      }|S ru   )rH  r  rJ  r  r  r  ru  s      r.   rt   zXcodec2SemanticAdapter.forward  sk    

=1		-0 

=1		-0

=1%0

=1r-   r   r|   s   @r.   r  r  i  s+    
} 
B	U\\ 	ell 	r-   r  c                   f     e Zd ZU eed<   dZdZdZdZdgZ	dZ
dZdZdZdZdZdZeedZ fd	Z xZS )
Xcodec2PreTrainedModelr6   xcodec2)r  TNr   input_values)r   
attentionsc                    t         |   |       t        |t              r?t	        j
                  |j                         t	        j
                  |j                         y t        |t              r@t        j                  |j                        }t	        j                  |j                  |       y t        |t              r|j                  |j                   j"                        \  }}}t	        j                  |j                   |       t	        j                  |j$                  |       t	        j                  |j&                  |       y t        |t(              rRt+        d|j,                  z  d|j,                  z  |j.                        }t	        j                  |j0                  |       y t        |t2              rLt+        |j4                  |j6                  |j.                        }t	        j                  |j0                  |       y y )Nr  r  r  )r=   _init_weightsrg   r   initzeros_r   r   r  r'   r  r  copy_r  rw  r~  ry  rH   rz  r{  r+  r  r  r	  r  r  r  r  )rG   r   r  ry  rz  r{  filter_tensorrJ   s          r.   r  z$Xcodec2PreTrainedModel._init_weights  sV   f%f./KK%KK$ 01&&v||4FJJv}}f- ?@&,&=&=V]]EYEY&=&Z#FE8JJv}}f-JJv||U+JJv1 120v||1CS6<<EWY_YkYklMJJv}}m4 340@Q@QSYSeSefMJJv}}m4 5r-   )r#   r$   r%   r   r)   base_model_prefixinput_modalitiessupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_cache_class_supports_attention_backend_can_compile_fullgraphmain_input_namer   _can_record_outputsr  r{   r|   s   @r.   r  r    sj    !!&*##4"5N "&!$O,)
5 5r-   r  z!Xcodec2 neural audio codec model.)custom_introc                       e Zd ZeZdef fdZee	 	 	 ddej                  dej                  dej                  dz  dej                  dz  de
d	ee   d
eez  fd              Zee	 	 ddej                  dz  dej                  dz  d	ee   d
eez  fd              Zee	 	 	 ddej                  dej                  dej                  dz  dej                  dz  de
d	ee   d
eez  fd              Z xZS )Xcodec2Modelr6   c                    t         |   |       |j                  | _        t        j                  |j
                        | _        t        |      | _        t        |      | _
        t        j                  |j                  |j
                  j                  z   |j                  |j
                  j                  z         | _        t        |      | _        t#        |      | _        | j'                          y ru   )r=   r>   r  r   from_configr  semantic_encoderr  semantic_adapterr\  acoustic_encoderr   r   rT   
fc_encoderr  r  r  acoustic_decoder	post_initr   s     r.   r>   zXcodec2Model.__init__  s      ++ ) 5 5f6R6R S 6v > .v 6))!=!=!I!II!=!=!I!II
 *&1 .v 6r-   Nr  input_featurespadding_maskinput_features_maskoutput_latentsr   rL   c                    t        j                         5  | j                  ||      }ddd       j                  j	                  dd      }| j                  |      }| j                  |      }	t        j                  ||	gd      }
| j                  |
j	                  dd            }
| j                  |
      \  }}|j	                  dd      }|j	                  dd      }d}|~|j                  dd      }|| j                  z  }t        j                  |j                  d   |j                  	      j                  dd      }||k  j!                  |j"                        }t%        ||r||
      S d|
      S # 1 sw Y   TxY w)a  
        input_values (`torch.Tensor` of shape `(batch_size, 1, sequence_length)`):
            Input audio waveform.
        input_features (`torch.Tensor` of shape `(batch_size, mel_bins, time_steps)`):
            Input audio mel spectrogram for semantic encoding.
        padding_mask (`torch.Tensor` of shape `(batch_size, 1, sequence_length)`):
            Padding mask used to pad `input_values`.
        input_features_mask (`torch.Tensor` of shape `(batch_size, time_steps)`, *optional*):
            Attention mask for the spectrogram input to the semantic encoder. `1` for valid frames, `0` for padding.
        output_latents (`bool`, *optional*, defaults to `False`):
            Whether to return the continuous latent representation from the quantizer.
        )r   Nr   r   rd   r^   T)r[   r   r  )r    r!   r"   )r'   rz   r  last_hidden_staterj   r  r  rk   r  r  r  r  rV   rf   rH   r   rX   rR   r0   )rG   r  r  r  r  r  r   semantic_outputsemantic_hidden_statesacoustic_hidden_statesr   r!   r    r"   audio_lengthtoken_lengthidxs                    r.   encodezXcodec2Model.encode  s   2 ]]_ 	h"33NSf3gO	h!0!B!B!L!LQPQ!R!%!6!67M!N "&!6!6|!D		#9;Q"RXYZ(?(?1(EF  $~~m<##Aq)!++Aq1  #'++D+AL'4??:L,,{004\=P=PQVVWXZ\]C #l 266|7I7IJ##-G-
 	
37-
 	
/	h 	hs   E;;Fr    r!   c                     ||t        d      |,| j                  j                  |j                  dd            }n|j                  dd      } | j                  |fi |}t        |      S )a3  
        audio_codes (`torch.LongTensor`  of shape `(batch_size, 1, codes_length)`):
            Discrete code indices computed using `model.encode`.
        latents (torch.Tensor of shape `(batch_size, dimension, time_steps)`, *optional*):
            Quantized continuous representation of input.
        z3Either `latents` or `audio_codes` must be provided.r   r   )r   )r  r  r  rj   r  r2   )rG   r    r!   r   recon_audios        r.   decodezXcodec2Model.decode
  su     ?{2RSS"nn//0E0Ea0KLG''1-G+d++G>v>#==r-   c                    |j                   d   }| j                  ||||dd      } | j                  d	|j                  dd|d   dd|f   }	t	        |	|j
                  |r|j                  nd|j                        S )
a  
        input_values (`torch.Tensor` of shape `(batch_size, 1, sequence_length)`):
            Input audio waveform.
        input_features (`torch.Tensor` of shape `(batch_size, mel_bins, time_steps)`):
            Input audio mel spectrogram for semantic encoding.
        padding_mask (`torch.Tensor` of shape `(batch_size, 1, sequence_length)`):
            Padding mask used to pad `input_values`.
        input_features_mask (`torch.Tensor` of shape `(batch_size, time_steps)`, *optional*):
            Attention mask for the spectrogram input to the semantic encoder. `1` for valid frames, `0` for padding.
        output_latents (`bool`, *optional*, defaults to `False`):
            Whether to return the continuous latent representation from the quantizer.

        Examples:

        ```python
        >>> from datasets import load_dataset
        >>> from transformers import AutoFeatureExtractor, Xcodec2Model

        >>> dataset = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        >>> audio = dataset["train"]["audio"][0]["array"]

        >>> model_id = "HKUSTAudio/xcodec2-hf"
        >>> model = Xcodec2Model.from_pretrained(model_id)
        >>> feature_extractor = AutoFeatureExtractor.from_pretrained(model_id)

        >>> inputs = feature_extractor(audio=audio, sampling_rate=feature_extractor.sampling_rate, return_tensors="pt")

        >>> outputs = model(**inputs)
        >>> audio_codes = outputs.audio_codes
        >>> audio_values = outputs.audio_values
        ```r^   T)r  r  r  r  return_dict)r!   r  r   .N)r   r    r!   r"   r,   )rf   r  r  r!   r   r    r"   )
rG   r  r  r  r  r  r   lengthencoder_outputsr   s
             r.   rt   zXcodec2Model.forward#  s    V ##B'++)% 3 & 
 #t{{_?+B+BPT_X^_`abcfhoiohocop%'33/=O++4,==	
 	
r-   )NNF)NN)r#   r$   r%   r   config_classr>   r   r   r'   r+   r   r   r   ry   r0   r  r2   r  r   rt   r{   r|   s   @r.   r  r    s    L}   
 -137$2
ll2
 2
 llT)	2

 #\\D02
 2
 +,2
 
%	%2
  2
h  ,0'+>\\D(> $> +,	>
 
%	%>  >. 
 -137$:
ll:
 :
 llT)	:

 #\\D0:
 :
 +,:
 
	:
  :
r-   r  )r   )r   )Qr  collections.abcr   dataclassesr   typingr   numpyr  r'   torch.nnr   torch.nn.functionalr   r#  r    r   r  activationsr	   cache_utilsr
   integrationsr   r   r   modeling_layersr   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   utils.genericr   autor   configuration_xcodec2r   r   r0   r2   Moduler4   r~   r   r   r+   rx   r   rY   r   r   r   r   r   r  r  r+  r1  r@  rP  r\  rg  rw  r  r  r  r  r  r  __all__r,   r-   r.   <module>r*     s!  (  $ !        & !   f f 9 K F & V V +  0 
1K 1  1( 
1; 1  1" 
2; 2  2><RYY ><B ( *+ ,2	UU\\ 	U# 	U%,, 	U& %II%<<% 
% <<	%
 LL4'% % % '(%2 )*C)ryy C) +C)L Y'JRYY J (J((4 (V&ryy &R+5\")) D		 6RYY 0!")) !H")) .RYY @: :0P1bii P1f2"ryy 2"j&ryy &,)3RYY )3X+RYY +\ %5_ %5 %5P @A^
) ^
 B^
B 3
4r-   