Ë
    ÛÓ^jO“  ã                   óZ  — d dl mZ d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZmZmZ dd	lmZmZmZ dd
lmZmZ ddlmZ ddlmZ ddlmZmZmZmZ ddlm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3m4Z4m5Z5 ddl6m7Z7  e,jp                  e9«      Z:e) G d„ de$«      «       Z; G d„ dejx                  «      Z= G d„ dejx                  «      Z> ed«       G d„ dejx                  «      «       Z? G d „ d!ejx                  «      Z@d"„ ZA ed#«      dFd$„«       ZBd%ej†                  d&eDd'ej†                  fd(„ZE	 dGd)ejx                  d*ej†                  d+ej†                  d,ej†                  d-ej†                  dz  d.eFd/eFd0e&e(   fd1„ZG eeB«       G d2„ d3ejx                  «      «       ZH G d4„ d5ejx                  «      ZI G d6„ d7e«      ZJ G d8„ d9e;«      ZK G d:„ d;e«      ZL G d<„ d=e;«      ZM e)d>¬?«       G d@„ dAe;«      «       ZN e)dB¬?«       G dC„ dDe;e7«      «       ZOg dE¢ZPy)Hé    )ÚCallable)ÚOptionalN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)Úuse_kernel_forward_from_hubÚuse_kernel_func_from_hubÚuse_kernelized_func)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_torchdynamo_compilingÚlogging)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú	DiaConfigÚDiaDecoderConfigÚDiaEncoderConfig)ÚDiaGenerationMixinc                   óN   ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZddgZˆ fd„Zˆ xZS )ÚDiaPreTrainedModelÚconfigÚmodelTÚ	input_idsÚDiaEncoderLayerÚDiaDecoderLayerc                 ó&  •— t         ‰| �  |«       t        |t        «      rqt	        j
                  | j                  j                  t        j                  ¬«      | j                  j                  z  }t        j                  |j                  |«       y y )N©Údtype)ÚsuperÚ_init_weightsÚ
isinstanceÚDiaMultiChannelEmbeddingÚtorchÚaranger+   Únum_channelsÚlongÚ
vocab_sizeÚinitÚcopy_Úoffsets)ÚselfÚmoduler>   Ú	__class__s      €úo/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/dia/modeling_dia.pyr4   z DiaPreTrainedModel._init_weightsA   sb   ø€ Ü‰Ñ˜fÔ%Ü�fÔ6Ô7Ü—l‘l 4§;¡;×#;Ñ#;Ä5Ç:Á:ÔNÐQU×Q\ÑQ\×QgÑQgÑgˆGÜ�J‰J�v—~‘~ wÕ/ð 8ó    )Ú__name__Ú
__module__Ú__qualname__r%   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚmain_input_nameÚ_no_split_modulesr4   Ú__classcell__©rA   s   @rB   r*   r*   5   sG   ø… àÓØÐØ&*Ð#ØÐØ€NØÐØ!ÐØ!€OØ*Ð,=Ð>Ð÷0ð 0rC   r*   c                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )r6   a  In order to efficiently compute the audio embedding from the 9 different channels,
    we vectorize the embedding process by using a single embedding layer and an offset.
    Example:
    - num_embeds = 4
    - vocab_size = 8
    - num_channels = 3
    We would have offsets = [0, 8, 16]
    If audio_codes = [0, 1, 2, 3], [1, 3, 4, 7], [5, 6, 7, 8],
    then tokens = audio_codes + offsets
                = [0, 1, 2, 3, 9, 11, 12, 15, 21, 22, 23, 24]
    This allows us to use a single embedding layer for all channels.
    r+   c                 ó~  •— t         ‰| �  «        t        j                  |j                  |j
                  z  |j                  «      | _        |j                  | _        |j
                  | _        t        j                  |j
                  t        j                  ¬«      |j                  z  }| j                  d|d¬«       y )Nr1   r>   F©Ú
persistent)r3   Ú__init__r   Ú	Embeddingr;   r9   Úhidden_sizeÚembedr7   r8   r:   Úregister_buffer)r?   r+   r>   rA   s      €rB   rV   z!DiaMultiChannelEmbedding.__init__V   s�   ø€ Ü‰ÑÔÜ—\‘\ &×"3Ñ"3°f×6IÑ6IÑ"IÈ6×K]ÑK]Ó^ˆŒ
Ø!×-Ñ-ˆÔØ"×/Ñ/ˆÔÜ—,‘,˜v×2Ñ2¼%¿*¹*ÔEÈ×HYÑHYÑYˆØ×Ñ˜Y¨¸EÐÕBrC   Úaudio_codesÚreturnc                 ó^  — || j                   j                  |j                  «      z   j                  d|j                  d   |j                  d   z  «      }| j                  |«      j                  |j                  d   |j                  d   d| j                  «      }|j                  d¬«      S )Néÿÿÿÿr$   é   r   ©Údim)r>   ÚtoÚdeviceÚviewÚshaperY   rX   Úsum)r?   r[   ÚtokensÚembedss       rB   Úforwardz DiaMultiChannelEmbedding.forward^   s—   € Ø §¡§¡°×0BÑ0BÓ CÑC×IÑIØ�×!Ñ! !Ñ$ {×'8Ñ'8¸Ñ';Ñ;ó
ˆð —‘˜FÓ#×(Ñ(¨¯©°a©¸+×:KÑ:KÈAÑ:NÐPRÐTX×TdÑTdÓeˆØ�z‰z˜aˆzÓ Ð rC   )
rD   rE   rF   Ú__doc__r&   rV   r7   ÚTensorri   rP   rQ   s   @rB   r6   r6   H   s2   ø„ ñðCÐ/õ Cð! 5§<¡<ð !°E·L±L÷ !rC   r6   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚDiaMLPc                 ó*  •— t         ‰| �  «        || _        t        j                  |j
                  d|j                  z  d¬«      | _        t        j                  |j                  |j
                  d¬«      | _        t        |j                     | _        y )Nr_   F©Úbias)r3   rV   r+   r   ÚLinearrX   Úintermediate_sizeÚgate_up_projÚ	down_projr   Ú
hidden_actÚactivation_fn©r?   r+   rA   s     €rB   rV   zDiaMLP.__init__g   sp   ø€ Ü‰ÑÔàˆŒÜŸI™I f×&8Ñ&8¸!¸f×>VÑ>VÑ:VÐ]bÔcˆÔÜŸ™ 6×#;Ñ#;¸V×=OÑ=OÐV[Ô\ˆŒÜ# F×$5Ñ$5Ñ6ˆÕrC   Úhidden_statesr\   c                 óš   — | j                  |«      }|j                  dd¬«      \  }}|| j                  |«      z  }| j                  |«      S )Nr_   r^   r`   )rs   Úchunkrv   rt   )r?   rx   Ú	up_statesÚgates       rB   ri   zDiaMLP.forwardo   sL   € Ø×%Ñ% mÓ4ˆ	à#Ÿ/™/¨!°˜/Ó4‰ˆˆiØ × 2Ñ 2°4Ó 8Ñ8ˆ	à�~‰~˜iÓ(Ð(rC   )rD   rE   rF   rV   r7   ÚFloatTensorri   rP   rQ   s   @rB   rm   rm   f   s'   ø„ ô7ð) U×%6Ñ%6ð )¸5×;LÑ;L÷ )rC   rm   ÚRMSNormc                   óh   ‡ — e Zd Zddeddfˆ fd„Zdej                  dej                  fd„Zd„ Zˆ xZ	S )	Ú
DiaRMSNormÚepsr\   Nc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)z9
        DiaRMSNorm is equivalent to T5LayerNorm
        N)r3   rV   r   Ú	Parameterr7   ÚonesÚweightÚvariance_epsilon)r?   rX   r�   rA   s      €rB   rV   zDiaRMSNorm.__init__z   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕrC   rx   c                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S )Nr_   r^   T)Úkeepdim)	r2   rb   r7   Úfloat32ÚpowÚmeanÚrsqrtr†   r…   )r?   rx   Úinput_dtypeÚvariances       rB   ri   zDiaRMSNorm.forward‚   sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:rC   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)Útupler…   re   r†   )r?   s    rB   Ú
extra_reprzDiaRMSNorm.extra_repr‰   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIrC   )g�íµ ÷Æ°>)
rD   rE   rF   ÚfloatrV   r7   rk   ri   r‘   rP   rQ   s   @rB   r€   r€   x   s7   ø„ ñ$¨ð $¸$õ $ð; U§\¡\ð ;°e·l±ló ;öJrC   r€   c                   óÄ   ‡ — e Zd ZU ej                  ed<   ddefˆ fd„Ze	 	 	 ddedz  de	d   de
dz  ded	ef   fd
„«       Z ej                  «       ed„ «       «       Zˆ xZS )ÚDiaRotaryEmbeddingÚinv_freqNr+   c                 ó®  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        | j
                  j                  d   | _        | j                  }| j                  dk7  rt        | j                     } || j
                  |«      \  }| _
        | j                  d|d¬«       | j                  d|j                  «       d¬«       y )NÚ	rope_typeÚdefaultr•   FrT   Úoriginal_inv_freq)r3   rV   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr+   Úrope_parametersr—   Úcompute_default_rope_parametersr   Úattention_scalingrZ   Úclone)r?   r+   rc   Úrope_init_fnr•   rA   s        €rB   rV   zDiaRotaryEmbedding.__init__�   s¶   ø€ Ü‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àˆŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÒ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆ�$Ô(à×Ñ˜Z¨¸eÐÔDØ×ÑÐ0°(·.±.Ó2BÈuÐÕUrC   rc   ztorch.deviceÚseq_lenr\   ztorch.Tensorc                 ó  — | j                   d   }t        | dd«      xs | j                  | j                  z  }d}d|t	        j
                  d|dt        j                  ¬«      j                  |t        j                  ¬«      |z  z  z  }||fS )	a¨  
        Computes the inverse frequencies according to the original RoPE implementation
        Args:
            config ([`~transformers.PreTrainedConfig`]):
                The model configuration.
            device (`torch.device`):
                The device to use for initialization of the inverse frequencies.
            seq_len (`int`, *optional*):
                The current sequence length. Unused for this type of RoPE.
        Returns:
            Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
            post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
        Ú
rope_thetaÚhead_dimNg      ð?r   r_   r1   )rc   r2   )	r�   ÚgetattrrX   Únum_attention_headsr7   r8   Úint64rb   r’   )r+   rc   r¢   Úbasera   Úattention_factorr•   s          rB   rž   z2DiaRotaryEmbedding.compute_default_rope_parameters    s�   € ð& ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/Òc°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐð Ø”U—\‘\ ! S¨!´5·;±;Ô?×BÑBÈ&ÔX]×XcÑXcÐBÓdÐgjÑjÑkñ
ˆð Ð)Ð)Ð)rC   c                 óN  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  ||fd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j	                  |j                   ¬
«      	j	                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   r^   r$   ÚmpsÚcpuF)Údevice_typeÚenabledr_   r`   r1   )r•   r’   Úexpandre   rb   rc   r5   ÚtypeÚstrr!   Ú	transposer7   ÚcatÚcosrŸ   Úsinr2   )
r?   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedr®   ÚfreqsÚembrµ   r¶   s
             rB   ri   zDiaRotaryEmbedding.forward¾   sR  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   ÃBFÆF$©N©NNN)rD   rE   rF   r7   rk   rG   r%   rV   Ústaticmethodr   Úintr�   r’   rž   Úno_gradr   ri   rP   rQ   s   @rB   r”   r”   �   s˜   ø… Ø�l‰lÓñV˜yõ Vð  à#'Ø+/Ø"ñ*Ø˜DÑ ð*à˜Ñ(ð*ð �t‘ð*ð 
ˆ~˜uÐ$Ñ	%ò	*ó ð*ð: €U‡]�]ƒ_Øñ<ó ó ô<rC   r”   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..Nr^   r_   r`   )re   r7   r´   )r·   Úx1Úx2s      rB   Úrotate_halfrÅ   Î   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'rC   Úrotary_pos_embc                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )a…  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezerÅ   )ÚqÚkrµ   r¶   Úunsqueeze_dimÚq_embedÚk_embeds          rB   Úapply_rotary_pos_embrÎ   Õ   sY   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐrC   rx   Ún_repr\   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r$   N)re   r°   Úreshape)rx   rÏ   ÚbatchÚnum_key_value_headsÚslenr¥   s         rB   Ú	repeat_kvrÕ   ï   so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTrC   r@   ÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 ó  — t        || j                  «      }t        || j                  «      }	t        j                  ||j	                  dd«      «      |z  }
|�|
|z   }
t
        j                  j                  |
dt        j                  ¬«      j                  |j                  «      }
t
        j                  j                  |
|| j                  ¬«      }
t        j                  |
|	«      }|j	                  dd«      j                  «       }||
fS )Nr_   r   r^   )ra   r2   )ÚpÚtrainingr$   )rÕ   Únum_key_value_groupsr7   Úmatmulr³   r   Ú
functionalÚsoftmaxr‰   rb   r2   rÛ   rß   Ú
contiguous)r@   rÖ   r×   rØ   rÙ   rÚ   rÛ   rÜ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               rB   Úeager_attention_forwardré   û   sà   € ô ˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$rC   c                   ó  ‡ — e Zd ZdZddeez  dedefˆ fd„Z	 	 	 dde	j                  dee	j                  e	j                  f   dz  d	e	j                  dz  d
edz  dee   dee	j                  e	j                  f   fd„Zˆ xZS )ÚDiaSelfAttentionú=Multi-headed attention from 'Attention Is All You Need' paperr+   Ú	layer_idxÚ	is_causalc                 ó�  •— t         ‰| �  «        || _        || _        |j                  | _        | j                  j
                  | _        | j                  j                  xs | j                  | _        | j                  | j                  z  | _        t        |d|j                  | j                  z  «      | _
        d| _        d| _        || _        t        j                  | j                  | j                  | j                  z  d¬«      | _        t        j                  | j                  | j                  | j                  z  d¬«      | _        t        j                  | j                  | j                  | j                  z  d¬«      | _        t        j                  | j                  | j                  z  | j                  d¬«      | _        y )Nr¥   r$   ç        Fro   )r3   rV   r+   rí   rX   r§   Ú	num_headsrÓ   rà   r¦   r¥   rÚ   Úattention_dropoutrî   r   rq   Úq_projÚk_projÚv_projÚo_proj)r?   r+   rí   rî   rA   s       €rB   rV   zDiaSelfAttention.__init__  sF  ø€ Ü‰ÑÔØˆŒØ"ˆŒØ!×-Ñ-ˆÔØŸ™×8Ñ8ˆŒØ#'§;¡;×#BÑ#BÒ#TÀdÇnÁnˆÔ Ø$(§N¡N°d×6NÑ6NÑ$NˆÔ!Ü ¨
°F×4FÑ4FÈ$Ï.É.Ñ4XÓYˆŒØˆŒØ!$ˆÔØ"ˆŒä—i‘i × 0Ñ 0°$·.±.À4Ç=Á=Ñ2PÐW\Ô]ˆŒÜ—i‘i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Ñ2ZÐafÔgˆŒÜ—i‘i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Ñ2ZÐafÔgˆŒÜ—i‘i §¡°·±Ñ >À×@PÑ@PÐW\Ô]ˆ�rC   Nrx   Úposition_embeddingsrÙ   Úpast_key_valuesrÜ   r\   c                 ó
  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
|\  }}t        ||	||«      \  }}	|� |j                  |	|
| j                  «      \  }	}
t        j                  | j                  j                  t        «      } || ||	|
|f| j                  sdn| j                   | j"                  dœ|¤Ž\  }} |j$                  g |¢d‘­Ž j'                  «       }| j)                  |«      }||fS )Nr^   r$   r_   rð   )rÛ   rÚ   )re   r¥   ró   rd   r³   rô   rõ   rÎ   Úupdaterí   r   Úget_interfacer+   Ú_attn_implementationré   rß   rò   rÚ   rÑ   rä   rö   )r?   rx   r÷   rÙ   rø   rÜ   Úinput_shapeÚhidden_shapeÚquery_statesrå   ræ   rµ   r¶   Úattention_interfacerè   rç   s                   rB   ri   zDiaSelfAttention.forward*  s‘  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ&Ø'6×'=Ñ'=¸jÈ,ÐX\×XfÑXfÓ'gÑ$ˆJ˜ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(rC   )Fr¾   )rD   rE   rF   rj   r'   r&   rÀ   ÚboolrV   r7   rk   r�   r	   r   r   ri   rP   rQ   s   @rB   rë   rë     s¸   ø„ áGñ^Ð/Ð2BÑBð ^Èsð ^Ð_cõ ^ð* IMØ.2Ø(,ñ&)à—|‘|ð&)ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð&)ð Ÿ™ tÑ+ð	&)ð
  ™ð&)ð Ð+Ñ,ð&)ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷&)rC   rë   c                   óØ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 ddej                  dej                  dej                  dz  d	e	dz  d
e
e   deej                  ej                  dz  f   fd„Zˆ xZS )ÚDiaCrossAttentionrì   r+   rí   c                 óf  •— t         ‰| �  «        || _        || _        |j                  | _        |j
                  | _        | j                  j                  | _        | j                  j                  | _	        | j                  | j                  z  | _
        |j                  | _        d| _        d| _        d| _        t!        j"                  | j                  | j                  | j                  z  d¬«      | _        t!        j"                  | j
                  | j                  | j                  z  d¬«      | _        t!        j"                  | j
                  | j                  | j                  z  d¬«      | _        t!        j"                  | j                  | j                  z  | j                  d¬«      | _        y )Nr$   rð   Fro   )r3   rV   r+   rí   rX   Úcross_hidden_sizeÚcross_num_attention_headsrñ   Úcross_num_key_value_headsrÓ   rà   Úcross_head_dimr¥   rÚ   rò   rî   r   rq   ró   rô   rõ   rö   ©r?   r+   rí   rA   s      €rB   rV   zDiaCrossAttention.__init__V  s?  ø€ Ü‰ÑÔØˆŒØ"ˆŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔØŸ™×>Ñ>ˆŒØ#'§;¡;×#HÑ#HˆÔ Ø$(§N¡N°d×6NÑ6NÑ$NˆÔ!Ø×-Ñ-ˆŒØˆŒØ!$ˆÔØˆŒä—i‘i × 0Ñ 0°$·.±.À4Ç=Á=Ñ2PÐW\Ô]ˆŒÜ—i‘i × 6Ñ 6¸×8PÑ8PÐSW×S`ÑS`Ñ8`ÐglÔmˆŒÜ—i‘i × 6Ñ 6¸×8PÑ8PÐSW×S`ÑS`Ñ8`ÐglÔmˆŒÜ—i‘i §¡°·±Ñ >À×@PÑ@PÐW\Ô]ˆ�rC   Nrx   Úcross_attention_statesrÙ   rø   rÜ   r\   c                 óF  — |j                   d d }g |¢d‘| j                  ‘­}g |j                   d d ¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }	|�%|j
                  j                  | j                  «      nd}
|�]|
r[|j                  j                  | j                     j                  }|j                  j                  | j                     j                  }n¥| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }|�C|j                  j                  ||| j                  «      \  }}d|j
                  | j                  <   t        j                   | j"                  j$                  t&        «      } || |	|||fd| j(                  i|¤Ž\  }}|j+                  g |¢d‘­«      j-                  «       }| j/                  |«      }||fS )Nr^   r$   r_   FTrÚ   )re   r¥   ró   rd   r³   Ú
is_updatedÚgetrí   Úcross_attention_cacheÚlayersÚkeysÚvaluesrô   rõ   rú   r   rû   r+   rü   ré   rÚ   rÑ   rä   rö   )r?   rx   r
  rÙ   rø   rÜ   rý   rþ   Úcross_shaperÿ   r  rå   ræ   r   rè   rç   s                   rB   ri   zDiaCrossAttention.forwardi  s  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆØMÐ.×4Ñ4°S°bÐ9ÐM¸2ÐM¸t¿}¹}ÑMˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆàGVÐGb�_×/Ñ/×3Ñ3°D·N±NÔCÐhmˆ
ØÐ&©:à(×>Ñ>×EÑEÀdÇnÁnÑU×ZÑZˆJØ*×@Ñ@×GÑGÈÏÉÑW×^Ñ^‰LàŸ™Ð%;Ó<×AÑAÀ+ÓN×XÑXÐYZÐ\]Ó^ˆJØŸ;™;Ð'=Ó>×CÑCÀKÓP×ZÑZÐ[\Ð^_Ó`ˆLàÐ*à+:×+PÑ+P×+WÑ+WØØ Ø—N‘Nó,Ñ(�
˜Lð >B�×*Ñ*¨4¯>©>Ñ:ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ%
ð —L‘Lð%
ð ñ%
Ñ!ˆ�\ð "×)Ñ)Ð*<¨KÐ*<¸Ñ*<Ó=×HÑHÓJˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(rC   ©NN)rD   rE   rF   rj   r&   rÀ   rV   r7   rk   r   r   r   r�   ri   rP   rQ   s   @rB   r  r  S  s—   ø„ ÙGð^Ð/ð ^¸Cõ ^ð. /3Ø6:ñ1)à—|‘|ð1)ð !&§¡ð1)ð Ÿ™ tÑ+ð	1)ð
 -¨tÑ3ð1)ð Ð-Ñ.ð1)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷1)rC   r  c                   óî   ‡ — e Zd Zdedefˆ fd„Z	 	 ddej                  deej                  ej                  f   dz  dej                  dz  de	e
   d	eej                  ej                  dz  f   f
d
„Zˆ xZS )r.   r+   rí   c                 ó   •— t         ‰| �  «        t        |j                  |j                  ¬«      | _        t        ||d¬«      | _        t        |j                  |j                  ¬«      | _        t        |«      | _
        y )N©r�   F©rî   )r3   rV   r€   rX   Únorm_epsÚpre_sa_normrë   Úself_attentionÚpost_sa_normrm   Úmlpr	  s      €rB   rV   zDiaEncoderLayer.__init__ž  s\   ø€ Ü‰ÑÔÜ% f×&8Ñ&8¸f¿o¹oÔNˆÔÜ.¨v°yÈEÔRˆÔÜ& v×'9Ñ'9¸v¿¹ÔOˆÔÜ˜&“>ˆ�rC   Nrx   r÷   rÙ   rÜ   r\   c                 óº   — |}| j                  |«      } | j                  |f||dœ|¤Ž\  }}||z   }|}| j                  |«      }| j                  |«      }	||	z   }|S )N)r÷   rÙ   )r  r  r  r  )
r?   rx   r÷   rÙ   rÜ   ÚresidualÚnormed_statesÚself_attn_outputÚ_Úmlp_outs
             rB   ri   zDiaEncoderLayer.forward¥  s‰   € ð !ˆØ×(Ñ(¨Ó7ˆØ1˜d×1Ñ1Øð
à 3Ø)ñ
ð ñ	
ÑÐ˜!ð !Ð#3Ñ3ˆà ˆØ×)Ñ)¨-Ó8ˆØ—(‘(˜=Ó)ˆØ  7Ñ*ˆàÐrC   r  )rD   rE   rF   r'   rÀ   rV   r7   rk   r�   r   r   ri   rP   rQ   s   @rB   r.   r.   �  s—   ø„ ð"Ð/ð "¸Cõ "ð IMØ.2ñ	à—|‘|ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ÿ™ tÑ+ð	ð
 Ð-Ñ.ðð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷rC   r.   c                   óœ   ‡ — e Zd ZeedœZdefˆ fd„Zee	e
	 d
dej                  dej                  dz  dee   defd	„«       «       «       Zˆ xZS )Ú
DiaEncoder)rx   Ú
attentionsr+   c           	      ó²  •— t         ‰| �  |«       || _        t        j                  |j
                  |j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                  ¬«      | _        t!        |¬«      | _        | j%                  «        y c c}w ©Nr  ©r+   )r3   rV   r+   r   rW   r;   rX   Ú	embeddingÚ
ModuleListÚrangeÚnum_hidden_layersr.   r  r€   r  Únormr”   Ú
rotary_embÚ	post_initr	  s      €rB   rV   zDiaEncoder.__init__Ä  s›   ø€ Ü‰Ñ˜Ô ØˆŒäŸ™ f×&7Ñ&7¸×9KÑ9KÓLˆŒÜ—m‘mÜAFÀv×G_ÑG_ÓA`Öa°IŒ_˜V YÕ/Òaó
ˆŒô ˜v×1Ñ1°v·±ÔGˆŒ	Ü,°FÔ;ˆŒà�‰Õùò bs   Á-CNr-   rÙ   rÜ   r\   c                 óZ  — | j                  |«      }t        j                  |j                  d   |j                  ¬«      d d d …f   }t        | j                  ||¬«      }| j                  ||¬«      }| j                  D ]  } ||f|||dœ|¤Ž}Œ | j                  |«      }t        |¬«      S )Nr^   ©rc   )r+   Úinputs_embedsrÙ   ©r¸   )rÙ   r¸   r÷   )Úlast_hidden_state)r)  r7   r8   re   rc   r   r+   r.  r  r-  r   )r?   r-   rÙ   rÜ   rx   r¸   r÷   Úencoder_layers           rB   ri   zDiaEncoder.forwardÑ  s¾   € ð Ÿ™ yÓ1ˆô
 —|‘| I§O¡O°BÑ$7À	×@PÑ@PÔQÐRVÒXYÐRYÑZˆä2Ø—;‘;Ø'Ø)ô
ˆð
 #Ÿo™o¨mÈ,˜oÓWÐà!Ÿ[™[ò 	ˆMÙ)Øðà-Ø)Ø$7ñ	ð
 ñ‰Mð	ð Ÿ	™	 -Ó0ˆä°Ô?Ð?rC   r½   )rD   rE   rF   r.   rë   Ú_can_record_outputsr'   rV   r"   r#   r   r7   rk   r   r   r   ri   rP   rQ   s   @rB   r$  r$  ¾  s†   ø„ à(Ø&ñÐð
Ð/õ ð  ØØð /3ñ@à—<‘<ð@ð Ÿ™ tÑ+ð@ð Ð+Ñ,ð	@ð
 
ò@ó ó ó  ô@rC   r$  c                   óL  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	ej                  dz  d
e	dz  deej                  ej                  dz  ej                  dz  f   fd„Z
ˆ xZS )r/   r+   rí   c                 ó�  •— t         ‰| �  «        |j                  | _        t	        ||d¬«      | _        t        ||«      | _        t        |j                  |j                  ¬«      | _
        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        |«      | _        y )NTr  r  )r3   rV   rX   Ú	embed_dimrë   r  r  Úcross_attentionr€   r  r  Úpre_ca_normÚpre_mlp_normrm   r  r	  s      €rB   rV   zDiaDecoderLayer.__init__÷  s‘   ø€ Ü‰ÑÔØ×+Ñ+ˆŒÜ.¨v°yÈDÔQˆÔÜ0°¸ÓCˆÔÜ% f×&8Ñ&8¸f¿o¹oÔNˆÔÜ% f×&8Ñ&8¸f¿o¹oÔNˆÔÜ& v×'9Ñ'9¸v¿¹ÔOˆÔÜ˜&“>ˆ�rC   Nrx   r÷   rÙ   Úencoder_hidden_statesÚencoder_attention_maskrø   r\   c                 óZ  — |}t        |t        «      r|j                  }|}	| j                  |«      }
 | j                  |
|||fi |¤Ž\  }}|	|z   }|}	| j                  |«      }
 | j                  |
|f||dœ|¤Ž\  }}|	|z   }|}	| j                  |«      }
| j                  |
«      }|	|z   }|S )N)rÙ   rø   )	r5   r   Úself_attention_cacher  r  r;  r:  r<  r  )r?   rx   r÷   rÙ   r=  r>  rø   rÜ   Úself_attn_cacher  r  r   r!  Úcross_statesr"  s                  rB   ri   zDiaDecoderLayer.forward  sø   € ð *ˆÜ�oÔ':Ô;Ø-×BÑBˆOà ˆØ×(Ñ(¨Ó7ˆØ1˜d×1Ñ1ØØØð ñ
ð ñ
ÑÐ˜!ð !Ð#3Ñ3ˆà ˆØ×(Ñ(¨Ó7ˆØ.˜$×.Ñ.ØØ!ð
ð 2Ø+ñ	
ð
 ñ
‰ˆ�að ! <Ñ/ˆà ˆØ×)Ñ)¨-Ó8ˆØ—(‘(˜=Ó)ˆØ  7Ñ*ˆàÐrC   ©NNNNN)rD   rE   rF   r&   rÀ   rV   r7   rk   r�   r   ri   rP   rQ   s   @rB   r/   r/   ö  sÐ   ø„ ð"Ð/ð "¸Cõ "ð IMØ.2Ø59Ø6:Ø6:ñ+à—|‘|ð+ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð+ð Ÿ™ tÑ+ð	+ð
  %Ÿ|™|¨dÑ2ð+ð !&§¡¨tÑ 3ð+ð -¨tÑ3ð+ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%·,±,ÀÑ2EÐEÑ	F÷+rC   r/   c                   ó  ‡ — e Zd ZdZeeedœZdefˆ fd„Z	e
ee	 	 	 	 	 ddej                  dej                  dz  dej                  dz  d	ej                   dz  d
ej                  dz  dedz  dee   deez  fd„«       «       «       Zˆ xZS )Ú
DiaDecoderz-Transformer Decoder Stack using DenseGeneral.)rx   r%  Úcross_attentionsr+   c           	      óª  •— t         ‰| �  |«       |j                  | _        |j                  | _        t	        |«      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                  ¬«      | _        t!        |¬«      | _        | j%                  «        y c c}w r'  )r3   rV   r9   r;   r6   Ú
embeddingsr   r*  r+  r,  r/   r  r€   rX   r  r-  r”   r.  r/  r	  s      €rB   rV   zDiaDecoder.__init__8  s�   ø€ Ü‰Ñ˜Ô Ø"×/Ñ/ˆÔØ ×+Ñ+ˆŒÜ2°6Ó:ˆŒÜ—m‘mÜAFÀv×G_ÑG_ÓA`Öa°IŒ_˜V YÕ/Òaó
ˆŒô ˜v×1Ñ1°v·±ÔGˆŒ	Ü,°FÔ;ˆŒà�‰Õùò bs   Á)CNr-   r¸   rÙ   r=  r>  rø   rÜ   r\   c                 óT  — |j                  «       dd \  }}	|�|j                  «       nd}
|€5t        j                  |	|j                  ¬«      |
z   }|j                  d«      }| j                  |«      }|€1t        «       s'|
|	z   }t        j                  |||j                  ¬«      }t        | j                  |||¬«      }t        | j                  |||¬«      }| j                  ||¬«      }| j                  D ]  } |||||f|||dœ|¤Ž}Œ | j                  |«      }t        ||¬	«      S )
a  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`):
            The original `decoder_input_ids` in 3D shape to facilitate more efficient computations.

            [What are input IDs?](../glossary#input-ids)
        Nr^   r   r1  )r+   r2  rÙ   rø   )r+   r2  rÙ   r=  r3  )r>  rø   r¸   )r4  rø   )ÚsizeÚget_seq_lengthr7   r8   rc   rÈ   rH  r   r„   r   r+   r   r.  r  r-  r   )r?   r-   r¸   rÙ   r=  r>  rø   rÜ   Ú
batch_sizeÚ
seq_lengthÚpast_key_values_lengthrx   Úmask_seq_lengthr÷   Úlayers                  rB   ri   zDiaDecoder.forwardE  sW  € ð( "+§¡Ó!1°#°2Ð!6Ñˆ
�JØETÐE` ×!?Ñ!?Ô!AÐfgÐàÐÜ Ÿ<™<¨
¸9×;KÑ;KÔLÐOeÑeˆLØ'×1Ñ1°!Ó4ˆLð Ÿ™¨	Ó2ˆàÐ!Ô*BÔ*Dà4°zÑAˆOÜ"ŸZ™Z¨
°OÈI×L\ÑL\Ô]ˆNä+Ø—;‘;Ø'Ø)Ø+ô	
ˆô ";Ø—;‘;Ø'Ø1Ø"7ô	"
Ðð #Ÿo™o¨mÈ,˜oÓWÐà—[‘[ò 	ˆEÙ!Øð $ØØ%ðð (>Ø /Ø)ñð ñ‰Mð	ð Ÿ	™	 -Ó0ˆä8Ø+Ø+ô
ð 	
rC   rC  )rD   rE   rF   rj   r/   rë   r  r6  r&   rV   r"   r#   r   r7   rk   Ú
LongTensorr}   r   r   r   r   r�   ri   rP   rQ   s   @rB   rE  rE  /  sñ   ø„ Ù7ð )Ø&Ø-ñÐðÐ/õ ð  ØØð 15Ø.2Ø:>Ø:>Ø6:ñA
à—<‘<ðA
ð ×&Ñ&¨Ñ-ðA
ð Ÿ™ tÑ+ð	A
ð
  %×0Ñ0°4Ñ7ðA
ð !&× 0Ñ 0°4Ñ 7ðA
ð -¨tÑ3ðA
ð Ð+Ñ,ðA
ð 
3°UÑ	:òA
ó ó ó  ôA
rC   rE  z[
    The bare Dia model outputting raw hidden-states without any specific head on top.
    )Úcustom_introc                   ó  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
z  dz  d
edz  dedz  de
ez  fd„«       «       Zˆ xZS )ÚDiaModelr+   c                 óº   •— t         ‰| �  |«       || _        t        |j                  «      | _        t        |j                  «      | _        | j                  «        y r½   )
r3   rV   r+   r$  Úencoder_configÚencoderrE  Údecoder_configÚdecoderr/  rw   s     €rB   rV   zDiaModel.__init__’  sE   ø€ Ü‰Ñ˜Ô ØˆŒÜ! &×"7Ñ"7Ó8ˆŒÜ! &×"7Ñ"7Ó8ˆŒØ�‰ÕrC   Nr-   rÙ   Údecoder_input_idsÚdecoder_position_idsÚdecoder_attention_maskÚencoder_outputsrø   Ú	use_cacher\   c	                 óø  — |€|€t        d«      ‚| j                  r%| j                  r|rt        j	                  d«       d}|r6|€4t        t        | j                  ¬«      t        | j                  ¬«      «      }|€ | j                  d||dœ|	¤Ž}nGt        |t        «      s7t        |d   t        |«      dkD  r|d   ndt        |«      d	kD  r|d	   nd¬
«      }|d   j                  d   d| j                  j                  j                  }}}
|€Ct        j                   |
d|f| j                  j                  j"                  | j$                  ¬«      }|j&                  d	k(  r#|j)                  |
||«      j+                  dd	«      } | j,                  d||||d   |||dœ|	¤Ž}t/        |j0                  |j2                  |j4                  |j6                  |j8                  |d   |j4                  |j6                  ¬«      S )a\  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
        or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
            1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
            the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
            tened audio logits which are used to calculate the loss.

            2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
            Dia to calculate embeddings and subsequent steps more efficiently.

            If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
            `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
            [`DiaProcessor.__call__`] for more details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

            [What are position IDs?](../glossary#position-ids)
        NzXYou should either provide text ids or the cached text encodings. Neither has been found.zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fr(  )r-   rÙ   r   r$   r_   )r4  rx   r%  r^   )rJ  Ú
fill_valuerc   )r-   r¸   rÙ   r=  r>  rø   r^  )r4  rø   Údecoder_hidden_statesÚdecoder_attentionsrF  Úencoder_last_hidden_stater=  Úencoder_attentions© )Ú
ValueErrorÚis_gradient_checkpointingrß   ÚloggerÚwarning_oncer   r
   r+   rW  r5   r   Úlenre   rX  r9   r7   ÚfullÚbos_token_idrc   ÚndimrÑ   r³   rY  r   r4  rø   rx   r%  rF  )r?   r-   rÙ   rZ  r[  r\  r]  rø   r^  rÜ   Úbszr¢   ÚchannelsÚdecoder_outputss                 rB   ri   zDiaModel.forward™  s  € ðH Ð Ð!8ÜØjóð ð ×)Ò)¨d¯mªmÙÜ×#Ñ#Øpôð "�	á˜Ð0Ü1´,ÀdÇkÁkÔ2RÔT`Ðhl×hsÑhsÔTtÓuˆOàÐ"Ø*˜dŸl™lð Ø#Ø-ñð ñ‰Oô ˜O¬_Ô=Ü-Ø"1°!Ñ"4Ü47¸Ó4HÈ1Ò4L˜o¨aÒ0ÐRVÜ14°_Ó1EÈÒ1I˜?¨1Ò-ÈtôˆOð #2°!Ñ"4×":Ñ":¸1Ñ"=¸rÀ4Ç;Á;×C]ÑC]×CjÑCj�hˆWˆØÐ$Ü %§
¡
Ø˜1˜hÐ'°D·K±K×4NÑ4N×4[Ñ4[Ðdh×doÑdoô!Ðð ×!Ñ! QÒ&Ø 1× 9Ñ 9¸#¸xÈÓ Q× [Ñ [Ð\]Ð_`Ó aÐà&˜$Ÿ,™,ð 	
Ø'Ø-Ø1Ø"1°!Ñ"4Ø#1Ø+Øñ	
ð ñ	
ˆô "Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5°aÑ&8Ø"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
rC   )NNNNNNNN)rD   rE   rF   r%   rV   r   r   r7   rQ  r   r�   r   r  r   ri   rP   rQ   s   @rB   rT  rT  Œ  sø   ø„ ð˜yõ ð Øð .2Ø26Ø59Ø8<Ø:>Ø:>Ø6:Ø!%ñ]
à×#Ñ# dÑ*ð]
ð ×(Ñ(¨4Ñ/ð]
ð !×+Ñ+¨dÑ2ð	]
ð
 $×.Ñ.°Ñ5ð]
ð !&× 0Ñ 0°4Ñ 7ð]
ð )¨5Ñ0°4Ñ7ð]
ð -¨tÑ3ð]
ð ˜$‘;ð]
ð 
Ð#Ñ	#ò]
ó ó ô]
rC   rT  zl
    The Dia model consisting of a (byte) text encoder and audio decoder with a prediction head on top.
    c                   ó>  ‡ — e Zd ZdZdZdefˆ fd„Zee	 	 	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  deez  dz  dedz  dedz  de	j                  dz  deez  fd„«       «       Zˆ xZS )ÚDiaForConditionalGenerationr,   )Úaudior+   c                 ó|  •— t         ‰| �  |«       || _        t        |«      | _        |j
                  j                  | _        |j
                  j                  | _        t        j                  |j
                  j                  | j                  | j                  z  d¬«      | _        d| _        | j                  «        y )NFro   ÚForMaskedLM)r3   rV   r+   rT  r,   rX  r9   r;   r   rq   rX   Úlogits_denseÚ	loss_typer/  rw   s     €rB   rV   z$DiaForConditionalGeneration.__init__  s“   ø€ Ü‰Ñ˜Ô ØˆŒÜ˜fÓ%ˆŒ
à"×1Ñ1×>Ñ>ˆÔØ ×/Ñ/×:Ñ:ˆŒÜŸI™IØ×!Ñ!×-Ñ-°×0AÑ0AÀDÇOÁOÑ0SÐ[`ô
ˆÔð 'ˆŒð 	�‰ÕrC   Nr-   rÙ   rZ  r[  r\  r]  rø   r^  Úlabelsr\   c
                 óX  —  | j                   d	||||||||dœ|
¤Ž}|d   }|j                  d   }| j                  |«      j                  |d| j                  | j
                  f«      j                  dd«      j                  «       j                  || j                  z  d| j
                  «      }d}|	�  | j                  d	||	| j
                  dœ|
¤Ž}t        |||j                  |j                  |j                  |j                  |j                  |j                  |j                   ¬«	      S )
a   
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
        or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
            1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
            the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
            tened audio logits which are used to calculate the loss.

            2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
            Dia to calculate embeddings and subsequent steps more efficiently.

            If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
            `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
            [`DiaProcessor.__call__`] for more details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

            [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size * num_codebooks,)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in
            `[0, ..., config.decoder_config.vocab_size - 1]` or -100. Tokens with indices set to `-100`
            are ignored (masked).
        )r-   rÙ   rZ  r[  r\  r]  rø   r^  r   r^   r$   r_   N)Úlogitsrx  r;   )	Úlossrz  rø   ra  rb  rF  rc  r=  rd  re  )r,   re   rv  rd   r9   r;   r³   rä   Úloss_functionr   rø   ra  rb  rF  rc  r=  rd  )r?   r-   rÙ   rZ  r[  r\  r]  rø   r^  rx  rÜ   Úoutputsr4  rL  Úaudio_logitsr{  s                   rB   ri   z#DiaForConditionalGeneration.forward  s?  € ðR �$—*‘*ð 

ØØ)Ø/Ø!5Ø#9Ø+Ø+Øñ

ð ñ

ˆð $ A™JÐØ&×,Ñ,¨QÑ/ˆ
ð ×ÑÐ/Ó0ß‰T�:˜r 4×#4Ñ#4°d·o±oÐFÓGß‰Y�q˜!‹_ß‰Z‹\ß‰T�*˜t×0Ñ0Ñ0°"°d·o±oÓFð 	ð ˆØÐØ%�4×%Ñ%Ðo¨\À&ÐUY×UdÑUdÑoÐhnÑoˆDäØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ô

ð 
	
rC   )	NNNNNNNNN)rD   rE   rF   rH   Úoutput_modalitiesr%   rV   r   r   r7   rQ  r   r�   r   r  r   ri   rP   rQ   s   @rB   rr  rr  û  s  ø„ ð  ÐØ"Ðð˜yõ ð Øð .2Ø26Ø59Ø8<Ø:>Ø:>Ø6:Ø!%Ø*.ñL
à×#Ñ# dÑ*ðL
ð ×(Ñ(¨4Ñ/ðL
ð !×+Ñ+¨dÑ2ð	L
ð
 $×.Ñ.°Ñ5ðL
ð !&× 0Ñ 0°4Ñ 7ðL
ð )¨5Ñ0°4Ñ7ðL
ð -¨tÑ3ðL
ð ˜$‘;ðL
ð × Ñ  4Ñ'ðL
ð 
�Ñ	 òL
ó ó ôL
rC   rr  )rT  r*   rr  )r$   )rð   )QÚcollections.abcr   Útypingr   r7   r   Ú r   r<   Úactivationsr   Úcache_utilsr	   r
   r   Úintegrationsr   r   r   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r    Úutils.genericr!   r"   Úutils.output_capturingr#   Úconfiguration_diar%   r&   r'   Úgeneration_diar(   Ú
get_loggerrD   rh  r*   ÚModuler6   rm   r€   r”   rÅ   rÎ   rk   rÀ   rÕ   r’   ré   rë   r  r.   r$  r/   rE  rT  rr  Ú__all__re  rC   rB   ú<module>r•     sn  ðõ* %Ý ã Ý å &Ý !ß CÑ Cß fÑ fß JÝ BÝ 9÷ó ÷ Lß FÝ &ß lÕ lß GÝ 5ß LÑ LÝ .ð 
ˆ×	Ñ	˜HÓ	%€ð ô0˜ó 0ó ð0ô$!˜rŸy™yô !ô<)ˆR�Y‰Yô )ñ$ ˜YÓ'ôJ�—‘ó Jó (ðJô(><˜Ÿ™ô ><òB(ñ Ð*Ó+òó ,ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(ó%ñ2 Ð)Ó*ô;)�r—y‘yó ;)ó +ð;)ô|G)˜Ÿ	™	ô G)ôTÐ0ô ôB5@Ð#ô 5@ôp6Ð0ô 6ôrZ
Ð#ô Z
ñz ðôô
g
Ð!ó g
óð
g
ñT ðôô
a
Ð"4Ð6Hó a
óð
a
òH L�rC   