Ë
    ÛÓ^jl  ã                   óÆ  — d Z ddlmZ ddlZddlmZ ddlmZ ddlm	Z	m
Z
 ddlmZmZ dd	lmZ dd
lmZ ddlmZmZmZmZ ddlmZmZ ddlmZ ddlmZmZmZm Z m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z'm(Z(m)Z)m*Z* ddl+m,Z, ddl-m.Z.m/Z/m0Z0 ddl1m2Z2  e!jf                  e4«      Z5e G d„ de«      «       Z6 G d„ dejn                  «      Z8 G d„ de,«      Z9 G d„ de(«      Z: G d„ d e)«      Z; G d!„ d"e'«      Z< G d#„ d$ejn                  «      Z= G d%„ d&e«      Z> G d'„ d(e6«      Z? G d)„ d*e«      Z@ G d+„ d,e6«      ZA ed-¬.«       G d/„ d0e6«      «       ZB ed1¬.«       G d2„ d3e6e2«      «       ZCg d4¢ZDy)5zPyTorch Dia model.é    )ÚCallableN)Únné   )Úinitialization)ÚDynamicCacheÚEncoderDecoderCache)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_torchdynamo_compilingÚlogging)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚLlamaAttentionÚLlamaRMSNormÚLlamaRotaryEmbeddingÚeager_attention_forward)ÚPhi3MLPé   )Ú	DiaConfigÚDiaDecoderConfigÚDiaEncoderConfig)ÚDiaGenerationMixinc                   óN   ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZddgZˆ fd„Zˆ xZS )ÚDiaPreTrainedModelÚconfigÚmodelTÚ	input_idsÚDiaEncoderLayerÚDiaDecoderLayerc                 ó&  •— t         ‰| �  |«       t        |t        «      rqt	        j
                  | j                  j                  t        j                  ¬«      | j                  j                  z  }t        j                  |j                  |«       y y )N©Údtype)ÚsuperÚ_init_weightsÚ
isinstanceÚDiaMultiChannelEmbeddingÚtorchÚaranger(   Únum_channelsÚlongÚ
vocab_sizeÚinitÚcopy_Úoffsets)ÚselfÚmoduler;   Ú	__class__s      €ún/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/dia/modular_dia.pyr1   z DiaPreTrainedModel._init_weights?   sb   ø€ Ü‰Ñ˜fÔ%Ü�fÔ6Ô7Ü—l‘l 4§;¡;×#;Ñ#;Ä5Ç:Á:ÔNÐQU×Q\ÑQ\×QgÑQgÑgˆGÜ�J‰J�v—~‘~ wÕ/ð 8ó    )Ú__name__Ú
__module__Ú__qualname__r"   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚmain_input_nameÚ_no_split_modulesr1   Ú__classcell__©r>   s   @r?   r'   r'   3   sG   ø… àÓØÐØ&*Ð#ØÐØ€NØÐØ!ÐØ!€OØ*Ð,=Ð>Ð÷0ð 0r@   r'   c                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )r3   a  In order to efficiently compute the audio embedding from the 9 different channels,
    we vectorize the embedding process by using a single embedding layer and an offset.
    Example:
    - num_embeds = 4
    - vocab_size = 8
    - num_channels = 3
    We would have offsets = [0, 8, 16]
    If audio_codes = [0, 1, 2, 3], [1, 3, 4, 7], [5, 6, 7, 8],
    then tokens = audio_codes + offsets
                = [0, 1, 2, 3, 9, 11, 12, 15, 21, 22, 23, 24]
    This allows us to use a single embedding layer for all channels.
    r(   c                 ó~  •— t         ‰| �  «        t        j                  |j                  |j
                  z  |j                  «      | _        |j                  | _        |j
                  | _        t        j                  |j
                  t        j                  ¬«      |j                  z  }| j                  d|d¬«       y )Nr.   r;   F)Ú
persistent)r0   Ú__init__r   Ú	Embeddingr8   r6   Úhidden_sizeÚembedr4   r5   r7   Úregister_buffer)r<   r(   r;   r>   s      €r?   rR   z!DiaMultiChannelEmbedding.__init__T   s�   ø€ Ü‰ÑÔÜ—\‘\ &×"3Ñ"3°f×6IÑ6IÑ"IÈ6×K]ÑK]Ó^ˆŒ
Ø!×-Ñ-ˆÔØ"×/Ñ/ˆÔÜ—,‘,˜v×2Ñ2¼%¿*¹*ÔEÈ×HYÑHYÑYˆØ×Ñ˜Y¨¸EÐÕBr@   Úaudio_codesÚreturnc                 ó^  — || j                   j                  |j                  «      z   j                  d|j                  d   |j                  d   z  «      }| j                  |«      j                  |j                  d   |j                  d   d| j                  «      }|j                  d¬«      S )Néÿÿÿÿr!   r   r   )Údim)r;   ÚtoÚdeviceÚviewÚshaperU   rT   Úsum)r<   rW   ÚtokensÚembedss       r?   Úforwardz DiaMultiChannelEmbedding.forward\   s—   € Ø §¡§¡°×0BÑ0BÓ CÑC×IÑIØ�×!Ñ! !Ñ$ {×'8Ñ'8¸Ñ';Ñ;ó
ˆð —‘˜FÓ#×(Ñ(¨¯©°a©¸+×:KÑ:KÈAÑ:NÐPRÐTX×TdÑTdÓeˆØ�z‰z˜aˆzÓ Ð r@   )
rA   rB   rC   Ú__doc__r#   rR   r4   ÚTensorrc   rM   rN   s   @r?   r3   r3   F   s2   ø„ ñðCÐ/õ Cð! 5§<¡<ð !°E·L±L÷ !r@   r3   c                   ó   — e Zd Zy)ÚDiaMLPN©rA   rB   rC   © r@   r?   rg   rg   d   ó   „ Ør@   rg   c                   ó   — e Zd Zy)Ú
DiaRMSNormNrh   ri   r@   r?   rl   rl   h   rj   r@   rl   c                   ó   — e Zd Zy)ÚDiaRotaryEmbeddingNrh   ri   r@   r?   rn   rn   l   rj   r@   rn   c                   ó,   — e Zd ZdZddeez  dedefd„Zy)ÚDiaSelfAttentionú=Multi-headed attention from 'Attention Is All You Need' paperr(   Ú	layer_idxÚ	is_causalc                 ó°  — t         j                  j                  | «       || _        || _        |j
                  | _        | j                  j                  | _        | j                  j                  xs | j                  | _        | j                  | j                  z  | _	        t        |d|j
                  | j                  z  «      | _        d| _        d| _        || _        t        j                  | j
                  | j                  | j                  z  d¬«      | _        t        j                  | j
                  | j                  | j                  z  d¬«      | _        t        j                  | j
                  | j                  | j                  z  d¬«      | _        t        j                  | j                  | j                  z  | j
                  d¬«      | _        y )NÚhead_dimr!   ç        F©Úbias)r   ÚModulerR   r(   rr   rT   Únum_attention_headsÚ	num_headsÚnum_key_value_headsÚnum_key_value_groupsÚgetattrru   ÚscalingÚattention_dropoutrs   ÚLinearÚq_projÚk_projÚv_projÚo_proj)r<   r(   rr   rs   s       r?   rR   zDiaSelfAttention.__init__s   sL  € Ü
�	‰	×Ñ˜4Ô ØˆŒØ"ˆŒØ!×-Ñ-ˆÔØŸ™×8Ñ8ˆŒØ#'§;¡;×#BÑ#BÒ#TÀdÇnÁnˆÔ Ø$(§N¡N°d×6NÑ6NÑ$NˆÔ!Ü ¨
°F×4FÑ4FÈ$Ï.É.Ñ4XÓYˆŒØˆŒØ!$ˆÔØ"ˆŒä—i‘i × 0Ñ 0°$·.±.À4Ç=Á=Ñ2PÐW\Ô]ˆŒÜ—i‘i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Ñ2ZÐafÔgˆŒÜ—i‘i × 0Ñ 0°$×2JÑ2JÈTÏ]É]Ñ2ZÐafÔgˆŒÜ—i‘i §¡°·±Ñ >À×@PÑ@PÐW\Ô]ˆ�r@   N)F)	rA   rB   rC   rd   r$   r#   ÚintÚboolrR   ri   r@   r?   rp   rp   p   s+   „ ÙGñ^Ð/Ð2BÑBð ^Èsð ^Ð_cô ^r@   rp   c                   óØ   ‡ — e Zd ZdZdedefˆ fd„Z	 	 ddej                  dej                  dej                  dz  d	e	dz  d
e
e   deej                  ej                  dz  f   fd„Zˆ xZS )ÚDiaCrossAttentionrq   r(   rr   c                 óf  •— t         ‰| �  «        || _        || _        |j                  | _        |j
                  | _        | j                  j                  | _        | j                  j                  | _	        | j                  | j                  z  | _
        |j                  | _        d| _        d| _        d| _        t!        j"                  | j                  | j                  | j                  z  d¬«      | _        t!        j"                  | j
                  | j                  | j                  z  d¬«      | _        t!        j"                  | j
                  | j                  | j                  z  d¬«      | _        t!        j"                  | j                  | j                  z  | j                  d¬«      | _        y )Nr!   rv   Frw   )r0   rR   r(   rr   rT   Úcross_hidden_sizeÚcross_num_attention_headsr{   Úcross_num_key_value_headsr|   r}   Úcross_head_dimru   r   r€   rs   r   r�   r‚   rƒ   r„   r…   ©r<   r(   rr   r>   s      €r?   rR   zDiaCrossAttention.__init__‰   s?  ø€ Ü‰ÑÔØˆŒØ"ˆŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔØŸ™×>Ñ>ˆŒØ#'§;¡;×#HÑ#HˆÔ Ø$(§N¡N°d×6NÑ6NÑ$NˆÔ!Ø×-Ñ-ˆŒØˆŒØ!$ˆÔØˆŒä—i‘i × 0Ñ 0°$·.±.À4Ç=Á=Ñ2PÐW\Ô]ˆŒÜ—i‘i × 6Ñ 6¸×8PÑ8PÐSW×S`ÑS`Ñ8`ÐglÔmˆŒÜ—i‘i × 6Ñ 6¸×8PÑ8PÐSW×S`ÑS`Ñ8`ÐglÔmˆŒÜ—i‘i §¡°·±Ñ >À×@PÑ@PÐW\Ô]ˆ�r@   NÚhidden_statesÚcross_attention_statesÚattention_maskÚpast_key_valuesÚkwargsrX   c                 óF  — |j                   d d }g |¢d‘| j                  ‘­}g |j                   d d ¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }	|�%|j
                  j                  | j                  «      nd}
|�]|
r[|j                  j                  | j                     j                  }|j                  j                  | j                     j                  }n¥| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }|�C|j                  j                  ||| j                  «      \  }}d|j
                  | j                  <   t        j                   | j"                  j$                  t&        «      } || |	|||fd| j(                  i|¤Ž\  }}|j+                  g |¢d‘­«      j-                  «       }| j/                  |«      }||fS )NrZ   r!   r   FTr   )r_   ru   r‚   r^   Ú	transposeÚ
is_updatedÚgetrr   Úcross_attention_cacheÚlayersÚkeysÚvaluesrƒ   r„   Úupdater   Úget_interfacer(   Ú_attn_implementationr   r   ÚreshapeÚ
contiguousr…   )r<   r�   r‘   r’   r“   r”   Úinput_shapeÚhidden_shapeÚcross_shapeÚquery_statesr—   Ú
key_statesÚvalue_statesÚattention_interfaceÚattn_outputÚattn_weightss                   r?   rc   zDiaCrossAttention.forwardœ   s  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆØMÐ.×4Ñ4°S°bÐ9ÐM¸2ÐM¸t¿}¹}ÑMˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆàGVÐGb�_×/Ñ/×3Ñ3°D·N±NÔCÐhmˆ
ØÐ&©:à(×>Ñ>×EÑEÀdÇnÁnÑU×ZÑZˆJØ*×@Ñ@×GÑGÈÏÉÑW×^Ñ^‰LàŸ™Ð%;Ó<×AÑAÀ+ÓN×XÑXÐYZÐ\]Ó^ˆJØŸ;™;Ð'=Ó>×CÑCÀKÓP×ZÑZÐ[\Ð^_Ó`ˆLàÐ*à+:×+PÑ+P×+WÑ+WØØ Ø—N‘Nó,Ñ(�
˜Lð >B�×*Ñ*¨4¯>©>Ñ:ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØñ%
ð —L‘Lð%
ð ñ%
Ñ!ˆ�\ð "×)Ñ)Ð*<¨KÐ*<¸Ñ*<Ó=×HÑHÓJˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r@   ©NN)rA   rB   rC   rd   r#   r†   rR   r4   re   r   r   r   Útuplerc   rM   rN   s   @r?   r‰   r‰   †   s—   ø„ ÙGð^Ð/ð ^¸Cõ ^ð. /3Ø6:ñ1)à—|‘|ð1)ð !&§¡ð1)ð Ÿ™ tÑ+ð	1)ð
 -¨tÑ3ð1)ð Ð-Ñ.ð1)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷1)r@   r‰   c                   óî   ‡ — e Zd Zdedefˆ fd„Z	 	 ddej                  deej                  ej                  f   dz  dej                  dz  de	e
   d	eej                  ej                  dz  f   f
d
„Zˆ xZS )r+   r(   rr   c                 ó   •— t         ‰| �  «        t        |j                  |j                  ¬«      | _        t        ||d¬«      | _        t        |j                  |j                  ¬«      | _        t        |«      | _
        y )N©ÚepsF©rs   )r0   rR   rl   rT   Únorm_epsÚpre_sa_normrp   Úself_attentionÚpost_sa_normrg   Úmlpr�   s      €r?   rR   zDiaEncoderLayer.__init__Ñ   s\   ø€ Ü‰ÑÔÜ% f×&8Ñ&8¸f¿o¹oÔNˆÔÜ.¨v°yÈEÔRˆÔÜ& v×'9Ñ'9¸v¿¹ÔOˆÔÜ˜&“>ˆ�r@   Nr�   Úposition_embeddingsr’   r”   rX   c                 óº   — |}| j                  |«      } | j                  |f||dœ|¤Ž\  }}||z   }|}| j                  |«      }| j                  |«      }	||	z   }|S )N)r·   r’   )r³   r´   rµ   r¶   )
r<   r�   r·   r’   r”   ÚresidualÚnormed_statesÚself_attn_outputÚ_Úmlp_outs
             r?   rc   zDiaEncoderLayer.forwardØ   s‰   € ð !ˆØ×(Ñ(¨Ó7ˆØ1˜d×1Ñ1Øð
à 3Ø)ñ
ð ñ	
ÑÐ˜!ð !Ð#3Ñ3ˆà ˆØ×)Ñ)¨-Ó8ˆØ—(‘(˜=Ó)ˆØ  7Ñ*ˆàÐr@   r«   )rA   rB   rC   r$   r†   rR   r4   re   r¬   r   r   rc   rM   rN   s   @r?   r+   r+   Ð   s—   ø„ ð"Ð/ð "¸Cõ "ð IMØ.2ñ	à—|‘|ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ÿ™ tÑ+ð	ð
 Ð-Ñ.ðð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷r@   r+   c                   óœ   ‡ — e Zd ZeedœZdefˆ fd„Zee	e
	 d
dej                  dej                  dz  dee   defd	„«       «       «       Zˆ xZS )Ú
DiaEncoder)r�   Ú
attentionsr(   c           	      ó²  •— t         ‰| �  |«       || _        t        j                  |j
                  |j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                  ¬«      | _        t!        |¬«      | _        | j%                  «        y c c}w ©Nr¯   ©r(   )r0   rR   r(   r   rS   r8   rT   Ú	embeddingÚ
ModuleListÚrangeÚnum_hidden_layersr+   rš   rl   r²   Únormrn   Ú
rotary_embÚ	post_initr�   s      €r?   rR   zDiaEncoder.__init__÷   s›   ø€ Ü‰Ñ˜Ô ØˆŒäŸ™ f×&7Ñ&7¸×9KÑ9KÓLˆŒÜ—m‘mÜAFÀv×G_ÑG_ÓA`Öa°IŒ_˜V YÕ/Òaó
ˆŒô ˜v×1Ñ1°v·±ÔGˆŒ	Ü,°FÔ;ˆŒà�‰Õùò bs   Á-CNr*   r’   r”   rX   c                 óZ  — | j                  |«      }t        j                  |j                  d   |j                  ¬«      d d d …f   }t        | j                  ||¬«      }| j                  ||¬«      }| j                  D ]  } ||f|||dœ|¤Ž}Œ | j                  |«      }t        |¬«      S )NrZ   ©r]   )r(   Úinputs_embedsr’   ©Úposition_ids)r’   rÏ   r·   )Úlast_hidden_state)rÄ   r4   r5   r_   r]   r	   r(   rÉ   rš   rÈ   r   )r<   r*   r’   r”   r�   rÏ   r·   Úencoder_layers           r?   rc   zDiaEncoder.forward  s¾   € ð Ÿ™ yÓ1ˆô
 —|‘| I§O¡O°BÑ$7À	×@PÑ@PÔQÐRVÒXYÐRYÑZˆä2Ø—;‘;Ø'Ø)ô
ˆð
 #Ÿo™o¨mÈ,˜oÓWÐà!Ÿ[™[ò 	ˆMÙ)Øðà-Ø)Ø$7ñ	ð
 ñ‰Mð	ð Ÿ	™	 -Ó0ˆä°Ô?Ð?r@   ©N)rA   rB   rC   r+   rp   Ú_can_record_outputsr$   rR   r   r   r   r4   re   r   r   r   rc   rM   rN   s   @r?   r¿   r¿   ñ   s†   ø„ à(Ø&ñÐð
Ð/õ ð  ØØð /3ñ@à—<‘<ð@ð Ÿ™ tÑ+ð@ð Ð+Ñ,ð	@ð
 
ò@ó ó ó  ô@r@   r¿   c                   óL  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 ddej                  deej                  ej                  f   dz  dej                  dz  dej                  dz  d	ej                  dz  d
e	dz  deej                  ej                  dz  ej                  dz  f   fd„Z
ˆ xZS )r,   r(   rr   c                 ó�  •— t         ‰| �  «        |j                  | _        t	        ||d¬«      | _        t        ||«      | _        t        |j                  |j                  ¬«      | _
        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        |«      | _        y )NTr±   r¯   )r0   rR   rT   Ú	embed_dimrp   r´   r‰   Úcross_attentionrl   r²   r³   Úpre_ca_normÚpre_mlp_normrg   r¶   r�   s      €r?   rR   zDiaDecoderLayer.__init__*  s‘   ø€ Ü‰ÑÔØ×+Ñ+ˆŒÜ.¨v°yÈDÔQˆÔÜ0°¸ÓCˆÔÜ% f×&8Ñ&8¸f¿o¹oÔNˆÔÜ% f×&8Ñ&8¸f¿o¹oÔNˆÔÜ& v×'9Ñ'9¸v¿¹ÔOˆÔÜ˜&“>ˆ�r@   Nr�   r·   r’   Úencoder_hidden_statesÚencoder_attention_maskr“   rX   c                 óZ  — |}t        |t        «      r|j                  }|}	| j                  |«      }
 | j                  |
|||fi |¤Ž\  }}|	|z   }|}	| j                  |«      }
 | j                  |
|f||dœ|¤Ž\  }}|	|z   }|}	| j                  |«      }
| j                  |
«      }|	|z   }|S )N)r’   r“   )	r2   r   Úself_attention_cacher³   r´   rØ   r×   rÙ   r¶   )r<   r�   r·   r’   rÚ   rÛ   r“   r”   Úself_attn_cacher¹   rº   r»   r¼   Úcross_statesr½   s                  r?   rc   zDiaDecoderLayer.forward4  sø   € ð *ˆÜ�oÔ':Ô;Ø-×BÑBˆOà ˆØ×(Ñ(¨Ó7ˆØ1˜d×1Ñ1ØØØð ñ
ð ñ
ÑÐ˜!ð !Ð#3Ñ3ˆà ˆØ×(Ñ(¨Ó7ˆØ.˜$×.Ñ.ØØ!ð
ð 2Ø+ñ	
ð
 ñ
‰ˆ�að ! <Ñ/ˆà ˆØ×)Ñ)¨-Ó8ˆØ—(‘(˜=Ó)ˆØ  7Ñ*ˆàÐr@   ©NNNNN)rA   rB   rC   r#   r†   rR   r4   re   r¬   r   rc   rM   rN   s   @r?   r,   r,   )  sÐ   ø„ ð"Ð/ð "¸Cõ "ð IMØ.2Ø59Ø6:Ø6:ñ+à—|‘|ð+ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð+ð Ÿ™ tÑ+ð	+ð
  %Ÿ|™|¨dÑ2ð+ð !&§¡¨tÑ 3ð+ð -¨tÑ3ð+ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%·,±,ÀÑ2EÐEÑ	F÷+r@   r,   c                   ó  ‡ — e Zd ZdZeeedœZdefˆ fd„Z	e
ee	 	 	 	 	 ddej                  dej                  dz  dej                  dz  d	ej                   dz  d
ej                  dz  dedz  dee   deez  fd„«       «       «       Zˆ xZS )Ú
DiaDecoderz-Transformer Decoder Stack using DenseGeneral.)r�   rÀ   Úcross_attentionsr(   c           	      óª  •— t         ‰| �  |«       |j                  | _        |j                  | _        t	        |«      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                  ¬«      | _        t!        |¬«      | _        | j%                  «        y c c}w rÂ   )r0   rR   r6   r8   r3   Ú
embeddingsr   rÅ   rÆ   rÇ   r,   rš   rl   rT   r²   rÈ   rn   rÉ   rÊ   r�   s      €r?   rR   zDiaDecoder.__init__k  s�   ø€ Ü‰Ñ˜Ô Ø"×/Ñ/ˆÔØ ×+Ñ+ˆŒÜ2°6Ó:ˆŒÜ—m‘mÜAFÀv×G_ÑG_ÓA`Öa°IŒ_˜V YÕ/Òaó
ˆŒô ˜v×1Ñ1°v·±ÔGˆŒ	Ü,°FÔ;ˆŒà�‰Õùò bs   Á)CNr*   rÏ   r’   rÚ   rÛ   r“   r”   rX   c                 óT  — |j                  «       dd \  }}	|�|j                  «       nd}
|€5t        j                  |	|j                  ¬«      |
z   }|j                  d«      }| j                  |«      }|€1t        «       s'|
|	z   }t        j                  |||j                  ¬«      }t        | j                  |||¬«      }t        | j                  |||¬«      }| j                  ||¬«      }| j                  D ]  } |||||f|||dœ|¤Ž}Œ | j                  |«      }t        ||¬	«      S )
a  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks)`):
            The original `decoder_input_ids` in 3D shape to facilitate more efficient computations.

            [What are input IDs?](../glossary#input-ids)
        NrZ   r   rÌ   )r(   rÍ   r’   r“   )r(   rÍ   r’   rÚ   rÎ   )rÛ   r“   rÏ   )rÐ   r“   )ÚsizeÚget_seq_lengthr4   r5   r]   Ú	unsqueezerå   r   Úonesr
   r(   r	   rÉ   rš   rÈ   r   )r<   r*   rÏ   r’   rÚ   rÛ   r“   r”   Ú
batch_sizeÚ
seq_lengthÚpast_key_values_lengthr�   Úmask_seq_lengthr·   Úlayers                  r?   rc   zDiaDecoder.forwardx  sW  € ð( "+§¡Ó!1°#°2Ð!6Ñˆ
�JØETÐE` ×!?Ñ!?Ô!AÐfgÐàÐÜ Ÿ<™<¨
¸9×;KÑ;KÔLÐOeÑeˆLØ'×1Ñ1°!Ó4ˆLð Ÿ™¨	Ó2ˆàÐ!Ô*BÔ*Dà4°zÑAˆOÜ"ŸZ™Z¨
°OÈI×L\ÑL\Ô]ˆNä+Ø—;‘;Ø'Ø)Ø+ô	
ˆô ";Ø—;‘;Ø'Ø1Ø"7ô	"
Ðð #Ÿo™o¨mÈ,˜oÓWÐà—[‘[ò 	ˆEÙ!Øð $ØØ%ðð (>Ø /Ø)ñð ñ‰Mð	ð Ÿ	™	 -Ó0ˆä8Ø+Ø+ô
ð 	
r@   rà   )rA   rB   rC   rd   r,   rp   r‰   rÓ   r#   rR   r   r   r   r4   re   Ú
LongTensorÚFloatTensorr   r   r   r   r¬   rc   rM   rN   s   @r?   râ   râ   b  sñ   ø„ Ù7ð )Ø&Ø-ñÐðÐ/õ ð  ØØð 15Ø.2Ø:>Ø:>Ø6:ñA
à—<‘<ðA
ð ×&Ñ&¨Ñ-ðA
ð Ÿ™ tÑ+ð	A
ð
  %×0Ñ0°4Ñ7ðA
ð !&× 0Ñ 0°4Ñ 7ðA
ð -¨tÑ3ðA
ð Ð+Ñ,ðA
ð 
3°UÑ	:òA
ó ó ó  ôA
r@   râ   z[
    The bare Dia model outputting raw hidden-states without any specific head on top.
    )Úcustom_introc                   ó  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
z  dz  d
edz  dedz  de
ez  fd„«       «       Zˆ xZS )ÚDiaModelr(   c                 óº   •— t         ‰| �  |«       || _        t        |j                  «      | _        t        |j                  «      | _        | j                  «        y rÒ   )
r0   rR   r(   r¿   Úencoder_configÚencoderrâ   Údecoder_configÚdecoderrÊ   ©r<   r(   r>   s     €r?   rR   zDiaModel.__init__Å  sE   ø€ Ü‰Ñ˜Ô ØˆŒÜ! &×"7Ñ"7Ó8ˆŒÜ! &×"7Ñ"7Ó8ˆŒØ�‰Õr@   Nr*   r’   Údecoder_input_idsÚdecoder_position_idsÚdecoder_attention_maskÚencoder_outputsr“   Ú	use_cacherX   c	                 óø  — |€|€t        d«      ‚| j                  r%| j                  r|rt        j	                  d«       d}|r6|€4t        t        | j                  ¬«      t        | j                  ¬«      «      }|€ | j                  d||dœ|	¤Ž}nGt        |t        «      s7t        |d   t        |«      dkD  r|d   ndt        |«      d	kD  r|d	   nd¬
«      }|d   j                  d   d| j                  j                  j                  }}}
|€Ct        j                   |
d|f| j                  j                  j"                  | j$                  ¬«      }|j&                  d	k(  r#|j)                  |
||«      j+                  dd	«      } | j,                  d||||d   |||dœ|	¤Ž}t/        |j0                  |j2                  |j4                  |j6                  |j8                  |d   |j4                  |j6                  ¬«      S )a\  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
        or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
            1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
            the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
            tened audio logits which are used to calculate the loss.

            2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
            Dia to calculate embeddings and subsequent steps more efficiently.

            If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
            `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
            [`DiaProcessor.__call__`] for more details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

            [What are position IDs?](../glossary#position-ids)
        NzXYou should either provide text ids or the cached text encodings. Neither has been found.zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...FrÃ   )r*   r’   r   r!   r   )rÐ   r�   rÀ   rZ   )rç   Ú
fill_valuer]   )r*   rÏ   r’   rÚ   rÛ   r“   rÿ   )rÐ   r“   Údecoder_hidden_statesÚdecoder_attentionsrã   Úencoder_last_hidden_staterÚ   Úencoder_attentionsri   )Ú
ValueErrorÚis_gradient_checkpointingÚtrainingÚloggerÚwarning_oncer   r   r(   r÷   r2   r   Úlenr_   rø   r6   r4   ÚfullÚbos_token_idr]   Úndimr    r–   rù   r   rÐ   r“   r�   rÀ   rã   )r<   r*   r’   rû   rü   rý   rþ   r“   rÿ   r”   ÚbszÚseq_lenÚchannelsÚdecoder_outputss                 r?   rc   zDiaModel.forwardÌ  s  € ðH Ð Ð!8ÜØjóð ð ×)Ò)¨d¯mªmÙÜ×#Ñ#Øpôð "�	á˜Ð0Ü1´,ÀdÇkÁkÔ2RÔT`Ðhl×hsÑhsÔTtÓuˆOàÐ"Ø*˜dŸl™lð Ø#Ø-ñð ñ‰Oô ˜O¬_Ô=Ü-Ø"1°!Ñ"4Ü47¸Ó4HÈ1Ò4L˜o¨aÒ0ÐRVÜ14°_Ó1EÈÒ1I˜?¨1Ò-ÈtôˆOð #2°!Ñ"4×":Ñ":¸1Ñ"=¸rÀ4Ç;Á;×C]ÑC]×CjÑCj�hˆWˆØÐ$Ü %§
¡
Ø˜1˜hÐ'°D·K±K×4NÑ4N×4[Ñ4[Ðdh×doÑdoô!Ðð ×!Ñ! QÒ&Ø 1× 9Ñ 9¸#¸xÈÓ Q× [Ñ [Ð\]Ð_`Ó aÐà&˜$Ÿ,™,ð 	
Ø'Ø-Ø1Ø"1°!Ñ"4Ø#1Ø+Øñ	
ð ñ	
ˆô "Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5°aÑ&8Ø"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
r@   )NNNNNNNN)rA   rB   rC   r"   rR   r   r   r4   rð   r   r¬   r   r‡   r   rc   rM   rN   s   @r?   rô   rô   ¿  sø   ø„ ð˜yõ ð Øð .2Ø26Ø59Ø8<Ø:>Ø:>Ø6:Ø!%ñ]
à×#Ñ# dÑ*ð]
ð ×(Ñ(¨4Ñ/ð]
ð !×+Ñ+¨dÑ2ð	]
ð
 $×.Ñ.°Ñ5ð]
ð !&× 0Ñ 0°4Ñ 7ð]
ð )¨5Ñ0°4Ñ7ð]
ð -¨tÑ3ð]
ð ˜$‘;ð]
ð 
Ð#Ñ	#ò]
ó ó ô]
r@   rô   zl
    The Dia model consisting of a (byte) text encoder and audio decoder with a prediction head on top.
    c                   ó>  ‡ — e Zd ZdZdZdefˆ fd„Zee	 	 	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  deez  dz  dedz  dedz  de	j                  dz  deez  fd„«       «       Zˆ xZS )ÚDiaForConditionalGenerationr)   )Úaudior(   c                 ó|  •— t         ‰| �  |«       || _        t        |«      | _        |j
                  j                  | _        |j
                  j                  | _        t        j                  |j
                  j                  | j                  | j                  z  d¬«      | _        d| _        | j                  «        y )NFrw   ÚForMaskedLM)r0   rR   r(   rô   r)   rø   r6   r8   r   r�   rT   Úlogits_denseÚ	loss_typerÊ   rú   s     €r?   rR   z$DiaForConditionalGeneration.__init__7  s“   ø€ Ü‰Ñ˜Ô ØˆŒÜ˜fÓ%ˆŒ
à"×1Ñ1×>Ñ>ˆÔØ ×/Ñ/×:Ñ:ˆŒÜŸI™IØ×!Ñ!×-Ñ-°×0AÑ0AÀDÇOÁOÑ0SÐ[`ô
ˆÔð 'ˆŒð 	�‰Õr@   Nr*   r’   rû   rü   rý   rþ   r“   rÿ   ÚlabelsrX   c
                 óX  —  | j                   d	||||||||dœ|
¤Ž}|d   }|j                  d   }| j                  |«      j                  |d| j                  | j
                  f«      j                  dd«      j                  «       j                  || j                  z  d| j
                  «      }d}|	�  | j                  d	||	| j
                  dœ|
¤Ž}t        |||j                  |j                  |j                  |j                  |j                  |j                  |j                   ¬«	      S )
a   
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size * num_codebooks, target_sequence_length)
        or (batch_size, target_sequence_length, num_codebooks)`, *optional*):
            1. (batch_size * num_codebooks, target_sequence_length): corresponds to the general use case where
            the audio input codebooks are flattened into the batch dimension. This also aligns with the flat-
            tened audio logits which are used to calculate the loss.

            2. (batch_size, sequence_length, num_codebooks): corresponds to the internally used shape of
            Dia to calculate embeddings and subsequent steps more efficiently.

            If no `decoder_input_ids` are provided, it will create a tensor of `bos_token_id` with shape
            `(batch_size, 1, num_codebooks)`. Indices can be obtained using the [`DiaProcessor`]. See
            [`DiaProcessor.__call__`] for more details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)
        decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
            Indices of positions of each input sequence tokens in the position embeddings.
            Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`.

            [What are position IDs?](../glossary#position-ids)
        labels (`torch.LongTensor` of shape `(batch_size * num_codebooks,)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in
            `[0, ..., config.decoder_config.vocab_size - 1]` or -100. Tokens with indices set to `-100`
            are ignored (masked).
        )r*   r’   rû   rü   rý   rþ   r“   rÿ   r   rZ   r!   r   N)Úlogitsr  r8   )	Úlossr  r“   r  r  rã   r  rÚ   r  ri   )r)   r_   r  r^   r6   r8   r–   r¡   Úloss_functionr   r“   r  r  rã   r  rÚ   r  )r<   r*   r’   rû   rü   rý   rþ   r“   rÿ   r  r”   ÚoutputsrÐ   rë   Úaudio_logitsr  s                   r?   rc   z#DiaForConditionalGeneration.forwardF  s?  € ðR �$—*‘*ð 

ØØ)Ø/Ø!5Ø#9Ø+Ø+Øñ

ð ñ

ˆð $ A™JÐØ&×,Ñ,¨QÑ/ˆ
ð ×ÑÐ/Ó0ß‰T�:˜r 4×#4Ñ#4°d·o±oÐFÓGß‰Y�q˜!‹_ß‰Z‹\ß‰T�*˜t×0Ñ0Ñ0°"°d·o±oÓFð 	ð ˆØÐØ%�4×%Ñ%Ðo¨\À&ÐUY×UdÑUdÑoÐhnÑoˆDäØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ô

ð 
	
r@   )	NNNNNNNNN)rA   rB   rC   rE   Úoutput_modalitiesr"   rR   r   r   r4   rð   r   r¬   r   r‡   r   rc   rM   rN   s   @r?   r  r  .  s  ø„ ð  ÐØ"Ðð˜yõ ð Øð .2Ø26Ø59Ø8<Ø:>Ø:>Ø6:Ø!%Ø*.ñL
à×#Ñ# dÑ*ðL
ð ×(Ñ(¨4Ñ/ðL
ð !×+Ñ+¨dÑ2ð	L
ð
 $×.Ñ.°Ñ5ðL
ð !&× 0Ñ 0°4Ñ 7ðL
ð )¨5Ñ0°4Ñ7ðL
ð -¨tÑ3ðL
ð ˜$‘;ðL
ð × Ñ  4Ñ'ðL
ð 
�Ñ	 òL
ó ó ôL
r@   r  )rô   r'   r  )Erd   Úcollections.abcr   r4   r   Ú r   r9   Úcache_utilsr   r   Úmasking_utilsr	   r
   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úllama.modeling_llamar   r   r   r   Úphi3.modeling_phi3r    Úconfiguration_diar"   r#   r$   Úgeneration_diar%   Ú
get_loggerrA   r	  r'   ry   r3   rg   rl   rn   rp   r‰   r+   r¿   r,   râ   rô   r  Ú__all__ri   r@   r?   ú<module>r4     sy  ðñ å $ã Ý å &ß <ß JÝ BÝ 9÷ó ÷ GÝ &ß lÕ lÝ 7Ý 5÷ó õ )ß LÑ LÝ .ð 
ˆ×	Ñ	˜HÓ	%€ð ô0˜ó 0ó ð0ô$!˜rŸy™yô !ô<	ˆWô 	ô	�ô 	ô	Ð-ô 	ô^�~ô ^ô,G)˜Ÿ	™	ô G)ôTÐ0ô ôB5@Ð#ô 5@ôp6Ð0ô 6ôrZ
Ð#ô Z
ñz ðôô
g
Ð!ó g
óð
g
ñT ðôô
a
Ð"4Ð6Hó a
óð
a
òH L�r@   