Ë
    ÜÓ^jÑÁ  ã                   óÐ  — d Z ddlZddlZddlZddlmZ ddlmZ ddlmZ ddl	m
Z ddlmZ dd	lmZmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZ ddlmZ ddlm Z m!Z!m"Z" ddl#m$Z$  e"jJ                  e&«      Z' G d„ dejP                  «      Z) G d„ dejP                  «      Z* G d„ dejP                  «      Z+ G d„ dejP                  «      Z, G d„ dejP                  «      Z- G d„ dejP                  «      Z. G d„ dejP                  «      Z/ G d „ d!e«      Z0e  G d"„ d#e«      «       Z1 G d$„ d%e1«      Z2 G d&„ d'ejP                  «      Z3 e d(¬)«       G d*„ d+e1e«      «       Z4d+d#gZ5y),zPyTorch Pop2Piano model.é    N)Únn)ÚCrossEntropyLoss)ÚGenerationConfigé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutput)ÚPreTrainedModel)Úauto_docstringÚis_torchdynamo_compilingÚloggingé   )ÚPop2PianoConfigc                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚPop2PianoLayerNormc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)zj
        Construct a layernorm module in the Pop2Piano style. No bias and no subtraction of mean.
        N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s      €ú{/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/pop2piano/modeling_pop2piano.pyr   zPop2PianoLayerNorm.__init__*   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕó    c                 óž  — |j                  t        j                  «      j                  d«      j	                  dd¬«      }|t        j
                  || j                  z   «      z  }| j                  j                  t        j                  t        j                  fv r%|j                  | j                  j                  «      }| j                  |z  S )Né   éÿÿÿÿT)Úkeepdim)Útor   Úfloat32ÚpowÚmeanÚrsqrtr"   r!   ÚdtypeÚfloat16Úbfloat16)r#   Úhidden_statesÚvariances      r'   ÚforwardzPop2PianoLayerNorm.forward2   s›   € ð !×#Ñ#¤E§M¡MÓ2×6Ñ6°qÓ9×>Ñ>¸rÈ4Ð>ÓPˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆð �;‰;×Ñ¤§¡´·±Ð ?Ñ?Ø)×,Ñ,¨T¯[©[×->Ñ->Ó?ˆMà�{‰{˜]Ñ*Ð*r(   )g�íµ ÷Æ°>)Ú__name__Ú
__module__Ú__qualname__r   r7   Ú__classcell__©r&   s   @r'   r   r   )   s   ø„ õ$ö+r(   r   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚPop2PianoDenseActDenseÚconfigc                 ó^  •— t         ‰| �  «        t        j                  |j                  |j
                  d¬«      | _        t        j                  |j
                  |j                  d¬«      | _        t        j                  |j                  «      | _
        t        |j                     | _        y ©NF©Úbias)r   r   r   ÚLinearÚd_modelÚd_ffÚwiÚwoÚDropoutÚdropout_rateÚdropoutr   Údense_act_fnÚact©r#   r?   r&   s     €r'   r   zPop2PianoDenseActDense.__init__D   sn   ø€ Ü‰ÑÔÜ—)‘)˜FŸN™N¨F¯K©K¸eÔDˆŒÜ—)‘)˜FŸK™K¨¯©¸eÔDˆŒÜ—z‘z &×"5Ñ"5Ó6ˆŒÜ˜&×-Ñ-Ñ.ˆ�r(   c                 ó  — | j                  |«      }| j                  |«      }| j                  |«      }t        | j                  j
                  t        j                  «      r�|j                  | j                  j
                  j                  k7  r`| j                  j
                  j                  t        j                  k7  r/|j                  | j                  j
                  j                  «      }| j	                  |«      }|S ©N)rG   rM   rK   Ú
isinstancerH   r!   r   ÚTensorr2   Úint8r-   )r#   r5   s     r'   r7   zPop2PianoDenseActDense.forwardK   sª   € ØŸ™ Ó.ˆØŸ™ Ó/ˆØŸ™ ]Ó3ˆä�t—w‘w—~‘~¤u§|¡|Ô4Ø×#Ñ# t§w¡w§~¡~×';Ñ';Ò;Ø—‘—‘×$Ñ$¬¯
©
Ò2à)×,Ñ,¨T¯W©W¯^©^×-AÑ-AÓBˆMØŸ™ Ó.ˆØÐr(   ©r8   r9   r:   r   r   r7   r;   r<   s   @r'   r>   r>   C   s   ø„ ð/˜õ /ör(   r>   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚPop2PianoDenseGatedActDenser?   c                 óÀ  •— t         ‰| �  «        t        j                  |j                  |j
                  d¬«      | _        t        j                  |j                  |j
                  d¬«      | _        t        j                  |j
                  |j                  d¬«      | _        t        j                  |j                  «      | _        t        |j                     | _        y rA   )r   r   r   rD   rE   rF   Úwi_0Úwi_1rH   rI   rJ   rK   r   rL   rM   rN   s     €r'   r   z$Pop2PianoDenseGatedActDense.__init__[   sŠ   ø€ Ü‰ÑÔÜ—I‘I˜fŸn™n¨f¯k©kÀÔFˆŒ	Ü—I‘I˜fŸn™n¨f¯k©kÀÔFˆŒ	Ü—)‘)˜FŸK™K¨¯©¸eÔDˆŒÜ—z‘z &×"5Ñ"5Ó6ˆŒÜ˜&×-Ñ-Ñ.ˆ�r(   c                 ó,  — | j                  | j                  |«      «      }| j                  |«      }||z  }| j                  |«      }t	        | j
                  j                  t        j                  «      r�|j                  | j
                  j                  j                  k7  r`| j
                  j                  j                  t        j                  k7  r/|j                  | j
                  j                  j                  «      }| j                  |«      }|S rP   )rM   rX   rY   rK   rQ   rH   r!   r   rR   r2   rS   r-   )r#   r5   Úhidden_geluÚhidden_linears       r'   r7   z#Pop2PianoDenseGatedActDense.forwardc   s¿   € Ø—h‘h˜tŸy™y¨Ó7Ó8ˆØŸ	™	 -Ó0ˆØ# mÑ3ˆØŸ™ ]Ó3ˆô �t—w‘w—~‘~¤u§|¡|Ô4Ø×#Ñ# t§w¡w§~¡~×';Ñ';Ò;Ø—‘—‘×$Ñ$¬¯
©
Ò2à)×,Ñ,¨T¯W©W¯^©^×-AÑ-AÓBˆMàŸ™ Ó.ˆØÐr(   rT   r<   s   @r'   rV   rV   Z   s   ø„ ð/˜õ /ör(   rV   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚPop2PianoLayerFFr?   c                 ó  •— t         ‰| �  «        |j                  rt        |«      | _        nt        |«      | _        t        |j                  |j                  ¬«      | _	        t        j                  |j                  «      | _        y )N©r%   )r   r   Úis_gated_actrV   ÚDenseReluDenser>   r   rE   Úlayer_norm_epsilonÚ
layer_normr   rI   rJ   rK   rN   s     €r'   r   zPop2PianoLayerFF.__init__y   s_   ø€ Ü‰ÑÔØ×ÒÜ"=¸fÓ"EˆDÕä"8¸Ó"@ˆDÔä,¨V¯^©^À×AZÑAZÔ[ˆŒÜ—z‘z &×"5Ñ"5Ó6ˆ�r(   c                 ór   — | j                  |«      }| j                  |«      }|| j                  |«      z   }|S rP   )rd   rb   rK   )r#   r5   Úforwarded_statess      r'   r7   zPop2PianoLayerFF.forwardƒ   s=   € ØŸ?™?¨=Ó9ÐØ×.Ñ.Ð/?Ó@ÐØ%¨¯©Ð5EÓ(FÑFˆØÐr(   rT   r<   s   @r'   r^   r^   x   s   ø„ ð7˜õ 7ör(   r^   c                   ó`   ‡ — e Zd Z	 	 ddededz  fˆ fd„Zed	d„«       Zd
d„Z	 	 	 	 	 dd„Z	ˆ xZ
S )ÚPop2PianoAttentionNr?   Ú	layer_idxc                 óê  •— t         ‰| �  «        |j                  | _        || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _
        |j                  | _        | j                  | j                  z  | _        || _        |€9| j                  r-t        j!                  d| j"                  j$                  › d�«       t'        j(                  | j                  | j                  d¬«      | _        t'        j(                  | j                  | j                  d¬«      | _        t'        j(                  | j                  | j                  d¬«      | _        t'        j(                  | j                  | j                  d¬«      | _        | j                  r/t'        j2                  | j                  | j                  «      | _        d| _        y )NzInstantiating a decoder z³ without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.FrB   )r   r   Ú
is_decoderÚhas_relative_attention_biasÚrelative_attention_num_bucketsÚrelative_attention_max_distancerE   Úd_kvÚkey_value_proj_dimÚ	num_headsÚn_headsrJ   rK   Ú	inner_dimri   ÚloggerÚwarning_oncer&   r8   r   rD   ÚqÚkÚvÚoÚ	EmbeddingÚrelative_attention_biasÚgradient_checkpointing©r#   r?   rl   ri   r&   s       €r'   r   zPop2PianoAttention.__init__Œ   si  ø€ ô 	‰ÑÔØ ×+Ñ+ˆŒØ+FˆÔ(Ø.4×.SÑ.SˆÔ+Ø/5×/UÑ/UˆÔ,Ø—~‘~ˆŒØ"(§+¡+ˆÔØ×'Ñ'ˆŒØ×*Ñ*ˆŒØŸ™¨×(?Ñ(?Ñ?ˆŒØ"ˆŒØÐ §¢Ü×ÑØ*¨4¯>©>×+BÑ+BÐ*Cð D,ð ,ôô —‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ>™>¨4¯<©<¸eÔDˆŒà×+Ò+Ü+-¯<©<¸×8[Ñ8[Ð]a×]iÑ]iÓ+jˆDÔ(à&+ˆÕ#r(   c                 óT  — d}|rC|dz  }|| dkD  j                  t        j                  «      |z  z  }t        j                  | «      } n*t        j                  | t        j
                  | «      «       } |dz  }| |k  }|t        j                  | j                  «       |z  «      t        j                  ||z  «      z  ||z
  z  j                  t        j                  «      z   }t        j                  |t        j                  ||dz
  «      «      }|t        j                  || |«      z  }|S )aÒ  
        Adapted from Mesh Tensorflow:
        https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593

        Translate relative position to a bucket number for relative attention. The relative position is defined as
        memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
        position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
        small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
        positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
        This should allow for more graceful generalization to longer sequences than the model has been trained on

        Args:
            relative_position: an int32 Tensor
            bidirectional: a boolean - whether the attention is bidirectional
            num_buckets: an integer
            max_distance: an integer

        Returns:
            a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
        r   r*   r   )r-   r   ÚlongÚabsÚminÚ
zeros_likeÚlogÚfloatÚmathÚ	full_likeÚwhere)Úrelative_positionÚbidirectionalÚnum_bucketsÚmax_distanceÚrelative_bucketsÚ	max_exactÚis_smallÚrelative_position_if_larges           r'   Ú_relative_position_bucketz,Pop2PianoAttention._relative_position_bucket®   s(  € ð, ÐÙØ˜AÑˆKØÐ!2°QÑ!6× :Ñ :¼5¿:¹:Ó FÈÑ TÑTÐÜ %§	¡	Ð*;Ó <Ñä!&§¡Ð+<¼e×>NÑ>NÐO`Ó>aÓ!bÐ bÐð   1Ñ$ˆ	Ø$ yÑ0ˆð &/Ü�I‰IÐ'×-Ñ-Ó/°)Ñ;Ó<Ü�h‰h�| iÑ/Ó0ñ1à˜YÑ&ñ(÷ ‰"ŒU�Z‰Z‹.ñ	&Ð"ô
 &+§Y¡YØ&¬¯©Ð8RÐT_ÐbcÑTcÓ(dó&
Ð"ð 	œEŸK™K¨Ð2CÐE_Ó`Ñ`ÐØÐr(   c                 óÜ  — |€ | j                   j                  j                  }t        j                  |t        j
                  |¬«      dd…df   |z   }t        j                  |t        j
                  |¬«      ddd…f   }||z
  }| j                  || j                   | j                  | j                  ¬«      }| j                  |«      }	|	j                  g d¢«      j                  d«      }	|	S )z%Compute binned relative position biasN)r2   Údevice)r‰   rŠ   r‹   )r*   r   r   r   )r{   r!   r’   r   Úaranger   r�   rk   rm   rn   ÚpermuteÚ	unsqueeze)
r#   Úquery_lengthÚ
key_lengthr’   Úpast_seen_tokensÚcontext_positionÚmemory_positionrˆ   Úrelative_position_bucketÚvaluess
             r'   Úcompute_biaszPop2PianoAttention.compute_biasÞ   sÜ   € àˆ>Ø×1Ñ1×8Ñ8×?Ñ?ˆFÜ Ÿ<™<¨¼E¿J¹JÈvÔVÒWXÐZ^ÐW^Ñ_ÐbrÑrÐÜŸ,™, z¼¿¹ÈFÔSÐTXÒZ[ÐT[Ñ\ˆØ+Ð.>Ñ>ÐØ#'×#AÑ#AØØ#Ÿ™Ð.Ø×;Ñ;Ø×=Ñ=ð	 $Bó $
Ð ð ×-Ñ-Ð.FÓGˆØ—‘¢	Ó*×4Ñ4°QÓ7ˆØˆr(   c                 óÊ  — |j                   dd }g |¢d‘| j                  ‘­}	|�|j                  | j                  «      nd}
t	        |
t
        j                  «      r|
j                  «       n|
}
|du}| j                  |«      j                  |	«      j                  dd«      }d}t	        |t        «      rA|j                  j                  | j                  «      }|r|j                  }n|j                  }n|}|r|n|}|rK|�I|rG|j                   | j                     j"                  }|j                   | j                     j$                  }nÍg |j                   dd ¢d‘| j                  ‘­}| j'                  |«      j                  |«      j                  dd«      }| j)                  |«      j                  |«      j                  dd«      }|�K|j+                  ||| j                  «      \  }}|r)t	        |t        «      rd|j                  | j                  <   t        j,                  ||j                  dd«      «      }|€Â|j                   d	   }| j.                  s`t        j0                  d|j                   d   |d   |f|j2                  |j4                  ¬
«      }| j6                  r6| j8                  r*d|_        n"| j=                  |d   ||j2                  |
¬«      }|�#|dd…dd…dd…d|j                   d	   …f   }||z   }|}||z  }t>        j@                  jC                  |jE                  «       d¬«      jG                  |«      }t>        j@                  jI                  || jH                  | j8                  ¬«      }t        j,                  ||«      }|j                  dd«      jK                  «       } |jL                  g |¢d‘­Ž }| jO                  |«      }||f}|r||fz   }|S )z€
        Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states).
        Nr+   r   r   r*   FTr   éþÿÿÿ)r’   r2   )r’   r˜   ©Údim)ÚpÚtraining)(Úshaperp   Úget_seq_lengthri   rQ   r   rR   Úclonerv   ÚviewÚ	transposer   Ú
is_updatedÚgetÚcross_attention_cacheÚself_attention_cacheÚlayersÚkeysrœ   rw   rx   ÚupdateÚmatmulrl   Úzerosr’   r2   r|   r£   Úrequires_gradr�   r   Ú
functionalÚsoftmaxr„   Útype_asrK   Ú
contiguousÚreshapery   )r#   r5   ÚmaskÚkey_value_statesÚposition_biasÚpast_key_valuesÚoutput_attentionsÚkwargsÚinput_shapeÚhidden_shaper˜   Úis_cross_attentionÚquery_statesr©   Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚkv_shapeÚscoresr—   Úcausal_maskÚposition_bias_maskedÚattn_weightsÚattn_outputÚoutputss                             r'   r7   zPop2PianoAttention.forwardï   s·  € ð $×)Ñ)¨#¨2Ð.ˆØB˜ÐB bÐB¨$×*AÑ*AÑBˆØM\ÐMh˜?×9Ñ9¸$¿.¹.ÔIÐnoÐä7AÐBRÔTY×T`ÑT`Ô7aÐ+×1Ñ1Ô3ÐgwÐð .°TÐ9Ðà—v‘v˜mÓ,×1Ñ1°,Ó?×IÑIÈ!ÈQÓOˆð ˆ
Ü�oÔ':Ô;Ø(×3Ñ3×7Ñ7¸¿¹ÓGˆJÙ!à'6×'LÑ'LÑ$à'6×'KÑ'KÑ$à#2Ð á-?Ñ)À]ˆÙ /Ð"=Á*à-×4Ñ4°T·^±^ÑD×IÑIˆJØ/×6Ñ6°t·~±~ÑF×MÑM‰LàP˜×-Ñ-¨c¨rÐ2ÐP°BÐP¸×8OÑ8OÑPˆHØŸ™ Ó/×4Ñ4°XÓ>×HÑHÈÈAÓNˆJØŸ6™6 .Ó1×6Ñ6°xÓ@×JÑJÈ1ÈaÓPˆLàÐ*Ø+?×+FÑ+FÀzÐS_Ðae×aoÑaoÓ+pÑ(�
˜Lá%¬*°_ÔFYÔ*ZØAE�O×.Ñ.¨t¯~©~Ñ>ô —‘˜l¨J×,@Ñ,@ÀÀAÓ,FÓGˆàÐ Ø#×)Ñ)¨"Ñ-ˆJØ×3Ò3Ü %§¡Ø˜×*Ñ*¨1Ñ-¨{¸1©~¸zÐJÐSY×S`ÑS`Ðhn×htÑhtô!�ð ×.Ò.°4·=²=Ø26�MÕ/à $× 1Ñ 1Ø ‘N J°v·}±}ÐWgð !2ó !�ð ÐØ"¢1¢aªÐ,B¨j×.>Ñ.>¸rÑ.BÐ,BÐ#BÑC�Ø -°Ñ ;�à,ÐØÐ&Ñ&ˆô —}‘}×,Ñ,¨V¯\©\«^ÀÐ,ÓD×LÑLÈVÓTˆÜ—}‘}×,Ñ,¨\¸T¿\¹\ÐTX×TaÑTaÐ,Óbˆä—l‘l <°Ó>ˆà!×+Ñ+¨A¨qÓ1×<Ñ<Ó>ˆØ)�k×)Ñ)Ð;¨;Ð;¸Ò;ˆØ—f‘f˜[Ó)ˆà Ð.ˆáØ  Ñ/ˆGØˆr(   ©FN)Té    é€   )Nr   )NNNNF)r8   r9   r:   r   Úintr   Ústaticmethodr�   r�   r7   r;   r<   s   @r'   rh   rh   ‹   sX   ø„ ð %*Ø $ñ	 ,àð ,ð ˜‘:õ	 ,ðD ò- ó ð- ó^ð( ØØØØ÷[r(   rh   c                   ó>   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 	 dd„Zˆ xZS )ÚPop2PianoLayerSelfAttentionNri   c                 óÜ   •— t         ‰| �  «        t        |||¬«      | _        t	        |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y )N©rl   ri   r`   )r   r   rh   ÚSelfAttentionr   rE   rc   rd   r   rI   rJ   rK   r}   s       €r'   r   z$Pop2PianoLayerSelfAttention.__init__O  sT   ø€ Ü‰ÑÔÜ/ØÐ0KÐW`ô
ˆÔô -¨V¯^©^À×AZÑAZÔ[ˆŒÜ—z‘z &×"5Ñ"5Ó6ˆ�r(   c                 ó–   — | j                  |«      }| j                  ||||||¬«      }	|| j                  |	d   «      z   }|f|	dd  z   }
|
S )N)r¸   rº   r»   Ú	use_cacher¼   r   r   )rd   rÖ   rK   )r#   r5   Úattention_maskrº   r»   rØ   r¼   r½   Únormed_hidden_statesÚattention_outputrÌ   s              r'   r7   z#Pop2PianoLayerSelfAttention.forwardW  sp   € ð  $Ÿ™¨}Ó=ÐØ×-Ñ-Ø ØØ'Ø+ØØ/ð .ó 
Ðð &¨¯©Ð5EÀaÑ5HÓ(IÑIˆØ Ð"Ð%5°a°bÐ%9Ñ9ˆØˆr(   rÍ   )NNNFF©r8   r9   r:   rÐ   r   r7   r;   r<   s   @r'   rÓ   rÓ   N  s*   ø„ ñ7ÈSÐSWÉZõ 7ð ØØØØ÷r(   rÓ   c                   ó<   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 dd„Zˆ xZS )ÚPop2PianoLayerCrossAttentionNri   c                 óÜ   •— t         ‰| �  «        t        |d|¬«      | _        t	        |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y )NFrÕ   r`   )r   r   rh   ÚEncDecAttentionr   rE   rc   rd   r   rI   rJ   rK   )r#   r?   ri   r&   s      €r'   r   z%Pop2PianoLayerCrossAttention.__init__q  sO   ø€ Ü‰ÑÔÜ1°&ÐV[ÐgpÔqˆÔÜ,¨V¯^©^À×AZÑAZÔ[ˆŒÜ—z‘z &×"5Ñ"5Ó6ˆ�r(   c                 ó–   — | j                  |«      }| j                  ||||||¬«      }	|| j                  |	d   «      z   }
|
f|	dd  z   }|S )N)r¸   r¹   rº   r»   r¼   r   r   )rd   rà   rK   )r#   r5   r¹   rÙ   rº   r»   r¼   r½   rÚ   rÛ   Úlayer_outputrÌ   s               r'   r7   z$Pop2PianoLayerCrossAttention.forwardw  so   € ð  $Ÿ™¨}Ó=ÐØ×/Ñ/Ø ØØ-Ø'Ø+Ø/ð 0ó 
Ðð % t§|¡|Ð4DÀQÑ4GÓ'HÑHˆØ�/Ð$4°Q°RÐ$8Ñ8ˆØˆr(   rP   )NNNFrÜ   r<   s   @r'   rÞ   rÞ   p  s&   ø„ ñ7¨#°©*õ 7ð ØØØ÷r(   rÞ   c                   óF   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 	 	 	 	 	 dd„Zˆ xZS )ÚPop2PianoBlockNri   c                 óp  •— t         ‰| �  «        |j                  | _        t        j                  «       | _        | j
                  j                  t        |||¬«      «       | j                  r&| j
                  j                  t        ||¬«      «       | j
                  j                  t        |«      «       y )NrÕ   )ri   )
r   r   rk   r   Ú
ModuleListÚlayerÚappendrÓ   rÞ   r^   r}   s       €r'   r   zPop2PianoBlock.__init__‘  s†   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒÜ—]‘]“_ˆŒ
Ø�
‰
×ÑÜ'ØÐ4OÐ[dôô	
ð
 �?Š?Ø�J‰J×ÑÔ:¸6ÈYÔWÔXà�
‰
×ÑÔ*¨6Ó2Õ3r(   c                 óx  —  | j                   d   ||||||	¬«      }|d   }|dd  }|j                  t        j                  k(  r¢t        j                  t        j
                  |«      j                  «       t        j                  |j                  «      j                  dz
  t        j                  |j                  «      j                  «      }t        j                  || |¬«      }| j                  xr |d u}|rç | j                   d   ||||||	¬«      }|d   }|j                  t        j                  k(  r¢t        j                  t        j
                  |«      j                  «       t        j                  |j                  «      j                  dz
  t        j                  |j                  «      j                  «      }t        j                  || |¬«      }||dd  z   } | j                   d   |«      }|j                  t        j                  k(  r¢t        j                  t        j
                  |«      j                  «       t        j                  |j                  «      j                  dz
  t        j                  |j                  «      j                  «      }t        j                  || |¬«      }|f}||z   S )Nr   )rÙ   rº   r»   rØ   r¼   r   iè  )r�   Úmax)r¹   rÙ   rº   r»   r¼   r+   )rç   r2   r   r3   r‡   ÚisinfÚanyÚfinforê   Úclamprk   )r#   r5   rÙ   rº   Úencoder_hidden_statesÚencoder_attention_maskÚencoder_decoder_position_biasr»   rØ   r¼   Úreturn_dictr½   Úself_attention_outputsÚattention_outputsÚclamp_valueÚdo_cross_attentionÚcross_attention_outputsrÌ   s                     r'   r7   zPop2PianoBlock.forwardŸ  sY  € ð "/ §¡¨A¡ØØ)Ø'Ø+ØØ/ô"
Ðð /¨qÑ1ˆØ2°1°2Ð6Ðð ×Ñ¤%§-¡-Ò/ÜŸ+™+Ü—‘˜MÓ*×.Ñ.Ó0Ü—‘˜M×/Ñ/Ó0×4Ñ4°tÑ;Ü—‘˜M×/Ñ/Ó0×4Ñ4óˆKô
 "ŸK™K¨¸K¸<È[ÔYˆMà!Ÿ_™_ÒRÐ1FÈdÐ1RÐÙØ&3 d§j¡j°¡mØØ!6Ø5Ø;Ø /Ø"3ô'Ð#ð 4°AÑ6ˆMð ×"Ñ"¤e§m¡mÒ3Ü#Ÿk™kÜ—K‘K Ó.×2Ñ2Ó4Ü—K‘K × 3Ñ 3Ó4×8Ñ8¸4Ñ?Ü—K‘K × 3Ñ 3Ó4×8Ñ8ó�ô
 !&§¡¨MÀ¸|ÐQ\Ô ]�ð !2Ð4KÈAÈBÐ4OÑ OÐð '˜Ÿ
™
 2™ }Ó5ˆð ×Ñ¤%§-¡-Ò/ÜŸ+™+Ü—‘˜MÓ*×.Ñ.Ó0Ü—‘˜M×/Ñ/Ó0×4Ñ4°tÑ;Ü—‘˜M×/Ñ/Ó0×4Ñ4óˆKô
 "ŸK™K¨¸K¸<È[ÔYˆMà Ð"ˆð Ð'Ñ'ð	
r(   rÍ   )	NNNNNNFFTrÜ   r<   s   @r'   rä   rä   �  s7   ø„ ñ4ÈSÐSWÉZõ 4ð" ØØ"Ø#Ø&*ØØØØ÷J
r(   rä   c                   ót   ‡ — e Zd ZU eed<   dZdZdZdZdgZ	dgZ
 ej                  «       ˆ fd„«       Zd	„ Zˆ xZS )
ÚPop2PianoPreTrainedModelr?   Útransformer)ÚaudioTFrä   rH   c                 óP  •— t         ‰| �  |«       | j                  j                  }t	        |t
        «      r$t        j                  |j                  |dz  «       yt	        |t        «      r0t        j                  |j                  j                  d|dz  ¬«       yt	        |t        «      rlt        j                  |j                  j                  d|dz  ¬«       t        |d«      r0t        j                  |j                  j                  d|dz  ¬«       yyt	        |t         «      �r9t        j                  |j"                  j                  d|| j                  j$                  dz  z  ¬«       t        |j"                  d«      r?|j"                  j&                  �)t        j(                  |j"                  j&                  «       t        j                  |j*                  j                  d|| j                  j,                  dz  z  ¬«       t        |j*                  d«      rA|j*                  j&                  �*t        j(                  |j*                  j&                  «       yyyt	        |t.        «      �rÔt        j                  |j0                  j                  d|| j                  j$                  dz  z  ¬«       t        |j0                  d«      r?|j0                  j&                  �)t        j(                  |j0                  j&                  «       t        j                  |j2                  j                  d|| j                  j$                  dz  z  ¬«       t        |j2                  d«      r?|j2                  j&                  �)t        j(                  |j2                  j&                  «       t        j                  |j*                  j                  d|| j                  j,                  dz  z  ¬«       t        |j*                  d«      rA|j*                  j&                  �*t        j(                  |j*                  j&                  «       yyyt	        |t4        «      �rP| j                  j$                  }| j                  j6                  }| j                  j8                  }t        j                  |j:                  j                  d|||z  dz  z  ¬«       t        j                  |j<                  j                  d||dz  z  ¬«       t        j                  |j>                  j                  d||dz  z  ¬«       t        j                  |j@                  j                  d|||z  dz  z  ¬«       |jB                  r3t        j                  |jD                  j                  d||dz  z  ¬«       yyy)zInitialize the weightsç      ð?ç        )r0   ÚstdÚlm_headç      à¿rC   N)#r   Ú_init_weightsr?   Úinitializer_factorrQ   r   ÚinitÚ	constant_r!   ÚPop2PianoConcatEmbeddingToMelÚnormal_Ú	embeddingÚ!Pop2PianoForConditionalGenerationÚsharedÚhasattrr   r>   rG   rE   rC   Úzeros_rH   rF   rV   rX   rY   rh   ro   rq   rv   rw   rx   ry   rl   r{   )r#   ÚmoduleÚfactorrE   rp   rr   r&   s         €r'   r  z&Pop2PianoPreTrainedModel._init_weights÷  sÒ  ø€ ô 	‰Ñ˜fÔ%Ø—‘×/Ñ/ˆÜ�fÔ0Ô1Ü�N‰N˜6Ÿ=™=¨&°3©,Õ7Ü˜Ô =Ô>Ü�L‰L˜×)Ñ)×0Ñ0°sÀÈÁÖMÜ˜Ô AÔBÜ�L‰L˜Ÿ™×-Ñ-°C¸VÀc¹\ÕJÜ�v˜yÔ)Ü—‘˜VŸ^™^×2Ñ2¸À&È3Á,ÖOð *ä˜Ô 6Õ7Ü�L‰L˜Ÿ™×)Ñ)°¸ÀDÇKÁK×DWÑDWÐ\`ÑC`Ñ9aÕbÜ�v—y‘y &Ô)¨f¯i©i¯n©nÐ.HÜ—‘˜FŸI™IŸN™NÔ+Ü�L‰L˜Ÿ™×)Ñ)°¸ÀDÇKÁK×DTÑDTÐY]ÑC]Ñ9^Õ_Ü�v—y‘y &Ô)¨f¯i©i¯n©nÐ.HÜ—‘˜FŸI™IŸN™NÕ+ð /IÐ)ä˜Ô ;Õ<Ü�L‰L˜Ÿ™×+Ñ+°#¸6ÀdÇkÁk×FYÑFYÐ^bÑEbÑ;cÕdÜ�v—{‘{ FÔ+°·±×0@Ñ0@Ð0LÜ—‘˜FŸK™K×,Ñ,Ô-Ü�L‰L˜Ÿ™×+Ñ+°#¸6ÀdÇkÁk×FYÑFYÐ^bÑEbÑ;cÕdÜ�v—{‘{ FÔ+°·±×0@Ñ0@Ð0LÜ—‘˜FŸK™K×,Ñ,Ô-Ü�L‰L˜Ÿ™×)Ñ)°¸ÀDÇKÁK×DTÑDTÐY]ÑC]Ñ9^Õ_Ü�v—y‘y &Ô)¨f¯i©i¯n©nÐ.HÜ—‘˜FŸI™IŸN™NÕ+ð /IÐ)ä˜Ô 2Õ3Ø—k‘k×)Ñ)ˆGØ!%§¡×!1Ñ!1ÐØ—k‘k×+Ñ+ˆGÜ�L‰L˜Ÿ™Ÿ™¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÕjÜ�L‰L˜Ÿ™Ÿ™¨s¸À'È4Á-Ñ8PÕQÜ�L‰L˜Ÿ™Ÿ™¨s¸À'È4Á-Ñ8PÕQÜ�L‰L˜Ÿ™Ÿ™¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÕjØ×1Ò1Ü—‘˜V×;Ñ;×BÑBÈÐRXÐ]dÐimÑ\mÑRnÖoð 2ð 4r(   c                 ó8  — | j                   j                  }| j                   j                  }|€t        d«      ‚|j	                  |j
                  «      }|dd d…f   j                  «       |ddd …f<   ||d<   |€t        d«      ‚|j                  |dk(  |«       |S )Nzoself.model.config.decoder_start_token_id has to be defined. In Pop2Piano it is usually set to the pad_token_id..r+   r   ).r   z1self.model.config.pad_token_id has to be defined.éœÿÿÿ)r?   Údecoder_start_token_idÚpad_token_idÚ
ValueErrorÚ	new_zerosr¤   r¦   Úmasked_fill_)r#   Ú	input_idsr  r  Úshifted_input_idss        r'   Ú_shift_rightz%Pop2PianoPreTrainedModel._shift_right   s±   € Ø!%§¡×!CÑ!CÐØ—{‘{×/Ñ/ˆà!Ð)Üð Bóð ð &×/Ñ/°	·±Ó@ÐØ%.¨s°C°R°C¨xÑ%8×%>Ñ%>Ó%@Ð˜#˜q™r˜'Ñ"Ø$:Ð˜&Ñ!àÐÜÐPÓQÐQà×&Ñ&Ð'8¸DÑ'@À,ÔOà Ð r(   )r8   r9   r:   r   Ú__annotations__Úbase_model_prefixÚoutput_modalitiesÚsupports_gradient_checkpointingÚ_can_compile_fullgraphÚ_no_split_modulesÚ_keep_in_fp32_modulesr   Úno_gradr  r  r;   r<   s   @r'   rù   rù   ì  sT   ø… àÓØ%ÐØ"ÐØ&*Ð#à"ÐØ)Ð*ÐØ!˜FÐà€U‡]�]ƒ_ó&pó ð&pöP!r(   rù   c                   ó@   ‡ — e Zd Zˆ fd„Zd„ Z	 	 	 	 	 	 	 	 	 	 dd„Zˆ xZS )ÚPop2PianoStackc                 ó  •— t         ‰| �  |«       t        j                  |j                  |j
                  «      | _        |j                  | _        t        j                  t        |j                  «      D �cg c]  }t        |t        |dk(  «      |¬«      ‘Œ c}«      | _        t        |j
                  |j                  ¬«      | _        t        j"                  |j$                  «      | _        | j)                  «        d| _        y c c}w )Nr   rÕ   r`   F)r   r   r   rz   Ú
vocab_sizerE   Úembed_tokensrk   ræ   ÚrangeÚ
num_layersrä   ÚboolÚblockr   rc   Úfinal_layer_normrI   rJ   rK   Ú	post_initr|   )r#   r?   Úir&   s      €r'   r   zPop2PianoStack.__init__7  sÉ   ø€ Ü‰Ñ˜Ô äŸL™L¨×):Ñ):¸F¿N¹NÓKˆÔØ ×+Ñ+ˆŒä—]‘]ô ˜v×0Ñ0Ó1öàô ˜vÄ4ÈÈQÉÃ<Ð[\Ö]òó
ˆŒ
ô !3°6·>±>Àv×G`ÑG`Ô aˆÔÜ—z‘z &×"5Ñ"5Ó6ˆŒð 	�‰ÔØ&+ˆÕ#ùòs   Á7!Dc                 ó   — || _         y rP   )r%  ©r#   Únew_embeddingss     r'   Úset_input_embeddingsz#Pop2PianoStack.set_input_embeddingsK  s
   € Ø*ˆÕr(   c                 ó®  — |�|n| j                   j                  }|�|n| j                   j                  }|	�|	n| j                   j                  }	|
�|
n| j                   j                  }
|�$|�"| j
                  rdnd}t        d|› d|› d�«      ‚|�&|j                  «       }|j                  d|d   «      }n8|�|j                  «       d d }n"| j
                  rdnd}t        d|› d|› d	�«      ‚| j                  r%| j                  r|rt        j                  d
«       d}|€(| j                  €t        d«      ‚| j                  |«      }|\  }}|du r| j
                  st        d| › d�«      ‚| j
                  rf|rr|€p| j                   j                  r5t        t!        | j                   ¬«      t!        | j                   ¬«      «      }n%t!        | j                   ¬«      }n| j
                  sd }|�|j#                  «       nd}|€1t%        «       s'||z   }t'        j(                  |||j*                  ¬«      }| j                   j
                  rt-        | j                   |||¬«      }nX|d d …d d d d …f   }|j/                  |j0                  ¬«      }d|z
  t'        j2                  |j0                  «      j4                  z  }|�t7        | j                   |||¬«      }|	rdnd }|rdnd }|r| j
                  rdnd }d }d }| j9                  |«      }t;        | j<                  «      D ]a  \  }}|	r||fz   } ||||||||||¬«	      }|d   }|d   }| j
                  r|�	||rdnd   }|sŒC||d   fz   }| j
                  sŒY||d   fz   }Œc | j?                  |«      }| j9                  |«      }|	r||fz   }|
stA        d„ |||||fD «       «      S tC        |||||¬«      S )NÚdecoder_Ú zYou cannot specify both zinput_ids and zinputs_embeds at the same timer+   zYou have to specify either zinput_ids or Úinputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fz<You have to initialize the model with valid token embeddingsTz)`use_cache` can only be set to `True` if z is used as a decoder)r?   r   ©r’   )r?   r4  rÙ   r»   )r2   rý   )r?   r4  rÙ   rï   © )r»   rØ   r¼   r   r   r*   é   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wrP   r6  )Ú.0rx   s     r'   ú	<genexpr>z)Pop2PianoStack.forward.<locals>.<genexpr>Ø  s   è ø€ ò 
àð �=ô ñ
ùs   ‚)Úlast_hidden_stater»   r5   Ú
attentionsÚcross_attentions)"r?   rØ   r¼   Úoutput_hidden_statesrò   rk   r  Úsizer§   r|   r£   rt   ru   r%  Úis_encoder_decoderr   r
   r¥   r   r   r    r’   r   r-   r2   rí   r�   r   rK   Ú	enumerater)  r*  Útupler   )r#   r  rÙ   rï   rð   r4  r»   rØ   r¼   r>  rò   r½   Úerr_msg_prefixr¾   Ú
batch_sizeÚ
seq_lengthÚpast_key_values_lengthÚmask_seq_lengthrÈ   Úall_hidden_statesÚall_attentionsÚall_cross_attentionsrº   rñ   r5   r,  Úlayer_moduleÚlayer_outputss                               r'   r7   zPop2PianoStack.forwardN  sH  € ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø1BÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐ  ]Ð%>Ø+/¯?ª?™ZÀˆNÜØ*¨>Ð*:¸.ÈÐHXÐXvÐwóð ð Ð"Ø#Ÿ.™.Ó*ˆKØ!Ÿ™ r¨;°r©?Ó;‰IØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰Kà+/¯?ª?™ZÀˆNÜÐ:¸>Ð:JÈ-ÐXfÐWgÐgtÐuÓvÐvà×&Ò&¨4¯=ª=ÙÜ×#Ñ#Øpôð "�	àÐ Ø× Ñ Ð(Ü Ð!_Ó`Ð`Ø ×-Ñ-¨iÓ8ˆMà!,Ñˆ
�Jà˜ÑØ—?’?Ü Ð#LÈTÈFÐRgÐ!hÓiÐià�?Š?Ù˜_Ð4Ø—;‘;×1Ò1Ü&9Ü$¨D¯K©KÔ8¼,ÈdÏkÉkÔ:Zó'‘Oô '3¸$¿+¹+Ô&F‘OØ—’ð #ˆOàETÐE` ×!?Ñ!?Ô!AÐfgÐØÐ!Ô*BÔ*Dà4°zÑAˆOÜ"ŸZ™Z¨
°OÈM×L`ÑL`ÔaˆNà�;‰;×!Ò!Ü,Ø—{‘{Ø+Ø-Ø /ô	‰Kð )ª¨D°$ºÐ)9Ñ:ˆKØ%Ÿ.™.¨}×/BÑ/B˜.ÓCˆKØ Ñ,´·±¸M×<OÑ<OÓ0P×0TÑ0TÑTˆKà!Ð-Ü%>Ø—{‘{Ø+Ø5Ø&;ô	&Ð"ñ #7™B¸DÐÙ0™°dˆÙ&7¸D¿OºO™rÐRVÐØˆØ(,Ð%àŸ™ ]Ó3ˆä(¨¯©Ó4ò 	V‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!á(ØØØØ%Ø&Ø-Ø /Ø#Ø"3ô
ˆMð *¨!Ñ,ˆMð
 *¨!Ñ,ˆMØ�ŠÐ#8Ð#DØ0=ÑCT¹aÐZ[Ñ0\Ð-â Ø!/°=ÀÑ3CÐ2EÑ!E�Ø—?“?Ø+?À=ÐQRÑCSÐBUÑ+UÑ(ð9	Vð< ×-Ñ-¨mÓ<ˆØŸ™ ]Ó3ˆñ  Ø 1°]Ð4DÑ DÐáÜñ 
ð "Ø#Ø%Ø"Ø(ðô
ó 
ð 
ô 9Ø+Ø+Ø+Ø%Ø1ô
ð 	
r(   )
NNNNNNNNNN)r8   r9   r:   r   r0  r7   r;   r<   s   @r'   r"  r"  5  s3   ø„ ô,ò(+ð
 ØØ"Ø#ØØØØØ!Ø÷[
r(   r"  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )r  z'Embedding Matrix for `composer` tokens.c                 ó‚   •— t         ‰| �  «        t        j                  |j                  |j
                  ¬«      | _        y )N)Únum_embeddingsÚembedding_dim)r   r   r   rz   Úcomposer_vocab_sizerE   r  rN   s     €r'   r   z&Pop2PianoConcatEmbeddingToMel.__init__ï  s-   ø€ Ü‰ÑÔÜŸ™°V×5OÑ5OÐ_e×_mÑ_mÔnˆ�r(   c                 ó‚   — ||z
  }| j                  |«      j                  d«      }t        j                  ||gd¬«      }|S )Nr   r    )r  r•   r   Úcat)r#   ÚfeatureÚindex_valueÚembedding_offsetÚindex_shiftedÚcomposer_embeddingr4  s          r'   r7   z%Pop2PianoConcatEmbeddingToMel.forwardó  sC   € Ø#Ð&6Ñ6ˆØ!Ÿ^™^¨MÓ:×DÑDÀQÓGÐÜŸ	™	Ð#5°wÐ"?ÀQÔGˆØÐr(   )r8   r9   r:   Ú__doc__r   r7   r;   r<   s   @r'   r  r  ì  s   ø„ Ù1ôoör(   r  zA
    Pop2Piano Model with a `language modeling` head on top.
    )Úcustom_introc            !       ó~  ‡ — e Zd ZdddœZdefˆ fd„Zd„ Zd„ Z	 ddej                  d	e
d
edej                  dz  fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  deeej"                        dz  dedz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  dedz  deej                     ez  fd„«       Z ej,                  «       	 	 	 dˆ fd„	«       Zdej"                  fd„Zˆ xZS ) r	  zshared.weight)zencoder.embed_tokens.weightzdecoder.embed_tokens.weightr?   c                 ó4  •— t         ‰| �  |«       || _        |j                  | _        t        j                  |j                  |j                  «      | _        t        |«      | _
        t        j                  |«      }d|_        d|_        t        |«      | _        t        j                  |«      }d|_        |j"                  |_        t        |«      | _        t        j(                  |j                  |j                  d¬«      | _        | j-                  «        y )NFTrB   )r   r   r?   rE   Ú	model_dimr   rz   r$  r
  r  Úmel_conditionerÚcopyÚdeepcopyrk   rØ   r"  ÚencoderÚnum_decoder_layersr'  ÚdecoderrD   r   r+  )r#   r?   Úencoder_configÚdecoder_configr&   s       €r'   r   z*Pop2PianoForConditionalGeneration.__init__  sÏ   ø€ Ü‰Ñ˜Ô ØˆŒØŸ™ˆŒä—l‘l 6×#4Ñ#4°f·n±nÓEˆŒä<¸VÓDˆÔäŸ™ vÓ.ˆØ$)ˆÔ!Ø#(ˆÔ ä% nÓ5ˆŒäŸ™ vÓ.ˆØ$(ˆÔ!Ø$*×$=Ñ$=ˆÔ!Ü% nÓ5ˆŒä—y‘y §¡°×1BÑ1BÈÔOˆŒð 	�‰Õr(   c                 ó   — | j                   S rP   )r
  )r#   s    r'   Úget_input_embeddingsz6Pop2PianoForConditionalGeneration.get_input_embeddings  s   € Ø�{‰{Ðr(   c                 ó~   — || _         | j                  j                  |«       | j                  j                  |«       y rP   )r
  ra  r0  rc  r.  s     r'   r0  z6Pop2PianoForConditionalGeneration.set_input_embeddings!  s-   € Ø$ˆŒØ�‰×)Ñ)¨.Ô9Ø�‰×)Ñ)¨.Õ9r(   NÚinput_featuresÚcomposerÚgeneration_configrÙ   c                 óþ  — |j                   }||vr(t        dt        |j                  «       «      › d|› �«      ‚||   }t	        j
                  || j                  ¬«      }|j                  |j                  d   «      }t        |j                  «       «      }| j                  |||¬«      }|�Od||dd…df   j                  «        <   t	        j                  |dd…df   j                  dd	«      |gd	¬
«      }||fS |dfS )aã  
        This method is used to concatenate mel conditioner tokens at the front of the input_features in order to
        control the type of MIDI token generated by the model.

        Args:
            input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
                input features extracted from the feature extractor.
            composer (`str`):
                composer token which determines the type of MIDI tokens to be generated.
            generation_config (`~generation.GenerationConfig`):
                The generation is used to get the composer-feature_token pair.
            attention_mask (``, *optional*):
                For batched generation `input_features` are padded to have the same shape across all examples.
                `attention_mask` helps to determine which areas were padded and which were not.
                - 1 for tokens that are **not padded**,
                - 0 for tokens that are **padded**.
        zPlease choose a composer from z. Composer received - r5  r   )rT  rU  rV  Nrþ   r+   r   )Úaxis)Úcomposer_to_feature_tokenr  Úlistr®   r   Útensorr’   Úrepeatr¤   r�   rœ   r^  r(  Úconcatenater§   )r#   ri  rj  rk  rÙ   rn  Úcomposer_valuerV  s           r'   Úget_mel_conditioner_outputsz=Pop2PianoForConditionalGeneration.get_mel_conditioner_outputs&  s%  € ð0 %6×$OÑ$OÐ!ØÐ4Ñ4ÜØ0´Ð6O×6TÑ6TÓ6VÓ1WÐ0XÐXnÐowÐnxÐyóð ð 3°8Ñ<ˆÜŸ™ n¸T¿[¹[ÔIˆØ'×.Ñ.¨~×/CÑ/CÀAÑ/FÓGˆäÐ8×?Ñ?ÓAÓBÐà×-Ñ-Ø"Ø&Ø-ð .ó 
ˆð
 Ð%Ø;>ˆN˜Nª1¨a¨4Ñ0×5Ñ5Ó7Ð7Ñ8ô #×.Ñ.°ºqÀ!¸tÑ0D×0IÑ0IÈ"ÈaÓ0PÐR`Ð/aÐhiÔjˆNØ! >Ð1Ð1à˜tÐ#Ð#r(   r  Údecoder_input_idsÚdecoder_attention_maskÚencoder_outputsr»   r4  Údecoder_inputs_embedsÚlabelsrØ   r¼   r>  rò   Úreturnc                 ó¶  — |�|n| j                   j                  }|�|n| j                   j                  }|�|�t        d«      ‚|�|€|}|€| j	                  ||||||¬«      }nI|rGt        |t        «      s7t        |d   t        |«      dkD  r|d   ndt        |«      dkD  r|d   nd¬«      }|d   }|
�|€|	€| j                  |
«      }| j                  |||	|||||||¬«
      }|d   }| j                   j                  r|| j                  d	z  z  }| j                  |«      }d}|
�Ct        d
¬«      } ||j                  d|j                  d«      «      |
j                  d«      «      }|s|f|dd z   |z   }|�|f|z   S |S t!        |||j"                  |j$                  |j&                  |j(                  |j*                  |j$                  |j&                  ¬«	      S )aq  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Pop2Piano is a model with relative position embeddings
            so you should be able to pad the inputs on both the right and the left. Indices can be obtained using
            [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for detail.
            [What are input IDs?](../glossary#input-ids) To know more on how to prepare `input_ids` for pretraining
            take a look a [Pop2Piano Training](./Pop2Piano#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using
            [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.
            [What are decoder input IDs?](../glossary#decoder-input-ids) Pop2Piano uses the `pad_token_id` as the
            starting token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last
            `decoder_input_ids` have to be input (see `past_key_values`). To know more on how to prepare
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
            config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
            labels in `[0, ..., config.vocab_size]`
        NzSBoth `inputs_embeds` and `input_features` received! Please provide only one of them)r  rÙ   r4  r¼   r>  rò   r   r   r*   )r;  r5   r<  )
r  rÙ   r4  r»   rï   rð   rØ   r¼   r>  rò   r  r  )Úignore_indexr+   )	ÚlossÚlogitsr»   Údecoder_hidden_statesÚdecoder_attentionsr=  Úencoder_last_hidden_staterï   Úencoder_attentions)r?   rØ   rò   r  ra  rQ   r   Úlenr  rc  Útie_word_embeddingsr]  r   r   r§   r?  r   r»   r5   r<  r=  r;  )r#   r  rÙ   ru  rv  rw  r»   r4  ri  rx  ry  rØ   r¼   r>  rò   r½   r5   Údecoder_outputsÚsequence_outputÚ	lm_logitsr}  Úloss_fctÚoutputs                          r'   r7   z)Pop2PianoForConditionalGeneration.forwardW  s0  € ðP "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐ$¨Ð)CÜÐrÓsÐsØÐ'¨MÐ,AØ*ˆMð Ð"à"Ÿl™lØ#Ø-Ø+Ø"3Ø%9Ø'ð +ó ‰Oñ ¤¨O¼_Ô!MÜ-Ø"1°!Ñ"4Ü47¸Ó4HÈ1Ò4L˜o¨aÒ0ÐRVÜ14°_Ó1EÈÒ1I˜?¨1Ò-ÈtôˆOð (¨Ñ*ˆàÐÐ"3Ð";Ð@UÐ@]à $× 1Ñ 1°&Ó 9Ðð Ÿ,™,Ø'Ø1Ø/Ø+Ø"/Ø#1ØØ/Ø!5Ø#ð 'ó 
ˆð *¨!Ñ,ˆà�;‰;×*Ò*Ø-°·±ÀÑ1EÑFˆOà—L‘L Ó1ˆ	àˆØÐÜ'°TÔ:ˆHÙ˜IŸN™N¨2¨y¯~©~¸bÓ/AÓBÀFÇKÁKÐPRÃOÓTˆDáØ�\ O°A°BÐ$7Ñ7¸/ÑIˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEäØØØ+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô

ð 
	
r(   c                 ó”  •— |€| j                   } |j                  d	i |¤Ž t        |d«      st        d«      ‚t	        |j
                  «      | j                  j                  k7  r9t        d| j                  j                  › dt	        |j
                  «      › d�«      ‚| j                  ||||¬«      \  }}t        ‰| �(  d	d|||dœ|¤ŽS )
aŽ  
        Generates token ids for midi outputs.

        <Tip warning={true}>

        Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the
        model's default generation configuration. You can override any `generation_config` by passing the corresponding
        parameters to generate(), e.g. `.generate(inputs, num_beams=4, do_sample=True)`. For an overview of generation
        strategies and code examples, check out the [following guide](./generation_strategies).

        </Tip>

        Parameters:
            input_features (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
                This is the featurized version of audio generated by `Pop2PianoFeatureExtractor`.
            attention_mask:
                For batched generation `input_features` are padded to have the same shape across all examples.
                `attention_mask` helps to determine which areas were padded and which were not.
                - 1 for tokens that are **not padded**,
                - 0 for tokens that are **padded**.
            composer (`str`, *optional*, defaults to `"composer1"`):
                This value is passed to `Pop2PianoConcatEmbeddingToMel` to generate different embeddings for each
                `"composer"`. Please make sure that the composer value is present in `composer_to_feature_token` in
                `generation_config`. For an example please see
                https://huggingface.co/sweetcocoa/pop2piano/blob/main/generation_config.json .
            generation_config (`~generation.GenerationConfig`, *optional*):
                The generation configuration to be used as base parametrization for the generation call. `**kwargs`
                passed to generate matching the attributes of `generation_config` will override them. If
                `generation_config` is not provided, the default will be used, which had the following loading
                priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model
                configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s
                default values, whose documentation should be checked to parameterize generation.
            kwargs:
                Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be
                forwarded to the `forward` function of the model. If the model is an encoder-decoder model, encoder
                specific kwargs should not be prefixed and decoder specific kwargs should be prefixed with *decoder_*.
        Return:
            [`~utils.ModelOutput`] or `torch.LongTensor`: A [`~utils.ModelOutput`] (if `return_dict_in_generate=True`
            or when `config.return_dict_in_generate=True`) or a `torch.FloatTensor`.
                Since Pop2Piano is an encoder-decoder model (`model.config.is_encoder_decoder=True`), the possible
                [`~utils.ModelOutput`] types are:
                    - [`~generation.GenerateEncoderDecoderOutput`],
                    - [`~generation.GenerateBeamEncoderDecoderOutput`]
        Nrn  z¢`composer_to_feature_token` was not found! Please refer to https://huggingface.co/sweetcocoa/pop2piano/blob/main/generation_config.jsonand parse a dict like that.ztconfig.composer_vocab_size must be same as the number of keys in generation_config.composer_to_feature_token! Found z vs ú.)ri  rÙ   rj  rk  )Úinputsr4  rÙ   rk  r6  )rk  r¯   r  r  rƒ  rn  r?   rQ  rt  r   Úgenerate)r#   ri  rÙ   rj  rk  r½   r&   s         €r'   r�  z*Pop2PianoForConditionalGeneration.generateÉ  s  ø€ ðl Ð$Ø $× 6Ñ 6ÐØ Ð× Ñ Ñ* 6Ò*ô Ð(Ð*EÔFÜð.óð ô Ð ×:Ñ:Ó;¸t¿{¹{×?^Ñ?^Ò^ÜðàŸ™×8Ñ8Ð9¸¼cÐBS×BmÑBmÓ>nÐ=oÐopðróð ð *.×)IÑ)IØ)Ø)ØØ/ð	 *Jó *
Ñ&ˆ˜ô ‰wÑð 
ØØ(Ø)Ø/ñ	
ð
 ñ
ð 	
r(   c                 ó$   — | j                  |«      S rP   )r  )r#   ry  s     r'   Ú%prepare_decoder_input_ids_from_labelszGPop2PianoForConditionalGeneration.prepare_decoder_input_ids_from_labels#  s   € Ø× Ñ  Ó(Ð(r(   rP   )NNNNNNNNNNNNNN)NÚ	composer1N)r8   r9   r:   Ú_tied_weights_keysr   r   rg  r0  r   ÚFloatTensorÚstrr   rt  r   Ú
LongTensorÚ
BoolTensorrB  rR   r	   r(  r   r7   r   r�  r�  r;   r<   s   @r'   r	  r	  ú  s  ø„ ð (7Ø'6ñÐð
˜õ ò2ò:ð 48ñ/$à×)Ñ)ð/$ð ð/$ð ,ð	/$ð
 ×)Ñ)¨DÑ0ó/$ðb ð .2Ø37Ø59Ø:>Ø=AØ(,Ø26Ø37Ø:>Ø*.Ø!%Ø)-Ø,0Ø#'ño
à×#Ñ# dÑ*ðo
ð ×)Ñ)¨DÑ0ðo
ð !×+Ñ+¨dÑ2ð	o
ð
 !&× 0Ñ 0°4Ñ 7ðo
ð ˜u U§\¡\Ñ2Ñ3°dÑ:ðo
ð  ™ðo
ð ×(Ñ(¨4Ñ/ðo
ð ×)Ñ)¨DÑ0ðo
ð  %×0Ñ0°4Ñ7ðo
ð × Ñ  4Ñ'ðo
ð ˜$‘;ðo
ð   $™;ðo
ð # T™kðo
ð ˜D‘[ðo
ð" 
ˆu× Ñ Ñ	! OÑ	3ò#o
ó ðo
ðb €U‡]�]ƒ_ð ØØôW
ó ðW
ðr)¸E¿L¹L÷ )r(   r	  )6rY  r_  r…   r   r   Útorch.nnr   Útransformers.generationr   r3  r   r  Úactivationsr   Úcache_utilsr	   r
   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_utilsr   Úutilsr   r   r   Úconfiguration_pop2pianor   Ú
get_loggerr8   rt   ÚModuler   r>   rV   r^   rh   rÓ   rÞ   rä   rù   r"  r  r	  Ú__all__r6  r(   r'   ú<module>r¤     sT  ðñ ã Û ã Ý Ý %å 4å &Ý !ß CÑ CÝ )ß JÝ 9ß kÑ kÝ -ß FÑ FÝ 4ð 
ˆ×	Ñ	˜HÓ	%€ô+˜Ÿ™ô +ô4˜RŸY™Yô ô. "§)¡)ô ô<�r—y‘yô ô&˜Ÿ™ô ôF "§)¡)ô ôD 2§9¡9ô ô@Y
Ð/ô Y
ðx ôE!˜ó E!ó ðE!ôPt
Ð-ô t
ôn B§I¡Iô ñ ðôô
e)Ð(@À/ó e)óð
e)ðP	 /Ð0JÐ
K�r(   