Ë
    ÜÓ^jeS ã                   óÊ  — d Z ddlZddlZddlmZ ddlZddlmZmZ ddlm	Z	 ddl
mZ ddlmZmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZmZ ddlmZ  ej>                  e «      Z!d?d„Z"d„ Z#d?d„Z$d„ Z% ed¬«      e G d„ de«      «       «       Z& ed¬«      e G d„ de«      «       «       Z' ed¬«      e G d„ de«      «       «       Z( ed¬«      e G d„ de«      «       «       Z)e G d „ d!e«      «       Z* G d"„ d#ejV                  «      Z, G d$„ d%ejZ                  «      Z. G d&„ d'ejZ                  «      Z/ G d(„ d)ejZ                  «      Z0 G d*„ d+e«      Z1 G d,„ d-e«      Z2 ed.¬«       G d/„ d0e*«      «       Z3 ed1¬«       G d2„ d3e*«      «       Z4e G d4„ d5e*«      «       Z5 ed6¬«       G d7„ d8e*e«      «       Z6 ed9¬«       G d:„ d;e*e«      «       Z7 G d<„ d=e*«      Z8g d>¢Z9y)@zRPyTorch ProphetNet model, ported from ProphetNet repo(fairsequery_states version).é    N)Ú	dataclass)ÚTensorÚnn)Ú	LayerNormé   )ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)ÚGradientCheckpointingLayer)ÚBaseModelOutput)ÚPreTrainedModel)ÚModelOutputÚauto_docstringÚloggingÚtorch_compilable_checké   )ÚProphetNetConfigc                 óÄ   — |r/t         j                  j                  | j                  «       |¬«      S t         j                  j                  | |t        j
                  ¬«      S )N©Údim©r   Údtype)r   Ú
functionalÚsoftmaxÚfloatÚtorchÚfloat32)Úhidden_stater   Ú
onnx_traces      ú}/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/prophetnet/modeling_prophetnet.pyr   r   %   sH   € ÙÜ�}‰}×$Ñ$ \×%7Ñ%7Ó%9¸sÐ$ÓCÐCä�}‰}×$Ñ$ \°sÄ%Ç-Á-Ð$ÓPÐPó    c                 ón  — t        j                  |«      j                  }t        j                  | |¬«      j	                  d| d«      }t        j                  | |¬«      j	                  dd| «      }t        j                  ||¬«       dz   j	                  |dd«      }||z
  |k\  |dk7  z  }||k7  j                  || | «      }	t        j                  |||¬«      }
t        j                  d||¬«      }t        j                  ||
|«      }t        j                  |	|
|«      }t        j                  ||gd¬«      S )z@
    This function computes the bias for the predict stream
    )Údevicer   r   ©r   r%   © é   r   )
r   ÚfinfoÚminÚarangeÚviewÚexpandÚtensorÚzerosÚwhereÚcat)Úsequence_lengthÚngramr%   r   Úneg_infÚrowsÚcolsÚstream_offsetsÚ	left_maskÚ
right_maskÚ	neg_inf_tÚzero_tÚ
left_blockÚright_blocks                 r"   Úngram_attention_biasr>   ,   s  € ô �k‰k˜%Ó ×$Ñ$€GÜ�<‰<˜°Ô7×<Ñ<¸QÀÐQRÓS€DÜ�<‰<˜°Ô7×<Ñ<¸QÀÀ?ÓS€DÜ—|‘| E°&Ô9Ð9¸AÑ=×CÑCÀEÈ1ÈaÓP€Nð ˜‘ Ñ.°4¸1±9Ñ=€Ià˜$‘,×&Ñ& u¨o¸ÓO€Jä—‘˜W¨E¸&ÔA€IÜ�[‰[˜ 5°Ô8€FÜ—‘˜Y¨	°6Ó:€JÜ—+‘+˜j¨)°VÓ<€KÜ�9‰9�j +Ð.°AÔ6Ð6r#   c                 ó¦  — | }d}|rX| dz  } |t        j                  |t        j                  |«      «      j                  «       | z  z   }t        j                  |«      }n)t        j
                  |t        j                  |«      «      }| dz  }t        j                  ||«      }|t        j                  |j                  «       |z  «      t        j                  ||z  «      z  | |z
  z  z   }t        j                  |t        j                  |«      | dz
  z  «      j                  «       }|t        j                  ||j                  «       |«      z   }|S )zo
    This function computes individual parts of the relative position buckets. For more detail, see paper.
    r   r(   r   )r   ÚltÚ
zeros_likeÚintÚabsÚmaxÚlogr   Úmathr*   Ú	ones_liker0   )	Únum_bucketsÚmax_distanceÚrelative_positionsÚis_bidirectionalÚinv_relative_positionsÚrel_positions_bucketÚ	max_exactÚis_smallÚval_if_larges	            r"   Úcompute_relative_bucketsrQ   B   sG  € ð 1Ð0ÐØÐáØ! QÑ&ˆà Ü�h‰hÐ-¬u×/?Ñ/?Ð@VÓ/WÓX×\Ñ\Ó^ÐalÑlñmð 	ô "'§¡Ð+AÓ!BÑä!&§¡Ð+AÄ5×CSÑCSÐTjÓCkÓ!lÐà˜qÑ €IÜ�x‰xÐ.°	Ó:€HØœuŸy™yÐ)?×)EÑ)EÓ)GÈ)Ñ)SÓTÔW[×W_ÑW_Ø�yÑ óXñ  à	�yÑ	 ñ "ñ "€Lô —9‘9˜\¬5¯?©?¸<Ó+HÈKÐZ[ÉOÑ+\Ó]×aÑaÓc€LØ/´%·+±+¸hÐH^×HbÑHbÓHdÐfrÓ2sÑsÐØÐr#   c                 ó’  — |j                  d«      j                  d|j                  d«      d«      }||j                  d«      z
  }t        j                  |dz
  |fd¬«      j                  d«      }|j                  d|j                  d«      d«      }||j                  d«      z
  }t        | ||d¬«      }t        | ||d¬«      }||fS )zm
    This function computes both main and predict relative position buckets. For more detail, see paper.
    r   éÿÿÿÿr   F)rK   )Ú	unsqueezeÚrepeatÚsizer   r1   rQ   )rH   rI   Úposition_idsÚmain_stream_relative_positionsÚ$predicting_stream_relative_positionsÚmain_relative_position_bucketsÚ!predict_relative_position_bucketss          r"   Ú#compute_all_stream_relative_bucketsr\   ]   sí   € ð
 &2×%;Ñ%;¸AÓ%>×%EÑ%EÀaÈ×IZÑIZÐ[]ÓI^Ð`aÓ%bÐ"Ø%CÀl×F\ÑF\Ð]_ÓF`Ñ%`Ð"ô ,1¯9©9°lÀQÑ6FÈÐ5UÐ[]Ô+^×+hÑ+hÐijÓ+kÐ(Ø+O×+VÑ+VÐWXÐZf×ZkÑZkÐlnÓZoÐqrÓ+sÐ(Ø+OÐR^×RhÑRhÐikÓRlÑ+lÐ(ô &>Ø�\Ð#AÐTYô&Ð"ô )AØ�\Ð#GÐZ_ô)Ð%ð *Ð+LÐLÐLr#   zF
    Base class for sequence-to-sequence language models outputs.
    )Úcustom_introc                   ó  — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
edz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed	<   dZeej                     dz  ed
<   dZeej                     dz  ed<   dZej                  dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   y)ÚProphetNetSeq2SeqLMOutputaÖ  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss.
    logits (`torch.FloatTensor` of shape `(batch_size, decoder_sequence_length, config.vocab_size)`):
        Prediction scores of the main stream language modeling head (scores for each vocabulary token before
        SoftMax).
    logits_ngram (`torch.FloatTensor` of shape `(batch_size, ngram * decoder_sequence_length, config.vocab_size)`):
        Prediction scores of the predict stream language modeling head (scores for each vocabulary token before
        SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
        used (see `past_key_values` input) to speed up sequential decoding.
    decoder_ngram_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.

        Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
        outputs.
    decoder_ngram_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
        decoder_sequence_length, decoder_sequence_length)`.

        Attentions weights of the predict stream of the decoder, after the attention softmax, used to compute the
        weighted average in the self-attention heads.
    encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
        Sequence of hidden-states at the output of the last layer of the encoder of the model.
    NÚlossÚlogitsÚlogits_ngramÚpast_key_valuesÚdecoder_hidden_statesÚdecoder_ngram_hidden_statesÚdecoder_attentionsÚdecoder_ngram_attentionsÚcross_attentionsÚencoder_last_hidden_stateÚencoder_hidden_statesÚencoder_attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r`   r   ÚFloatTensorÚ__annotations__ra   rb   rc   r	   rd   Útuplere   rf   rg   rh   ri   rj   rk   r'   r#   r"   r_   r_   t   s)  … ñð< &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø'+€FˆE×Ñ Ñ$Ó+Ø-1€L�%×#Ñ# dÑ*Ó1Ø$(€O�U˜T‘\Ó(Ø=AÐ˜5 ×!2Ñ!2Ñ3°dÑ:ÓAØCGÐ  u×'8Ñ'8Ñ!9¸DÑ!@ÓGØ:>Ð˜˜e×/Ñ/Ñ0°4Ñ7Ó>Ø@DÐ˜e E×$5Ñ$5Ñ6¸Ñ=ÓDØ8<Ð�e˜E×-Ñ-Ñ.°Ñ5Ó<Ø:>Ð˜u×0Ñ0°4Ñ7Ó>Ø=AÐ˜5 ×!2Ñ!2Ñ3°dÑ:ÓAØ:>Ð˜˜e×/Ñ/Ñ0°4Ñ7Ô>r#   r_   z‹
    Base class for model encoder's outputs that also contains : pre-computed hidden states that can speed up sequential
    decoding.
    c                   óÖ  — e Zd ZU dZej
                  ed<   dZej
                  dz  ed<   dZe	dz  ed<   dZ
eej
                     dz  ed<   dZeej
                     dz  ed<   dZeej
                     dz  ed<   dZeej
                     dz  ed	<   dZeej
                     dz  ed
<   dZej
                  dz  ed<   dZeej
                     dz  ed<   dZeej
                     dz  ed<   y)ÚProphetNetSeq2SeqModelOutputaÀ  
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, decoder_sequence_length, hidden_size)`):
        Sequence of main stream hidden-states at the output of the last layer of the decoder of the model.

        If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
        hidden_size)` is output.
    last_hidden_state_ngram (`torch.FloatTensor` of shape `(batch_size,ngram * decoder_sequence_length, config.vocab_size)`, *optional*):
        Sequence of predict stream hidden-states at the output of the last layer of the decoder of the model.
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
        used (see `past_key_values` input) to speed up sequential decoding.
    decoder_ngram_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.

        Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
        outputs.
    decoder_ngram_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
        decoder_sequence_length, decoder_sequence_length)`.

        Attentions weights of the predict stream of the decoder, after the attention softmax, used to compute the
        weighted average in the
    encoder_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
        Sequence of hidden-states at the output of the last layer of the encoder of the model.
    Úlast_hidden_stateNÚlast_hidden_state_ngramrc   rd   re   rf   rg   rh   ri   rj   rk   )rl   rm   rn   ro   r   rp   rq   rv   rc   r	   rd   rr   re   rf   rg   rh   ri   rj   rk   r'   r#   r"   rt   rt   §   s  … ñð: ×(Ñ(Ó(Ø8<Ð˜U×.Ñ.°Ñ5Ó<Ø$(€O�U˜T‘\Ó(Ø=AÐ˜5 ×!2Ñ!2Ñ3°dÑ:ÓAØCGÐ  u×'8Ñ'8Ñ!9¸DÑ!@ÓGØ:>Ð˜˜e×/Ñ/Ñ0°4Ñ7Ó>Ø@DÐ˜e E×$5Ñ$5Ñ6¸Ñ=ÓDØ8<Ð�e˜E×-Ñ-Ñ.°Ñ5Ó<Ø:>Ð˜u×0Ñ0°4Ñ7Ó>Ø=AÐ˜5 ×!2Ñ!2Ñ3°dÑ:ÓAØ:>Ð˜˜e×/Ñ/Ñ0°4Ñ7Ô>r#   rt   zs
    Base class for model's outputs that may also contain a past key/values (to speed up sequential decoding).
    c                   óR  — e Zd ZU dZej
                  ed<   dZej
                  dz  ed<   dZe	dz  ed<   dZ
eej
                     dz  ed<   dZeej
                     dz  ed<   dZeej
                     dz  ed<   dZeej
                     dz  ed	<   dZeej
                     dz  ed
<   y)ÚProphetNetDecoderModelOutputaÇ  
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, decoder_sequence_length, hidden_size)`):
        Sequence of main stream hidden-states at the output of the last layer of the decoder of the model.

        If `past_key_values` is used only the last hidden-state of the sequences of shape `(batch_size, 1,
        hidden_size)` is output.
    last_hidden_state_ngram (`torch.FloatTensor` of shape `(batch_size, ngram * decoder_sequence_length, config.vocab_size)`):
        Sequence of predict stream hidden-states at the output of the last layer of the decoder of the model.
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
        used (see `past_key_values` input) to speed up sequential decoding.
    hidden_states_ngram (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.

        Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
        outputs.
    ngram_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
        decoder_sequence_length, decoder_sequence_length)`.

        Attentions weights of the predict stream of the decoder, after the attention softmax, used to compute the
        weighted average in the
    ru   Nrv   rc   Úhidden_statesÚhidden_states_ngramÚ
attentionsÚngram_attentionsrh   )rl   rm   rn   ro   r   rp   rq   rv   rc   r	   ry   rr   rz   r{   r|   rh   r'   r#   r"   rx   rx   Ù   s¾   … ñð6 ×(Ñ(Ó(Ø8<Ð˜U×.Ñ.°Ñ5Ó<Ø$(€O�U˜T‘\Ó(Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø;?Ð˜˜u×0Ñ0Ñ1°DÑ8Ó?Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ó6Ø8<Ð�e˜E×-Ñ-Ñ.°Ñ5Ó<Ø8<Ð�e˜E×-Ñ-Ñ.°Ñ5Ô<r#   rx   c                   ó„  — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
edz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed	<   dZeej                     dz  ed
<   dZeej                     dz  ed<   y)ÚProphetNetDecoderLMOutputa¶	  
    ngram_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.

        Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
        outputs.
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Language modeling loss.
    logits (`torch.FloatTensor` of shape `(batch_size, decoder_sequence_length, config.vocab_size)`):
        Prediction scores of the main stream language modeling head (scores for each vocabulary token before
        SoftMax).
    logits_ngram (`torch.FloatTensor` of shape `(batch_size, ngram * decoder_sequence_length, config.vocab_size)`):
        Prediction scores of the predict stream language modeling head (scores for each vocabulary token before
        SoftMax).
    past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
        It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

        Contains pre-computed hidden-states (key and values in the attention blocks) of the decoder that can be
        used (see `past_key_values` input) to speed up sequential decoding.
    hidden_states_ngram (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, ngram * decoder_sequence_length, hidden_size)`.

        Hidden-states of the predict stream of the decoder at the output of each layer plus the initial embedding
        outputs.
    ngram_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_attn_heads,
        decoder_sequence_length, decoder_sequence_length)`.

        Attentions weights of the predict stream of the decoder, after the attention softmax, used to compute the
        weighted average in the
    Nr`   ra   rb   rc   ry   rz   r{   r|   rh   )rl   rm   rn   ro   r`   r   rp   rq   ra   rb   rc   r	   ry   rr   rz   r{   r|   rh   r'   r#   r"   r~   r~     sÜ   … ñ ðD &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø'+€FˆE×Ñ Ñ$Ó+Ø-1€L�%×#Ñ# dÑ*Ó1Ø$(€O�U˜T‘\Ó(Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø;?Ð˜˜u×0Ñ0Ñ1°DÑ8Ó?Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ó6Ø8<Ð�e˜E×-Ñ-Ñ.°Ñ5Ó<Ø8<Ð�e˜E×-Ñ-Ñ.°Ñ5Ô<r#   r~   c                   ó&   — e Zd ZU eed<   dZdZd„ Zy)ÚProphetNetPreTrainedModelÚconfigÚ
prophetnetTc                 ó‚  — | j                   j                  }| j                   j                  }|€J d«       ‚|j                  |j                  «      }|dd d…f   j                  «       |ddd …f<   ||d<   |€J d«       ‚|j                  |dk(  |«       t        j                  |dk\  «      j                  «       sJ d	«       ‚|S )
Nz™self.model.config.decoder_start_token_id has to be defined. In ProphetNet it is usually set to the pad_token_id. See ProphetNet docs for more information.rS   r   ).r   z1self.model.config.pad_token_id has to be defined.éœÿÿÿr   z8Verify that `shifted_input_ids` has only positive values)
r�   Údecoder_start_token_idÚpad_token_idÚ	new_zerosÚshapeÚcloneÚmasked_fill_r   ÚallÚitem)ÚselfÚ	input_idsr…   r†   Úshifted_input_idss        r"   Ú_shift_rightz&ProphetNetPreTrainedModel._shift_right?  sØ   € Ø!%§¡×!CÑ!CÐØ—{‘{×/Ñ/ˆà%Ð1ð 	
ðFó	
Ð1ð &×/Ñ/°	·±Ó@ÐØ%.¨s°C°R°C¨xÑ%8×%>Ñ%>Ó%@Ð˜#˜q™r˜'Ñ"Ø$:Ð˜&Ñ!àÐ'Ð\Ð)\Ó\Ð'à×&Ñ&Ð'8¸DÑ'@À,ÔOä�y‰yÐ*¨aÑ/Ó0×5Ñ5Ô7ÐsÐ9sÓsÐ7à Ð r#   N)rl   rm   rn   r   rq   Úbase_model_prefixÚsupports_gradient_checkpointingr�   r'   r#   r"   r€   r€   9  s   … àÓØ$ÐØ&*Ð#ó!r#   r€   c                   óB   ‡ — e Zd ZdZdeddfˆ fd„Zdˆ fd„	Zˆ fd„Zˆ xZS )	ÚProphetNetPositionalEmbeddingsa  
    This module learns positional embeddings up to a fixed maximum size. Padding ids are ignored by either offsetting
    based on padding_idx or by setting padding_idx to None and ensuring that the appropriate position ids are passed to
    the forward function.
    r�   ÚreturnNc                 ó†   •— |j                   | _        t        ‰| �  |j                   |j                  |j
                  «       y ©N)Úmax_position_embeddingsÚ
max_lengthÚsuperÚ__init__Úhidden_sizer†   ©r�   r�   Ú	__class__s     €r"   r›   z'ProphetNetPositionalEmbeddings.__init__]  s3   ø€ Ø ×8Ñ8ˆŒÜ‰Ñ˜×7Ñ7¸×9KÑ9KÈV×M`ÑM`Õar#   c                 óD  •— |�| j                   �J d«       ‚|€ø|�k|j                  «       dk7  rX|j                  «       }|d   |z   }t        j                  dt        j                  |¬«      t        | j                   |z   «      z  }n‹|€&t        j                  |t        j                  |¬«      }t        j                  |d¬«      j                  |«      |z  j	                  «       | j                   z   }|j                  d| j                  dz
  «      }t        ‰| �-  |«      |fS )NzCIf position_ids is pre-computed then padding_idx should not be set.r   r   )r   r   r&   r   )Úpadding_idxÚget_seq_lengthr   ÚonesÚlongrB   ÚcumsumÚtype_asÚclampr™   rš   Úforward)	r�   Úinputs_shaper%   Úattention_maskrc   rW   Úprev_num_input_idsÚnum_input_idsrž   s	           €r"   r§   z&ProphetNetPositionalEmbeddings.forwarda  s  ø€ ØÐ$¨$×*:Ñ*:Ð*Bð 	
ØQó	
ÐCð ÐØÐ*¨×/MÑ/MÓ/OÐSTÒ/Tð &5×%CÑ%CÓ%EÐ"Ø ,¨Q¡Ð2DÑ D�Ü$Ÿz™z¨&¼¿
¹
È6ÔRÜ˜×(Ñ(¨=Ñ8Ó9ñ ‘ð "Ð)Ü%*§Z¡Z°ÄEÇJÁJÐW]Ô%^�Nô —L‘L °QÔ7×?Ñ?ÀÓOÐR`Ñ`ß‘$“&˜4×+Ñ+ñ ,�ð
  ,×1Ñ1°!°T·_±_ÀqÑ5HÓI�ä‰w‰˜|Ó,¨lÐ:Ð:r#   c                 ó"   •— t         ‰| �  |«      S r—   )rš   r§   )r�   rW   rž   s     €r"   Ú_forwardz'ProphetNetPositionalEmbeddings._forward}  s   ø€ Ü‰w‰˜|Ó,Ð,r#   )NNN)	rl   rm   rn   ro   r   r›   r§   r­   Ú__classcell__©rž   s   @r"   r”   r”   V  s.   ø„ ñðbÐ/ð b°Dõ bõ;÷8-ð -r#   r”   c                   ó†   ‡ — e Zd ZdZddedededz  fˆ fd„Z	 	 	 	 ddedz  dedz  d	edz  d
e	dz  de
eedz  f   f
d„Zˆ xZS )ÚProphetNetAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNr�   Únum_attn_headsÚ	layer_idxc                 ó¸  •— t         ‰| �  «        |j                  }|j                  | _        |j                  | _        || _        ||z  | _        || _        | j                  |z  |k(  sJ d«       ‚t        j                  ||«      | _
        t        j                  ||«      | _        t        j                  ||«      | _        t        j                  ||«      | _        y )Nzw`config.hidden_size` must be divisible by `config.num_encoder_attention_heads` and `config.num_decoder_attention_heads`)rš   r›   rœ   Úattention_dropoutÚdropoutr²   Úhead_dimr³   r   ÚLinearÚkey_projÚ
value_projÚ
query_projÚout_proj)r�   r�   r²   r³   rœ   rž   s        €r"   r›   zProphetNetAttention.__init__„  sº   ø€ Ü‰ÑÔØ×(Ñ(ˆà!'×!9Ñ!9ˆÔØ—~‘~ˆŒØ,ˆÔØ# ~Ñ5ˆŒØ"ˆŒà�}‰}˜~Ñ-°Ò<ð 	
ð4ó	
Ð<ô
 Ÿ	™	 +¨{Ó;ˆŒÜŸ)™) K°Ó=ˆŒÜŸ)™) K°Ó=ˆŒäŸ	™	 +¨{Ó;ˆ�r#   Úkey_value_statesr©   rc   Úoutput_attentionsr•   c                 ó†  — |j                  «       \  }}}	|d u}
t        |j                  «       «      |||	gk(  sJ d|||	f› d|j                  «       › �«       ‚| j                  |«      | j                  dz  z  }d}|�St	        |t
        «      rA|j                  j                  | j                  «      }|
r|j                  }n|j                  }n|}|
r|n|}|
rK|�I|rGj                  | j                     j                  }|j                  | j                     j                  }nß| j                  |«      }| j                  |«      }|j!                  |d| j"                  | j                  «      j%                  dd«      }|j!                  |d| j"                  | j                  «      j%                  dd«      }|�Kj'                  ||| j                  «      \  }}|
r)t	        |t
        «      rd|j                  | j                  <   |j!                  ||| j"                  | j                  «      j%                  dd«      }|j                  d«      }t)        j*                  d	||j%                  dd
«      «      }|| j"                  ||f}|j                  «       |k7  rt-        d|› d|j                  «       › �«      ‚|�|j/                  «       dk(  rd }|| j"                  d|f}|�2|j                  «       |k7  rt-        d|› d|j                  «       › �«      ‚|�||z   }|r|}nd }t0        j2                  j5                  |d¬«      }t0        j2                  j7                  || j8                  | j:                  ¬«      }t)        j*                  d	||«      }|| j"                  || j                  f}|j                  «       |k7  rt-        d|› d|j                  «       › �«      ‚|j%                  dd«      j=                  |||	«      }| j?                  |«      }t0        j2                  j7                  || j6                  | j:                  ¬«      }||fS )Nz Size of hidden states should be z	, but is ç      à?FrS   r   r(   Tzbsij,bsjk->bsikr   z#Attention weights should have size r   z Attention mask should have size r   ©ÚpÚtrainingz `attn_output` should have shape ú, but is of shape ) rV   Úlistr»   r·   Ú
isinstancer   Ú
is_updatedÚgetr³   Úcross_attention_cacheÚself_attention_cacheÚlayersÚkeysÚvaluesr¹   rº   r,   r²   Ú	transposeÚupdater   ÚeinsumÚ
ValueErrorr   r   r   r   r¶   rµ   rÃ   Úreshaper¼   )r�   ry   r½   r©   rc   r¾   ÚkwargsÚ
batch_sizeÚtgt_lenrœ   Úis_cross_attentionÚquery_statesrÇ   Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚsrc_lenÚattn_weightsÚexpected_shapeÚattn_weights_reshapedÚ
attn_probsÚattn_outputs                          r"   r§   zProphetNetAttention.forward™  s<  € ð ,9×+=Ñ+=Ó+?Ñ(ˆ
�G˜[ð .°TÐ9ÐÜ�M×&Ñ&Ó(Ó)ØØØð.
ò 
ð 	pð .¨j¸'À;Ð.NÐ-OÈyÐYf×YkÑYkÓYmÐXnÐoó		pð 
ð —‘ }Ó5¸¿¹ÈÑ9KÑLˆàˆ
ØÐ&Ü˜/Ô+>Ô?Ø,×7Ñ7×;Ñ;¸D¿N¹NÓK�
Ù%à+:×+PÑ+PÑ(à+:×+OÑ+OÑ(à'6Ð$á-?Ñ)À]ˆÙ /Ð"=Á*à-×4Ñ4°T·^±^ÑD×IÑIˆJØ/×6Ñ6°t·~±~ÑF×MÑM‰LàŸ™ ~Ó6ˆJØŸ?™?¨>Ó:ˆLØ#Ÿ™¨°R¸×9LÑ9LÈdÏmÉmÓ\×fÑfÐghÐjkÓlˆJØ'×,Ñ,¨Z¸¸T×=PÑ=PÐRV×R_ÑR_Ó`×jÑjÐklÐnoÓpˆLàÐ*à+?×+FÑ+FÀzÐS_Ðae×aoÑaoÓ+pÑ(�
˜Lá%¬*°_ÔFYÔ*ZØAE�O×.Ñ.¨t¯~©~Ñ>à#×(Ñ(¨°W¸d×>QÑ>QÐSW×S`ÑS`Óa×kÑkÐlmÐopÓqˆØ—/‘/ !Ó$ˆä—|‘|Ð$5°|ÀZ×EYÑEYÐZ[Ð]^ÓE_Ó`ˆØ$ d×&9Ñ&9¸7ÀGÐLˆØ×ÑÓ .Ò0ÜÐBÀ>ÐBRÐR[Ð\h×\mÑ\mÓ\oÐ[pÐqÓrÐrð Ð%¨.×*<Ñ*<Ó*>À!Ò*CØ!ˆNà$ d×&9Ñ&9¸1¸gÐFˆØÐ%¨.×*=Ñ*=Ó*?À>Ò*QÜÐ?ÀÐ?OÈyÐYg×YlÑYlÓYnÐXoÐpÓqÐqØÐ%Ø'¨.Ñ8ˆLÙØ$0Ñ!à$(Ð!ä—}‘}×,Ñ,¨\¸rÐ,ÓBˆä—]‘]×*Ñ*ØØ×$Ñ$Ø—]‘]ð +ó 
ˆ
ô
 —l‘lÐ#4°jÀ,ÓOˆØ$ d×&9Ñ&9¸7ÀDÇMÁMÐRˆØ×ÑÓ Ò/ÜÐ?ÀÐ?OÐOaÐbm×brÑbrÓbtÐauÐvÓwÐwà!×+Ñ+¨A¨qÓ1×9Ñ9¸*ÀgÈ{Ó[ˆØ—m‘m KÓ0ˆä—m‘m×+Ñ+¨K¸4¿<¹<ÐRV×R_ÑR_Ð+Ó`ˆØÐ1Ð1Ð1r#   r—   )NNNF)rl   rm   rn   ro   r   rB   r›   r   r	   Úboolrr   r§   r®   r¯   s   @r"   r±   r±   �  s‘   ø„ ÙGñ<Ð/ð <Àð <ÐQTÐW[ÑQ[õ <ð0 +/Ø(,Ø(,Ø).ñ[2ð ! 4™-ð[2ð  ™ð	[2ð
  ™ð[2ð   $™;ð[2ð 
ˆv�v ‘}Ð$Ñ	%÷[2r#   r±   c                   ó2   ‡ — e Zd ZdZdedefˆ fd„Zd„ Zˆ xZS )ÚProphetNetFeedForwardzm
    This is the residual two feed-forward layer block based on the original Transformer implementation.
    r�   Úffn_dimc                 ó*  •— t         ‰| �  «        t        |j                     | _        t        j                  |j                  |«      | _        t        j                  ||j                  «      | _	        |j                  | _
        |j                  | _        y r—   )rš   r›   r   Úactivation_functionÚactivation_fnr   r¸   rœ   ÚintermediateÚoutputÚactivation_dropoutr¶   )r�   r�   rå   rž   s      €r"   r›   zProphetNetFeedForward.__init__ü  sk   ø€ Ü‰ÑÔÜ# F×$>Ñ$>Ñ?ˆÔÜŸI™I f×&8Ñ&8¸'ÓBˆÔÜ—i‘i ¨×);Ñ);Ó<ˆŒØ"(×";Ñ";ˆÔØ—~‘~ˆ�r#   c                 óD  — | j                  |«      }| j                  |«      }t        j                  j	                  || j
                  | j                  ¬«      }| j                  |«      }t        j                  j	                  || j                  | j                  ¬«      }|S )NrÁ   )ré   rè   r   r   r¶   rë   rÃ   rê   )r�   ry   s     r"   r§   zProphetNetFeedForward.forward  s„   € Ø×)Ñ)¨-Ó8ˆØ×*Ñ*¨=Ó9ˆäŸ™×-Ñ-¨m¸t×?VÑ?VÐae×anÑanÐ-ÓoˆØŸ™ MÓ2ˆÜŸ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØÐr#   )	rl   rm   rn   ro   r   rB   r›   r§   r®   r¯   s   @r"   rä   rä   ÷  s!   ø„ ñð&Ð/ð &¸#õ &ör#   rä   c                   ó^   ‡ — e Zd Zd
defˆ fd„Zd„ Zd„ Z	 	 	 	 	 	 ddedz  fd„Zd„ Z	d	„ Z
ˆ xZS )ÚProphetNetNgramSelfAttentionNr�   c                 ó²  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | j                  z  | _	        |j                  | _
        || _        | j                  | j                  z  |j                  k(  sJ d«       ‚t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        t        j                  |j                  | j                  | j                  z  «      | _        d| _        y )Nz6config.hidden_size must be divisible by num_attn_headsF)rš   r›   rœ   rH   Úrelative_max_distanceÚnum_decoder_attention_headsr²   r¶   rµ   r·   r3   r³   r   r¸   r¹   rº   r»   r¼   Úrelative_pos_embeddingsr!   ©r�   r�   r³   rž   s      €r"   r›   z%ProphetNetNgramSelfAttention.__init__  sh  ø€ Ü‰ÑÔØ!×-Ñ-ˆÔà!×-Ñ-ˆÔØ%+×%AÑ%AˆÔ"Ø$×@Ñ@ˆÔØ—~‘~ˆŒØ!'×!9Ñ!9ˆÔØ×*Ñ*¨d×.AÑ.AÑAˆŒØ—\‘\ˆŒ
Ø"ˆŒà�}‰}˜t×2Ñ2Ñ2°f×6HÑ6HÒHð 	
ØDó	
ÐHô Ÿ	™	 &×"4Ñ"4°f×6HÑ6HÓIˆŒÜŸ)™) F×$6Ñ$6¸×8JÑ8JÓKˆŒÜŸ)™) F×$6Ñ$6¸×8JÑ8JÓKˆŒô Ÿ	™	 &×"4Ñ"4°f×6HÑ6HÓIˆŒô (*§y¡y°×1CÑ1CÀT×EUÑEUÐX\×XkÑXkÑEkÓ'lˆÔ$ð  ˆ�r#   c                 óŽ   — |j                  ||| j                  | j                  «      j                  dd«      j	                  «       S ©Nr   r(   )r,   r²   r·   rÎ   Ú
contiguous)r�   r.   Úseq_lenrÔ   s       r"   Ú_shapez#ProphetNetNgramSelfAttention._shape-  s9   € Ø�{‰{˜: w°×0CÑ0CÀTÇ]Á]ÓS×]Ñ]Ð^_ÐabÓc×nÑnÓpÐpr#   c                 ó   — d| _         y )NT)r!   ©r�   s    r"   Úprepare_for_onnx_export_z5ProphetNetNgramSelfAttention.prepare_for_onnx_export_0  s	   € Øˆ�r#   rc   c           	      ó2  — |j                  «       \  }	}
}t        |j                  «       «      |	|
|gk(  sJ d|	|
|f› d|j                  › �«       ‚| j                  |«      }| j	                  |«      }| j                  |«      }|| j                  dz  z  }| j                  ||
|	«      }| j                  |d|	«      }| j                  |d|	«      }|	| j                  d| j                  f} |j                  |Ž } |j                  |Ž } |j                  |Ž }|j                  d| j                  z   d¬«      }|j                  d| j                  z   d¬«      }|j                  d| j                  z   d¬«      }|j                  d| j                  z   d¬«      }|d   |dd  }}|d   |dd  }}|d   |dd  }}|d   |dd  }}|�?t        |t        «      r|j                  }n|}|j                  ||| j                   «      \  }}|
d| j                  z   z  }t#        j$                  d	||j'                  dd
«      «      }| j)                  ||||«      }||z   }|�||z   }t+        |d| j,                  ¬«      j/                  |«      } t0        j2                  j5                  | | j6                  | j8                  ¬«      } t#        j$                  d	| |«      }!|!j'                  dd«      j                  |	d||«      }!| j;                  |!«      }!t#        j<                  |d«      j?                  |	| j                  | j                  || j                  «      }"t#        j<                  |D �#cg c]  }#t#        j@                  ||#gd«      ‘Œ c}#d«      }$t#        j<                  |d¬«      }%t#        j@                  |D �&cg c])  }&t#        j@                  ||&gd«      jC                  d«      ‘Œ+ c}&d«      }'t#        j$                  d|"|$f«      }(| jE                  |%|(||«      })|(|)z   }(|�5|jG                  dddd
d«      }|jI                  |(jJ                  «      }|(|z   }(t+        |(d| j,                  ¬«      j/                  |(«      }*t0        j2                  j5                  |*| j6                  | j8                  ¬«      }*t#        j$                  d|*|'j'                  dd«      f«      }+|+j'                  dd
«      }+|+j                  |	| j                  ||«      }+| j;                  |+«      }+t#        j@                  |!|+gd«      j?                  |	d|«      },| j?                  |	| j                  |d«      } t0        j2                  j5                  |,| j4                  | j8                  ¬«      },|,| |*fS c c}#w c c}&w )Nz#`hidden_states` should be of shape rÄ   rÀ   rS   r   r   r(   r   zbntc,bncs->bntsr   )r   r!   rÁ   zbnhtc,bnhsc->bnhtsé   zbnhts,bnhsc->bnhtc)&rV   rÅ   rˆ   r»   r¹   rº   r·   rø   r²   rÒ   Úchunkr3   rÆ   r   rÊ   rÏ   r³   r   rÐ   rÎ   Ú get_main_relative_pos_embeddingsr   r!   r¥   r   r   r¶   rµ   rÃ   r¼   Ústackr,   r1   rT   Ú#get_predict_relative_pos_embeddingsÚpermuteÚtor   )-r�   ry   rc   r©   Úextended_predict_attention_maskrZ   r[   rW   rÓ   rÔ   Úngram_sequence_lengthrœ   r×   rÚ   rÛ   Ú
proj_shapeÚhidden_states_listÚquery_states_listÚkey_states_listÚvalue_states_listÚmain_hidden_statesÚhidden_states_predict_listÚmain_query_statesÚpredict_query_states_listÚmain_key_statesÚpredict_key_states_listÚmain_value_statesÚpredict_value_states_listrØ   r2   Úmain_attn_weightsÚmain_relative_pos_embeddingsÚmain_attn_probsÚmain_attn_outputÚpredict_query_statesÚkeyÚpredict_key_statesÚpredict_hidden_statesÚv_pÚpredict_value_statesÚpredict_attn_weightsÚpredict_relative_pos_embeddingsÚpredict_attn_probsÚpredict_attn_outputrá   s-                                                r"   r§   z$ProphetNetNgramSelfAttention.forward3  s	  € ð :G×9KÑ9KÓ9MÑ6ˆ
Ð)¨;Ü�M×&Ñ&Ó(Ó)¨jÐ:OÐQ\Ð-]Ò]ð 	
Ø1°*Ð>SÐU`Ð2`Ð1að bØ×#Ñ#Ð$ð&ó	
Ð]ð —‘ }Ó5ˆØ—]‘] =Ó1ˆ
Ø—‘ }Ó5ˆð $ t§}¡}°cÑ'9Ñ:ˆð —{‘{ <Ð1FÈ
ÓSˆØ—[‘[ ¨R°Ó<ˆ
Ø—{‘{ <°°ZÓ@ˆØ  $×"5Ñ"5°r¸4¿=¹=ÐIˆ
à+�|×+Ñ+¨ZÐ8ˆØ'�Z×'Ñ'¨Ð4ˆ
Ø+�|×+Ñ+¨ZÐ8ˆð +×0Ñ0°°T·Z±Z±ÀQÐ0ÓGÐØ(×.Ñ.¨q°4·:±:©~À1Ð.ÓEÐØ$×*Ñ*¨1¨t¯z©z©>¸qÐ*ÓAˆØ(×.Ñ.¨q°4·:±:©~À1Ð.ÓEÐà9KÈAÑ9NÐPbÐcdÐceÐPfÐ6ÐØ7HÈÑ7KÐM^Ð_`Ð_aÐMbÐ4ÐØ3BÀ1Ñ3EÀÐWXÐWYÐGZÐ0ˆØ7HÈÑ7KÐM^Ð_`Ð_aÐMbÐ4Ðð Ð&Ü˜/Ô+>Ô?Ø'6×'KÑ'KÑ$à'6Ð$Ø1E×1LÑ1LØÐ!2°D·N±Nó2Ñ.ˆOÐ.ð
 0°A¸¿
¹
±NÑCˆô "ŸL™LÐ):Ð<MÈ×OhÑOhÐijÐlmÓOnÓoÐð (,×'LÑ'LØÐ 1°<ÐA_ó(
Ð$ð .Ð0LÑLÐàÐ%Ø 1°NÑ BÐä!ØØØ—‘ô
÷ ‰'Ð#Ó
$ð	 	ô Ÿ-™-×/Ñ/°À4×CYÑCYÐdh×dqÑdqÐ/Órˆô
 !Ÿ<™<Ð(9¸?ÐL]Ó^Ðà+×5Ñ5°a¸Ó;×CÑCÀJÐPQÐSbÐdoÓpÐØŸ=™=Ð)9Ó:Ðô  %Ÿ{™{Ð+DÀaÓH×MÑMØ˜Ÿ
™
 D×$7Ñ$7¸È$Ï-É-ó 
Ðô
 #Ÿ[™[ÐZqÖ)rÐSV¬%¯)©)°_ÀcÐ4JÈAÕ*NÒ)rÐtuÓvÐô !&§¡Ð,FÈAÔ NÐô  %Ÿy™yØLeÖfÀSŒU�Y‰YÐ)¨3Ð/°Ó3×=Ñ=¸aÕ@ÒfÐhió 
Ðô  %Ÿ|™|Ð,@ÐCWÐYkÐBlÓmÐð +/×*RÑ*RØ!Ð#7¸ÐGhó+
Ð'ð
  4Ð6UÑUÐà*Ð6à.M×.UÑ.UÐVWÐYZÐ\]Ð_`ÐbcÓ.dÐ+Ø.M×.PÑ.PÐQe×QkÑQkÓ.lÐ+Ø#7Ð:YÑ#YÐ ä$Ø ØØ—‘ô
÷ ‰'Ð&Ó
'ð	 	ô  Ÿ]™]×2Ñ2Ø $×"8Ñ"8À4Ç=Á=ð 3ó 
Ðô $Ÿl™lØ Ð#5Ð7K×7UÑ7UÐVWÐYZÓ7[Ð"\ó
Ðð 2×;Ñ;¸A¸qÓAÐØ1×9Ñ9¸*ÀdÇjÁjÐRaÐcnÓoÐØ"Ÿm™mÐ,?Ó@Ðô —i‘iÐ!1Ð3FÐ GÈÓK×PÑPÐQ[Ð]_ÐalÓmˆà)×.Ñ.¨z¸4×;NÑ;NÐP_ÐacÓdˆä—m‘m×+Ñ+¨K¸4¿<¹<ÐRV×R_ÑR_Ð+Ó`ˆà˜OÐ-?Ð?Ð?ùò{ *sùò gs   ÍVÎ).Vc                 ó  — |j                   \  }}}}|j                  ||||«      }|€Ç|j                   d d \  }}	t        j                  d|j                   d   dz   «      j	                  d«      j	                  d«      j                  ||	d«      j                  |j                  «      }
|
|j	                  d«      j                  ||	d«      z
  }
t        | j                  | j                  |
d«      }| j                  |«      }|j                  |j                   d d | j                  | j                  fz   «      }|j                  dddd«      }|j                  |j                   d d dz   «      }|j                  d| j                  d«      }|j                  d|j                   d   «      }|j                  «       }|j                  d|j!                  d«      «      }t        j"                  |d|¬«      }|j                  |||d«      }|S )	Nr(   r   rS   r   Fr   )rS   ©r   Úindex)rˆ   r,   r   r+   rT   rU   r  r%   rQ   rH   rð   rò   r²   r  rÒ   r£   rV   Úgather)r�   ry   rÝ   rW   rZ   rÔ   r²   rÕ   rÜ   r2   rJ   Úrel_pos_embeddingsr  s                r"   rÿ   z=ProphetNetNgramSelfAttention.get_main_relative_pos_embeddingsÖ  s  € ð 8D×7IÑ7IÑ4ˆ
�N G¨WØ#×(Ñ(¨°^ÀWÈgÓVˆØ)Ð1Ø*7×*=Ñ*=¸b¸qÐ*AÑ'ˆJ˜ä—‘˜Q × 2Ñ 2°2Ñ 6¸Ñ :Ó;ß‘˜1“ß‘˜1“ß‘˜
 O°QÓ7ß‘�L×'Ñ'Ó(ð ð "4°l×6LÑ6LÈQÓ6O×6VÑ6VÐWaÐcrÐtuÓ6vÑ!vÐÜ-EØ× Ñ  $×"<Ñ"<Ð>PÐRWó.Ð*ð
 "×9Ñ9¸-ÓHÐØ/×4Ñ4Ø×$Ñ$ R aÐ(¨D×,<Ñ,<¸d×>QÑ>QÐ+RÑRó
Ðð 0×7Ñ7¸¸1¸aÀÓCÐà/×7Ñ7¸×8JÑ8JÈ2ÈAÐ8NÐQVÑ8VÓWÐà)G×)NÑ)NÈqÐRV×ReÑReÐghÓ)iÐ&à)G×)LÑ)LØÐ.×4Ñ4°RÑ8ó*
Ð&ð *H×)LÑ)LÓ)NÐ&à/×7Ñ7¸Ð<N×<SÑ<SÐTVÓ<WÓXÐä',§|¡|Ð4FÈAÐUsÔ'tÐ$Ø'C×'HÑ'HÈÐUcÐelÐnpÓ'qÐ$Ø+Ð+r#   c                 ó.  — |j                   dd \  }}|€Ì|j                   d   }t        |d   d   |dz
  k(  d«       t        j                  d|«      j	                  d«      j	                  d«      j                  ||d«      j                  |j                  «      }||j	                  d«      j                  ||d«      z
  }t        | j                  | j                  |d«      }|j                  dd«      }| j                  |«      }	|	j                  |j                   d d | j                  | j                  fz   «      }	|	j                  ddddd«      }	|	j!                  d| j                  «      }	|j	                  d«      }|j                  | j"                  d| j                  d«      }|j                  d|j%                  d«      «      j'                  «       }t        j(                  |	d|¬	«      }
|
j                  || j"                  | j                  |d«      }
|
S )
Nr   r(   rS   r   zb`position_ids` are incorrect. They should be of the format 1 2 3 4 5 ... (key_sequence_length - 1)Frý   r   r"  )rˆ   r   r   r+   rT   rU   r  r%   rQ   rH   rð   rÎ   rò   r,   r²   r  rÒ   r3   rV   r£   r$  )r�   ry   rÝ   rW   r[   rÔ   r2   Úkey_sequence_lengthrJ   r%  r  s              r"   r  z@ProphetNetNgramSelfAttention.get_predict_relative_pos_embeddings  s&  € ð '4×&9Ñ&9¸!¸AÐ&>Ñ#ˆ
�Oà,Ð4Ø".×"4Ñ"4°RÑ"8ÐÜ"Ø˜Q‘ Ñ"Ð&9¸AÑ&=Ñ=Øtôô
 —‘˜QÐ 3Ó4ß‘˜1“ß‘˜1“ß‘˜
 O°QÓ7ß‘�L×'Ñ'Ó(ð ð "4°l×6LÑ6LÈQÓ6O×6VÑ6VÐWaÐcrÐtuÓ6vÑ!vÐÜ0HØ× Ñ  $×"<Ñ"<Ð>PÐRWó1Ð-ð
 &×/Ñ/°°1Ó5ˆØ!×9Ñ9¸-ÓHÐð 0×4Ñ4Ø×Ñ  Ð$¨×(8Ñ(8¸$×:MÑ:MÐ'NÑNó
Ðð 0×7Ñ7¸¸1¸aÀÀAÓFÐà/×7Ñ7¸¸D×<LÑ<LÓMÐà,M×,WÑ,WÐXYÓ,ZÐ)Ø,M×,TÑ,TØ�J‰J˜˜4×.Ñ.°ó-
Ð)ð -N×,RÑ,RØÐ1×6Ñ6°rÓ:ó-
ç
‰$‹&ð 	*ô +0¯,©,Ø AÐ-Nô+
Ð'ð
 +J×*NÑ*NØ˜Ÿ
™
 D×$7Ñ$7¸È"ó+
Ð'ð /Ð.r#   r—   ©NNNNNN)rl   rm   rn   r   r›   rø   rû   r	   r§   rÿ   r  r®   r¯   s   @r"   rî   rî     sR   ø„ ñ Ð/õ  ò<qòð )-ØØ(,Ø'+Ø*.Øña@ð  ™óa@òF+,öZ:/r#   rî   c                   ó8   ‡ — e Zd ZdZdefˆ fd„Z	 ddefd„Zˆ xZS )ÚProphetNetEncoderLayerz&
    Encoder block for Prophetnet
    r�   c                 óö   •— t         ‰| �  «        t        ||j                  «      | _        t        |j                  «      | _        t        ||j                  «      | _
        t        |j                  «      | _        y r—   )rš   r›   r±   Únum_encoder_attention_headsÚ	self_attnr   rœ   Úself_attn_layer_normrä   Úencoder_ffn_dimÚfeed_forwardÚfeed_forward_layer_normr�   s     €r"   r›   zProphetNetEncoderLayer.__init__E  s_   ø€ Ü‰ÑÔä,¨V°V×5WÑ5WÓXˆŒÜ$-¨f×.@Ñ.@Ó$AˆÔ!ô 2°&¸&×:PÑ:PÓQˆÔÜ'0°×1CÑ1CÓ'DˆÕ$r#   r¾   c                 ó¼   — | j                  |||¬«      \  }}| j                  ||z   «      }| j                  |«      }| j                  ||z   «      }|f}|r||fz  }|S )N)ry   r©   r¾   )r-  r.  r0  r1  )r�   ry   r©   r¾   Úattention_outputrÝ   Úfeed_forward_outputÚoutputss           r"   r§   zProphetNetEncoderLayer.forwardO  sƒ   € ð *.¯©Ø'Ø)Ø/ð *8ó *
Ñ&Ð˜,ð
 ×1Ñ1Ð2BÀ]Ñ2RÓSˆð #×/Ñ/°Ó>ÐØ×4Ñ4Ð5HÈ=Ñ5XÓYˆà Ð"ˆáØ˜�Ñ&ˆGàˆr#   ©F©	rl   rm   rn   ro   r   r›   râ   r§   r®   r¯   s   @r"   r*  r*  @  s+   ø„ ñðEÐ/õ Eð #(ñ	ð  ÷	r#   r*  c                   ó\   ‡ — e Zd ZdZddefˆ fd„Z	 	 	 	 	 	 	 	 	 	 d	dedz  dedz  fd„Zˆ xZS )
ÚProphetNetDecoderLayerz&
    Decoder block for Prophetnet
    Nr�   c                 ój  •— t         ‰| �  «        t        ||¬«      | _        t	        |j
                  «      | _        |j                  r7t        ||j                  |¬«      | _
        t	        |j
                  «      | _        t        ||j                  «      | _        t	        |j
                  «      | _        y )N©r³   )rš   r›   rî   r-  r   rœ   r.  Úadd_cross_attentionr±   rñ   Ú
cross_attnÚcross_attn_layer_normrä   Údecoder_ffn_dimr0  r1  ró   s      €r"   r›   zProphetNetDecoderLayer.__init__n  sŽ   ø€ Ü‰ÑÔä5°fÈ	ÔRˆŒÜ$-¨f×.@Ñ.@Ó$AˆÔ!ð ×%Ò%Ü1°&¸&×:\Ñ:\ÐhqÔrˆDŒOÜ)2°6×3EÑ3EÓ)FˆDÔ&ô 2°&¸&×:PÑ:PÓQˆÔÜ'0°×1CÑ1CÓ'DˆÕ$r#   Ú	use_cacher¾   c           	      ó,  — | j                  ||	|||||¬«      \  }}}| j                  ||z   «      }d }|�-| j                  ||||	|¬«      \  }}| j                  ||z   «      }| j	                  |«      }| j                  ||z   «      }|f}|r||||fz  }|S )N)ry   rc   r©   r  rZ   r[   rW   )ry   r½   r©   rc   r¾   )r-  r.  r=  r>  r0  r1  )r�   ry   r©   rj   Úencoder_attn_maskr  rZ   r[   rW   rc   r@  r¾   rÓ   Úngram_attention_outputÚself_attn_weightsÚself_attn_weights_ngramÚcross_attn_weightsr3  r4  r5  s                       r"   r§   zProphetNetDecoderLayer.forward}  sî   € ð  NRÏ^É^Ø'Ø+Ø)Ø,KØ+IØ.OØ%ð N\ó N
ÑJÐÐ 1Ð3Jð ×1Ñ1°-ÐBXÑ2XÓYˆà!ÐØ Ð,à37·?±?Ø+Ø!6Ø0Ø /Ø"3ð 4Có 4Ñ0ÐÐ0ð !×6Ñ6Ð7GÈ-Ñ7WÓXˆMð #×/Ñ/°Ó>ÐØ×4Ñ4Ð5HÈ=Ñ5XÓYˆà Ð"ˆáØÐ)Ð+BÐDVÐWÑWˆGàˆr#   r—   )
NNNNNNNNTFr7  r¯   s   @r"   r9  r9  i  sX   ø„ ññEÐ/õ Eð$ Ø"ØØ(,Ø'+Ø*.ØØØ!%Ø).ñ0ð ˜$‘;ð0ð   $™;÷0r#   r9  z=
    The standalone encoder part of the ProphetNetModel.
    c                   óÒ   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  d	e
dz  d
e
dz  de
dz  deez  fd„«       Zˆ xZS )ÚProphetNetEncoderr�   c                 ó®  •— t         ‰| �  |«       t        j                  |j                  |j
                  |j                  ¬«      | _        t        |«      | _	        t        |j
                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _        | j%                  «        y c c}w )N©r    F)rš   r›   r   Ú	EmbeddingÚ
vocab_sizerœ   r†   Úword_embeddingsr”   Úposition_embeddingsr   Úembeddings_layer_normÚ
ModuleListÚrangeÚnum_encoder_layersr*  rË   Úgradient_checkpointingÚ	post_init)r�   r�   Ú_rž   s      €r"   r›   zProphetNetEncoder.__init__¶  s�   ø€ Ü‰Ñ˜Ô ä!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ#AÀ&Ó#IˆÔ Ü%.¨v×/AÑ/AÓ%BˆÔ"ä—m‘mÌUÐSY×SlÑSlÓMmÖ$nÈÔ%;¸FÕ%CÒ$nÓoˆŒà&+ˆÔ#à�‰Õùò	 %os   ÂCc                 ó   — | j                   S r—   ©rM  rú   s    r"   Úget_input_embeddingsz&ProphetNetEncoder.get_input_embeddingsÃ  ó   € Ø×#Ñ#Ð#r#   c                 ó   — || _         y r—   rW  ©r�   Úvalues     r"   Úset_input_embeddingsz&ProphetNetEncoder.set_input_embeddingsÆ  ó
   € Ø$ˆÕr#   NrŽ   r©   Úinputs_embedsr¾   Úoutput_hidden_statesÚreturn_dictr•   c                 óü  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|€|€t	        d«      ‚|�|�t	        d«      ‚|�|€| j                  |«      }|�|d|dd…dddd…f   j                  d| j                   j                  dd«      z
  t        j                  | j                  «      j                  z  }|j                  |j                  «      }nd}| j                  |j                  dd |j                  «      \  }	}
||	z   }| j!                  |«      }t"        j$                  j'                  || j                   j&                  | j(                  ¬«      }|rdnd}|rdnd}t+        | j,                  «      D ])  \  }}|r||fz   } ||||¬	«      }|d
   }|sŒ!||d   fz   }Œ+ |r||fz   }|st/        d„ |||fD «       «      S t1        |||¬«      S )a	  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, ProphetNetEncoder
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased")
        >>> model = ProphetNetEncoder.from_pretrained("patrickvonplaten/prophetnet-large-uncased-standalone")
        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> last_hidden_states = outputs.last_hidden_state
        ```Nz3Either input_ids or inputs_embeds has to be passed.z2Make sure to only pass input_ids or inputs_embeds.ç      ð?r   r(   rÁ   r'   )r©   r¾   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr—   r'   ©Ú.0Úvs     r"   ú	<genexpr>z,ProphetNetEncoder.forward.<locals>.<genexpr>  s   è ø€ Òl˜qÐ^_Ñ^kœÑlùó   ‚Š)ru   ry   r{   )r�   r¾   r`  ra  rÑ   rM  rU   r,  r   r)   r   r*   r  rN  rˆ   r%   rO  r   r   r¶   rÃ   Ú	enumeraterË   rr   r   )r�   rŽ   r©   r_  r¾   r`  ra  rÓ   Úextended_attention_maskrN  rW   ry   rj   Úall_attentionsÚidxÚencoder_layerÚlayer_outputss                    r"   r§   zProphetNetEncoder.forwardÉ  sD  € ð4 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐ Ð!6ÜÐRÓSÐSØÐ" }Ð'@ÜÐQÓRÐRØÐ" }Ð'<Ø ×0Ñ0°Ó;ˆMð Ð%à�n¢Q¨¨d²AÐ%5Ñ6×=Ñ=¸aÀÇÁ×AhÑAhÐjkÐmnÓoÑoÜ—‘˜DŸJ™JÓ'×+Ñ+ñ',Ð#ð '>×&@Ñ&@À×ATÑATÓ&UÑ#à&*Ð#à,0×,DÑ,DÀ]×EXÑEXÐY[ÐZ[ÐE\Ð^k×^rÑ^rÓ,sÑ)Ð˜\à%Ð(;Ñ;ˆØ×2Ñ2°=ÓAˆÜŸ™×-Ñ-¨m¸t¿{¹{×?RÑ?RÐ]a×]jÑ]jÐ-Ókˆá&:¡ÀÐÙ0™°dˆä"+¨D¯K©KÓ"8ò 	FÑˆC�Ù#Ø(=ÀÐ@PÑ(PÐ%á)ØØ6Ø"3ôˆMð *¨!Ñ,ˆMâ Ø!/°=ÀÑ3CÐ2EÑ!E‘ð	Fñ  Ø$9¸]Ð<LÑ$LÐ!áÜÑl ]Ð4IÈ>Ð$ZÔlÓlÐlÜØ+Ð;PÐ]kô
ð 	
r#   r(  )rl   rm   rn   r   r›   rX  r]  r   r   r   râ   rr   r   r§   r®   r¯   s   @r"   rH  rH  °  s»   ø„ ðÐ/õ ò$ò%ð ð *.Ø.2Ø-1Ø)-Ø,0Ø#'ñN
à—<‘< $Ñ&ðN
ð Ÿ™ tÑ+ðN
ð —|‘| dÑ*ð	N
ð
   $™;ðN
ð # T™kðN
ð ˜D‘[ðN
ð 
�Ñ	 òN
ó ôN
r#   rH  z=
    The standalone decoder part of the ProphetNetModel.
    c                   ó<  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
e
dz  dej                  dz  dedz  dedz  dedz  dedz  deez  fd„«       Zd„ Zd„ Zd„ Zˆ xZS )ÚProphetNetDecoderr�   c           	      ó¼  •— t         ‰| �  |«       |j                  | _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        t        j                  |j                  |j                  |j                  ¬«      | _        t        |«      | _        t        j                  | j                  |j                  d «      | _        t        j"                  t%        |j&                  «      D �cg c]  }t)        ||¬«      ‘Œ c}«      | _        t-        |j                  «      | _        d| _        | j3                  «        y c c}w )NrJ  r;  F)rš   r›   r3   rH   rð   r¶   r˜   Úmax_target_positionsr   rK  rL  rœ   r†   rM  r”   rN  Úngram_embeddingsrP  rQ  Únum_decoder_layersr9  rË   r   rO  rS  rT  )r�   r�   Úirž   s      €r"   r›   zProphetNetDecoder.__init__!  s  ø€ Ü‰Ñ˜Ô à—\‘\ˆŒ
Ø!×-Ñ-ˆÔØ%+×%AÑ%AˆÔ"Ø—~‘~ˆŒØ$*×$BÑ$BˆÔ!ä!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ#AÀ&Ó#IˆÔ ä "§¡¨T¯Z©Z¸×9KÑ9KÈTÓ RˆÔÜ—m‘mÜBGÈ×HaÑHaÓBbÖc¸QÔ# F°aÖ8Òcó
ˆŒô &/¨v×/AÑ/AÓ%BˆÔ"à&+ˆÔ#à�‰Õùò ds   ÄEc                 ó   — | j                   S r—   rW  rú   s    r"   rX  z&ProphetNetDecoder.get_input_embeddings7  rY  r#   c                 ó   — || _         y r—   rW  r[  s     r"   r]  z&ProphetNetDecoder.set_input_embeddings:  r^  r#   NrŽ   r©   rj   Úencoder_attention_maskrc   r_  r@  r¾   r`  ra  r•   c                 ó
  — |�|n| j                   j                  }|�|n| j                   j                  }|	�|	n| j                   j                  }	|
�|
n| j                   j                  }
|€|€t        d«      ‚|�|�t        d«      ‚|�|€| j                  |«      }|j                  dd \  }}| j                  r%| j                  r|rt        j                  d«       d}|rd|€b|€| j                   j                  r4t        t        | j                   ¬«      t        | j                   ¬«      «      nt        | j                   ¬«      }|�|j                  «       nd}| j!                  ||f|j"                  |¬	«      \  }}|dk7  rd
\  }}n| j%                  |«      \  }}| j                   j'                  |dz   «      }||z   }| j(                  j*                  }|dk7  r\|j-                  d«      dk(  sJ d«       ‚t/        | j0                  «      D �cg c]  }||dz
     |z   j3                  |dd«      ‘Œ  }}d}d}nOt/        | j0                  «      D �cg c]  }||dz
     |z   ‘Œ }}| j5                  ||«      }| j7                  ||«      }|�|d|dd…dddd…f   j3                  d| j                   j8                  dd«      z
  t;        j<                  | j>                  «      j@                  z  }|jC                  |j>                  «      }nd}t;        jD                  |g|z   d«      }| jF                  r| jG                  |«      }tH        jJ                  jM                  || jL                  | j                  ¬«      }|	rdnd}|	r| j                   j0                  dkD  rdnd}|rdnd}|rdnd}|r| j                   jN                  rdnd}tQ        | jR                  «      D ]‹  \  } }!|	r7||dd…d|…f   fz  }| j                   j0                  dkD  r||dd…|d…f   fz  } |!|||||||||||¬«      }"|"d   }|sŒZ||"d   fz  }||"d   fz  }| j                   jN                  sŒƒ||"d   fz  }Œ� |	r7||dd…d|…f   fz  }| j                   j0                  dkD  r||dd…|d…f   fz  }|dd…d|…f   }#| j                   j0                  dkD  r|dd…|d…f   nd}$|
stU        d„ |#|$||||||fD «       «      S tW        |#|$||||||¬«      S c c}w c c}w )a  
        Example:

        ```python
        >>> from transformers import AutoTokenizer, ProphetNetDecoder
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased")
        >>> model = ProphetNetDecoder.from_pretrained("microsoft/prophetnet-large-uncased", add_cross_attention=False)
        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> last_hidden_states = outputs.last_hidden_state
        ```NzGEither `decoder_input_ids` or `decoder_inputs_embeds` has to be passed.zFMake sure to only pass `decoder_input_ids` or `decoder_inputs_embeds`.r(   zZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)r�   r   )r%   rc   )NNr   zOAt the moment `use_cache` is only supported for `decoder_input_ids` of length 1rc  rÁ   r'   )rB  r  rZ   r[   rW   rc   r@  r¾   r   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wr—   r'   re  s     r"   rh  z,ProphetNetDecoder.forward.<locals>.<genexpr>à  s   è ø€ ò àð �=ô ñùs   ‚)ru   rv   rc   ry   rz   r{   r|   rh   ),r�   r@  r¾   r`  ra  rÑ   rM  rˆ   rS  rÃ   ÚloggerÚwarning_onceÚis_encoder_decoderr   r
   r¡   rN  r%   Ú!compute_buffered_relative_bucketsr­   rt  ÚweightrV   rQ  r3   rU   Úprepare_attention_maskÚprepare_predict_attention_maskrñ   r   r)   r   r*   r  r1   rO  r   r   r¶   r<  rj  rË   rr   rx   )%r�   rŽ   r©   rj   ry  rc   r_  r@  r¾   r`  ra  rÓ   rÔ   r2   Úpast_key_values_lengthÚmain_stream_pos_embedrW   rZ   r[   Úpredicting_stream_pos_embedry   rt  r3   Úngram_hidden_statesrk  r  Úextended_encoder_attention_maskÚall_main_stream_hidden_statesÚall_ngram_stream_hidden_statesÚall_main_stream_attnsÚall_ngram_stream_attnsÚall_cross_attnsrm  Údecoder_layerro  ru   rv   s%                                        r"   r§   zProphetNetDecoder.forward=  sí  € ð: "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø1BÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐ Ð!6ÜÐfÓgÐgØÐ" }Ð'@ÜÐeÓfÐfØÐ" }Ð'<Ø ×0Ñ0°Ó;ˆMà&3×&9Ñ&9¸"¸1Ð&=Ñ#ˆ
�Oà×&Ò&¨4¯=ª=ÙÜ×#Ñ#Øpôð "�	á˜Ð0ð )Ð4¸¿¹×8VÒ8Vô $¤L¸¿¹Ô$DÄlÐZ^×ZeÑZeÔFfÔgä!¨¯©Ô5ð ð FUÐE` ×!?Ñ!?Ô!AÐfgÐà.2×.FÑ.FØ˜Ð)Ø ×'Ñ'Ø+ð /Gó /
Ñ+Ð˜|ð " QÒ&ØPZÑMÐ*Ñ,Mð
 ×6Ñ6°|ÓDñØ.Ø1à&*×&>Ñ&>×&GÑ&GÈÐWXÑHXÓ&YÐ#ð &Ð(=Ñ=ˆà×0Ñ0×7Ñ7Ðð " QÒ&Ø ×%Ñ% aÓ(¨AÒ-ð ØaóÐ-ô # 4§:¡:Ó.ö#àð " %¨!¡)Ñ,Ð/JÑJ×RÑRÐS]Ð_`ÐbcÕdð#Ðð #ð '+Ð#Ø.2Ñ+ô Z_Ð_c×_iÑ_iÓYjö#ØPUÐ! %¨!¡)Ñ,Ð/JÓJð#Ðð #ð '+×&AÑ&AÀ-ÐQ_Ó&`Ð#Ø.2×.QÑ.QÐR_ÐaoÓ.pÐ+ð "Ð-àÐ,ªQ°°dºAÐ-=Ñ>×EÑEÀaÈÏÉ×IpÑIpÐrsÐuvÓwÑwÜ—‘˜DŸJ™JÓ'×+Ñ+ñ/,Ð+ð /N×.PÑ.PÐQ^×QdÑQdÓ.eÑ+à.2Ð+äŸ	™	 = /Ð4GÑ"GÈÓKˆà×%Ò%Ø ×6Ñ6°}ÓEˆMäŸ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-Ódˆñ /C©ÈÐ%Ù/CÈÏÉ×HYÑHYÐ\]ÒH]©ÐcgÐ&á&7¡¸TÐÙ'8¡¸dÐÙ 1°d·k±k×6UÒ6U™"Ð[_ˆä"+¨D¯K©KÓ"8ò 	;ÑˆC�Ù#à-°-ÂÐCSÀOÐCSÐ@SÑ2TÐ1VÑVÐ-Ø—;‘;×$Ñ$ qÒ(Ø2°}ÂQÈÑHXÐEXÑ7YÐ6[Ñ[Ð2á)ØØ'Ø%Ø"AØ0OØ/MØ2SØ)Ø /Ø#Ø"3ôˆMð *¨!Ñ,ˆMÚ Ø%¨-¸Ñ*:Ð)<Ñ<Ð%Ø&¨=¸Ñ+;Ð*=Ñ=Ð&à—;‘;×2Ó2Ø#¨°aÑ(8Ð':Ñ:‘Oð7	;ñ:  Ø)¨mºAÐ?OÀÐ?OÐ<OÑ.PÐ-RÑRÐ)Ø�{‰{× Ñ  1Ò$Ø.°=ÂÀOÑDTÐATÑ3UÐ2WÑWÐ.ð *ª!Ð-=¨oÐ-=Ð*=Ñ>ÐØHLÏÉ×HYÑHYÐ\]ÒH] -²°?Ñ3CÐ0CÒ"DÐcgÐáÜñ ð &Ø+Ø#Ø1Ø2Ø)Ø*Ø#ð	ôó ð ô ,Ø/Ø$;Ø+Ø7Ø >Ø,Ø3Ø,ô	
ð 		
ùòu#ùò#s   È#S:ÉS?c           	      óð  — |j                   \  }}t        j                  d| j                  «      j	                  |j
                  «      j                  dd«      }t        | j                  | j                  |«      \  }}|d d …d |…d |…f   j                  |dd«      }t        j                  |d d …d |…d |…f   |d d …d |…| j                  | j                  |z   …f   gd«      j                  |dd«      }||fS rõ   )rˆ   r   r+   rs  r  r%   rU   r\   rH   rð   r1   )r�   rW   rÔ   r2   Úmain_relative_bucketsÚpredict_relative_bucketss         r"   r  z3ProphetNetDecoder.compute_buffered_relative_bucketsù  s!  € Ø&2×&8Ñ&8Ñ#ˆ
�Oä—|‘| A t×'@Ñ'@ÓA×DÑDÀ\×EXÑEXÓY×`Ñ`ÐabÐdeÓfˆÜ:]Ø×Ñ˜d×8Ñ8¸,ó;
Ñ7ÐÐ7ð
 !6²aÐ9I¸/Ð9IÐK[ÈOÐK[Ð6[Ñ \× cÑ cÐdnÐpqÐstÓ uÐÜ#(§9¡9à(ªÐ,<¨_Ð,<Ð>N¸Ð>NÐ)NÑOØ(ÚÐ'˜Ð'¨×)BÑ)BÀT×E^ÑE^ÐapÑEpÐ)pÐpñðð ó$
÷ ‰&�˜Q Ó
"ð 	!ð %Ð&>Ð>Ð>r#   c                 óL  — |j                   d d \  }}t        j                  ||ft        j                  |j                  «      j
                  |j                  |j                  ¬«      }t        j                  |d«      }|d |…d |…f   d d d d …d d …f   j                  || j                  j                  f|j                   z   «      }|�@d|d d …d d d d …f   z
  t        j                  | j                  «      j
                  z  }||z   }n|}|j                  |j                  «      S )Nr(   r&   r   rc  )rˆ   r   Úfullr)   r   r*   r%   Útriur-   r�   rñ   r  )r�   ry   r©   rÔ   Ú
seq_lengthÚcausal_maskÚextended_causal_maskrk  s           r"   r�  z(ProphetNetDecoder.prepare_attention_mask  s%  € Ø!.×!4Ñ!4°R°aÐ!8Ñˆ
�Jô —j‘jØ˜Ð$Ü�K‰K˜×+Ñ+Ó,×0Ñ0Ø×%Ñ%Ø ×'Ñ'ô	
ˆô —j‘j ¨aÓ0ˆà*¨;¨J¨;¸¸¸Ð+CÑDÀTÈ4ÒQRÒTUÐEUÑV×]Ñ]Ø˜Ÿ™×@Ñ@ÐAÀK×DUÑDUÑUó 
Ðð
 Ð%Ø'*¨^ºA¸tÀTÊ1Ð<LÑ-MÑ'MÔQV×Q\ÑQ\Ð]a×]gÑ]gÓQh×QlÑQlÑ&lÐ#Ø&:Ð=TÑ&TÑ#à&:Ð#Ø&×)Ñ)¨-×*=Ñ*=Ó>Ð>r#   c           	      ó&  — |j                   d d \  }}t        | j                  | j                  |j                  |j
                  «      }t        j                  |d d …d |…d |…f   |d d …d |…| j                  | j                  |z   …f   gd¬«      }|d d d d …d d …d d …f   j                  || j                  j                  f|j                   z   «      }|�¡d|d d …d d d d d …f   z
  t        j                  | j
                  «      j                  z  }|j                  || j                  j                  | j                  ||f«      }t        j                  |t        j                  |«      gd¬«      }||z   }n|}|j                  |j
                  «      S )Nr(   rS   r   rc  )rˆ   r>   rs  r3   r%   r   r   r1   r-   r�   rñ   r)   r*   rA   r  )	r�   ry   r©   rÔ   r”  Úpredict_causal_maskÚextended_predict_causal_maskrk  r  s	            r"   r‚  z0ProphetNetDecoder.prepare_predict_attention_mask'  sª  € Ø!.×!4Ñ!4°R°aÐ!8Ñˆ
�Jô 3Ø×%Ñ% t§z¡z°=×3GÑ3GÈ×I\ÑI\ó
Ðô $Ÿi™ià#¢A {¨
 {°K°Z°KÐ$?Ñ@Ø#Ú�{˜
�{ D×$=Ñ$=À×@YÑ@YÐ\fÑ@fÐ$fÐfñðð ô
Ðð (;¸4ÀÂqÊ!ÊQÐ;NÑ'O×'VÑ'VØ˜Ÿ™×@Ñ@ÐAÐDW×D]ÑD]Ñ]ó(
Ð$ð
 Ð%Ø'*¨^ºA¸tÀTÈ4ÒQRÐ<RÑ-SÑ'SÔW\×WbÑWbÐcg×cmÑcmÓWn×WrÑWrÑ&rÐ#Ø&=×&DÑ&DØ˜TŸ[™[×DÑDÀdÇjÁjÐR\Ð^hÐió'Ð#ô ',§i¡iØ(¬%×*:Ñ*:Ð;RÓ*SÐTÐZ\ô'Ð#ð /KÐMdÑ.dÑ+à.JÐ+Ø.×1Ñ1°-×2EÑ2EÓFÐFr#   )
NNNNNNNNNN)rl   rm   rn   r   r›   rX  r]  r   r   r   r	   râ   rr   rx   r§   r  r�  r‚  r®   r¯   s   @r"   rq  rq    s  ø„ ðÐ/õ ò,$ò%ð ð *.Ø.2Ø59Ø6:Ø(,Ø-1Ø!%Ø)-Ø,0Ø#'ñy
à—<‘< $Ñ&ðy
ð Ÿ™ tÑ+ðy
ð  %Ÿ|™|¨dÑ2ð	y
ð
 !&§¡¨tÑ 3ðy
ð  ™ðy
ð —|‘| dÑ*ðy
ð ˜$‘;ðy
ð   $™;ðy
ð # T™kðy
ð ˜D‘[ðy
ð 
Ð-Ñ	-òy
ó ðy
òv?ò,?ö0!Gr#   rq  c                   ó`  ‡ — e Zd ZdddœZdefˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 	 	 	 dde	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  dedz  dedz  de	j                  dz  de	j                  dz  dedz  dedz  dedz  dedz  deez  fd„«       Zˆ xZS )ÚProphetNetModelúword_embeddings.weight)zencoder.word_embeddings.weightúdecoder.word_embeddings.weightr�   c                 ój  •— t         ‰| �  |«       t        j                  |j                  |j
                  |j                  ¬«      | _        t        j                  |«      }d|_
        t        |«      | _        t        j                  |«      }d|_        t        |«      | _        | j!                  «        y )NrJ  FT)rš   r›   r   rK  rL  rœ   r†   rM  ÚcopyÚdeepcopyr@  rH  ÚencoderÚ
is_decoderrq  ÚdecoderrT  )r�   r�   Úencoder_configÚdecoder_configrž   s       €r"   r›   zProphetNetModel.__init__R  sˆ   ø€ Ü‰Ñ˜Ô Ü!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔäŸ™ vÓ.ˆØ#(ˆÔ Ü(¨Ó8ˆŒäŸ™ vÓ.ˆØ$(ˆÔ!Ü(¨Ó8ˆŒð 	�‰Õr#   c                 ó   — | j                   S r—   rW  rú   s    r"   rX  z$ProphetNetModel.get_input_embeddingsa  rY  r#   c                 ó~   — || _         | j                   | j                  _         | j                   | j                  _         y r—   )rM  r¡  r£  r[  s     r"   r]  z$ProphetNetModel.set_input_embeddingsd  s.   € Ø$ˆÔØ'+×';Ñ';ˆ�‰Ô$Ø'+×';Ñ';ˆ�‰Õ$r#   NrŽ   r©   Údecoder_input_idsÚdecoder_attention_maskÚencoder_outputsrc   r_  Údecoder_inputs_embedsr@  r¾   r`  ra  r•   c                 óV  — |	�|	n| j                   j                  }	|
�|
n| j                   j                  }
|�|n| j                   j                  }|�|n| j                   j                  }|€| j                  ||||
||¬«      }| j                  |||d   ||||
||	|¬«
      }|s||z   S t        |j                  |j                  |j                  |j                  |j                  |j                  |j                  |j                  |j                  |j                  |j                  ¬«      S )añ  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            ProphetNet uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If
            `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
            `past_key_values`).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, ProphetNetModel

        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased")
        >>> model = ProphetNetModel.from_pretrained("microsoft/prophetnet-large-uncased")

        >>> input_ids = tokenizer(
        ...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids  # Batch size 1
        >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)

        >>> last_hidden_states = outputs.last_hidden_state  # main stream hidden states
        >>> last_hidden_states_ngram = outputs.last_hidden_state_ngram  # predict hidden states
        ```)rŽ   r©   r_  r¾   r`  ra  r   )
rŽ   r©   rj   ry  rc   r_  r¾   r`  r@  ra  )ru   rv   rc   rd   re   rf   rg   rh   ri   rj   rk   )r�   r@  r¾   r`  ra  r¡  r£  rt   ru   rv   rc   ry   rz   r{   r|   rh   )r�   rŽ   r©   r¨  r©  rª  rc   r_  r«  r@  r¾   r`  ra  rÓ   Údecoder_outputss                  r"   r§   zProphetNetModel.forwardi  sK  € ðd "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø1BÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐ"Ø"Ÿl™lØ#Ø-Ø+Ø"3Ø%9Ø'ð +ó ˆOð Ÿ,™,Ø'Ø1Ø"1°!Ñ"4Ø#1Ø+Ø/Ø/Ø!5ØØ#ð 'ó 
ˆñ Ø" _Ñ4Ð4Ü+Ø-×?Ñ?Ø$3×$KÑ$KØ+×;Ñ;Ø"1×"?Ñ"?Ø(7×(KÑ(KØ.×9Ñ9Ø%4×%EÑ%EØ,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô
ð 	
r#   )NNNNNNNNNNNN)rl   rm   rn   Ú_tied_weights_keysr   r›   rX  r]  r   r   r   Ú
BoolTensorrr   r	   râ   rt   r§   r®   r¯   s   @r"   r›  r›  K  sJ  ø„ ð +CØ*BñÐð
Ð/õ ò$ò<ð
 ð *.Ø.2Ø15Ø:>Ø(,Ø(,Ø-1Ø59Ø!%Ø)-Ø,0Ø#'ñ^
à—<‘< $Ñ&ð^
ð Ÿ™ tÑ+ð^
ð !Ÿ<™<¨$Ñ.ð	^
ð
 !&× 0Ñ 0°4Ñ 7ð^
ð  ™ð^
ð  ™ð^
ð —|‘| dÑ*ð^
ð  %Ÿ|™|¨dÑ2ð^
ð ˜$‘;ð^
ð   $™;ð^
ð # T™kð^
ð ˜D‘[ð^
ð 
Ð-Ñ	-ò^
ó ô^
r#   r›  zh
    The ProphetNet Model with a language modeling head. Can be used for sequence generation tasks.
    c                   óÀ  ‡ — e Zd ZddiZdefˆ fd„Zd„ Ze	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dedz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  dedz  deez  fd„«       Zdd„Zdej                  fd„Zdˆ fd„	Zˆ xZS )Ú"ProphetNetForConditionalGenerationúlm_head.weightú!prophetnet.word_embeddings.weightr�   c                 ó
  •— t         ‰| �  |«       t        |«      | _        |j                  | _        |j                  | _        t        j                  |j                  |j                  d¬«      | _        | j                  «        y )NF©Úbias)rš   r›   r›  r‚   r†   r    Údisable_ngram_lossr   r¸   rœ   rL  Úlm_headrT  r�   s     €r"   r›   z+ProphetNetForConditionalGeneration.__init__Õ  sd   ø€ Ü‰Ñ˜Ô Ü)¨&Ó1ˆŒØ!×.Ñ.ˆÔØ"(×";Ñ";ˆÔä—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Õr#   c                 ó.   — | j                   j                  S r—   )r‚   rM  rú   s    r"   rX  z7ProphetNetForConditionalGeneration.get_input_embeddingsà  s   € Ø�‰×.Ñ.Ð.r#   NrŽ   r©   r¨  r©  rª  rc   r_  r«  Úlabelsr@  r¾   r`  ra  r•   c                 óZ  — |�|n| j                   j                  }|	�|€|€| j                  |	«      }| j                  |||||||||
|||¬«      }|�|j                  n|j                  dd \  }}|d   j                  || j                   j                  |d«      }| j                  |«      }|dd…df   }| j                   j                  dkD  r|dd…dd…f   nd}|j                  «       s|j                  «       }d}|	�| j                  ||	«      }|s*t        d„ ||fD «       «      }|�|f|z   |dd z   S ||dd z   S t        ||||j                  |j                  |j                  |j                   |j"                  |j$                  |j&                  |j(                  |j*                  ¬«      S )	a…  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            ProphetNet uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If
            `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
            `past_key_values`).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
            config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
            labels in `[0, ..., config.vocab_size]`

        Example:

        ```python
        >>> from transformers import AutoTokenizer, ProphetNetForConditionalGeneration

        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased")
        >>> model = ProphetNetForConditionalGeneration.from_pretrained("microsoft/prophetnet-large-uncased")

        >>> input_ids = tokenizer(
        ...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids  # Batch size 1
        >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)

        >>> logits_next_token = outputs.logits  # logits to predict next token as usual
        >>> logits_ngram_next_tokens = outputs.logits_ngram  # logits to predict 2nd, 3rd, ... next tokens
        ```N)rŽ   r©   r¨  r©  rª  rc   r_  r«  r@  r¾   r`  ra  r(   r   rS   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr—   r'   re  s     r"   rh  z=ProphetNetForConditionalGeneration.forward.<locals>.<genexpr>A  ó   è ø€ ÒR QÀAÁMœqÑRùri  )r`   ra   rb   rc   rd   re   rf   rg   rh   ri   rj   rk   )r�   ra  r�   r‚   rˆ   r,   r3   r¸  Úis_contiguousrö   Ú_compute_lossrr   r_   rc   rd   re   rf   rg   rh   ri   rj   rk   )r�   rŽ   r©   r¨  r©  rª  rc   r_  r«  rº  r@  r¾   r`  ra  rÓ   r5  rÔ   r2   Úpredicting_streamsÚpredict_logitsra   rb   r`   Ú
all_logitss                           r"   r§   z*ProphetNetForConditionalGeneration.forwardã  sð  € ðn &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐÐ"3Ð";Ð@UÐ@]à $× 1Ñ 1°&Ó 9Ðà—/‘/ØØ)Ø/Ø#9Ø+Ø+Ø'Ø"7ØØ/Ø!5Ø#ð "ó 
ˆð (9Ð'DÐ×#Ò#ÐJ_×JeÑJeÐfhÐghÐJiñ 	$ˆ
�Oð % Q™ZŸ_™_¨Z¸¿¹×9JÑ9JÈOÐ]_Ó`ÐØŸ™Ð&8Ó9ˆà¢ 1 Ñ%ˆØ04·±×0AÑ0AÀAÒ0E�~¢a¨© eÒ,È4ˆð ×#Ñ#Ô%Ø×&Ñ&Ó(ˆFàˆØÐØ×%Ñ% n°fÓ=ˆDáÜÑR¨6°<Ð*@ÔRÓRˆJØ9=Ð9I�D�7˜ZÑ'¨'°!°"¨+Ñ5ÐgÈzÐ\cÐdeÐdfÐ\gÑOgÐgä,ØØØ)Ø '× 7Ñ 7Ø&-×&CÑ&CØ,3×,OÑ,OØ#*×#=Ñ#=Ø)0×)IÑ)IØ!(×!9Ñ!9Ø*1×*KÑ*KØ&-×&CÑ&CØ#*×#=Ñ#=ôð r#   c                 óÌ  — |j                  | j                  j                  |j                  d«      |j                  d«      «      j	                  |«      }t        | j                  j                  «      D ]!  }|dkD  r| j                  r n|||d d …d d …f<   Œ# |j                  dd«      j                  «       }t        j                  j                  |j                  d|j                  d«      «      dt        j                  ¬«      }t        j                  j                  ||j                  d«      d¬«      }| j                  j                   dkD  r“|j#                  dd¬	«       }|j%                  |«      j                  d«      }	||	   }|j'                  «       }| j                  j                   |j                  d«      z  }
d
| j                  j                   z
  |z  |
|z  z   }|S ©Nr   r   rS   r   Úmean)Ú	reductiong        T)r   Úkeepdimrc  ©r‡   r�   r3   rV   Úfill_rQ  r·  rÎ   rö   r   r   Úlog_softmaxr,   r   r   Únll_lossÚepsÚsumÚnerÅ  ©r�   ra   rº  Úignore_indexÚexpend_targetsrv  Úlprobsr`   Úsmooth_lossÚnon_masked_tokensÚeps_is              r"   r¿  z0ProphetNetForConditionalGeneration._compute_lossS  ó–  € Ø×)Ñ)¨$¯+©+×*;Ñ*;¸V¿[¹[È»^ÈVÏ[É[ÐYZË^Ó\×bÑbÐcoÓpˆä�t—{‘{×(Ñ(Ó)ò 	-ˆAØ�1Šu˜×0Ò0ÙØ&,ˆN˜1ša¢˜7Ò#ð	-ð
 ×!Ñ! ! QÓ'×2Ñ2Ó4ˆÜ—‘×*Ñ*Ø�K‰K˜˜FŸK™K¨›OÓ,ØÜ—-‘-ð +ó 
ˆô �}‰}×%Ñ% f¨n×.AÑ.AÀ"Ó.EÐQWÐ%ÓXˆà�;‰;�?‰?˜SÒ Ø!Ÿ:™:¨"°d˜:Ó;Ð;ˆKØ .× 1Ñ 1°,Ó ?× DÑ DÀRÓ HÐØ%Ð&7Ñ8ˆKØ%×*Ñ*Ó,ˆKà—K‘K—O‘O f§k¡k°"£oÑ5ˆEØ˜$Ÿ+™+Ÿ/™/Ñ)¨TÑ1°E¸KÑ4GÑGˆDàˆr#   c                 ó$   — | j                  |«      S r—   )r�   )r�   rº  s     r"   Ú%prepare_decoder_input_ids_from_labelszHProphetNetForConditionalGeneration.prepare_decoder_input_ids_from_labelso  s   € Ø× Ñ  Ó(Ð(r#   c                 óT   •— |€| j                   j                  S t        ‰| �  |¬«      S )N)Úmodality)r‚   r¡  rš   Úget_encoder)r�   rÚ  rž   s     €r"   rÛ  z.ProphetNetForConditionalGeneration.get_encoderr  s,   ø€ ØÐØ—?‘?×*Ñ*Ð*ä‘7Ñ&°Ð&Ó9Ð9r#   )NNNNNNNNNNNNN©r„   r—   )rl   rm   rn   r®  r   r›   rX  r   r   r   r¯  r	   râ   rr   r_   r§   r¿  rØ  rÛ  r®   r¯   s   @r"   r±  r±  Ë  s€  ø„ ð 	Ð=ðÐð	Ð/õ 	ò/ð ð *.Ø.2Ø15Ø:>Ø/3Ø(,Ø-1Ø59Ø&*Ø!%Ø)-Ø,0Ø#'ñmà—<‘< $Ñ&ðmð Ÿ™ tÑ+ðmð !Ÿ<™<¨$Ñ.ð	mð
 !&× 0Ñ 0°4Ñ 7ðmð Ÿ™¨Ñ,ðmð  ™ðmð —|‘| dÑ*ðmð  %Ÿ|™|¨dÑ2ðmð —‘˜tÑ#ðmð ˜$‘;ðmð   $™;ðmð # T™kðmð ˜D‘[ðmð  
Ð*Ñ	*ò!mó ðmó^ð8)¸E¿L¹Ló )÷:ñ :r#   r±  zt
    The standalone decoder part of the ProphetNetModel with a lm head on top. The model can be used for causal
    c                   ó\  ‡ — e Zd ZdddœZdefˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 	 	 dde	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  dedz  de	j                  dz  de	j                  dz  dedz  dedz  dedz  dedz  deez  fd„«       Zdd„Zˆ xZS )ÚProphetNetForCausalLMr³  )r²  z)prophetnet.decoder.word_embeddings.weightr�   c                 óP  •— t        j                  |«      }d|_        d|_        t        ‰| �  |«       t        |«      | _        |j                  | _	        |j                  | _
        t        j                  |j                  |j                  d¬«      | _        | j!                  «        y )NTFrµ  )rŸ  r   r¢  r~  rš   r›   ÚProphetNetDecoderWrapperr‚   r†   r    r·  r   r¸   rœ   rL  r¸  rT  r�   s     €r"   r›   zProphetNetForCausalLM.__init__„  s‚   ø€ ä—‘˜vÓ&ˆØ ˆÔØ$)ˆÔ!Ü‰Ñ˜Ô Ü2°6Ó:ˆŒà!×.Ñ.ˆÔØ"(×";Ñ";ˆÔä—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Õr#   c                 óB   — | j                   j                  j                  S r—   ©r‚   r£  rM  rú   s    r"   rX  z*ProphetNetForCausalLM.get_input_embeddings”  s   € Ø�‰×&Ñ&×6Ñ6Ð6r#   c                 ó:   — || j                   j                  _        y r—   râ  r[  s     r"   r]  z*ProphetNetForCausalLM.set_input_embeddings—  s   € Ø27ˆ�‰×ÑÕ/r#   NrŽ   r©   rj   ry  rc   r_  rº  r@  r¾   r`  ra  r•   c                 óº  — |�|n| j                   j                  }| j                  j                  ||||||||	|
|¬«
      }|�|j                  n|j                  dd \  }}|d   j                  || j                   j                  |d«      }| j                  |«      }|dd…df   }| j                   j                  dkD  r|dd…dd…f   nd}d}|�| j                  ||«      }|s*t        d„ ||fD «       «      }|�|f|z   |dd z   S ||dd z   S t        ||||j                  |j                  |j                  |j                  |j                  |j                   ¬«	      S )	aª  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]`

        Example:

        ```python
        >>> from transformers import AutoTokenizer, ProphetNetForCausalLM
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased")
        >>> model = ProphetNetForCausalLM.from_pretrained("microsoft/prophetnet-large-uncased")
        >>> assert model.config.is_decoder, f"{model.__class__} has to be configured as a decoder."
        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> logits = outputs.logits

        >>> # Model can also be used with EncoderDecoder framework
        >>> from transformers import BertTokenizer, EncoderDecoderModel, AutoTokenizer
        >>> import torch

        >>> tokenizer_enc = BertTokenizer.from_pretrained("google-bert/bert-large-uncased")
        >>> tokenizer_dec = AutoTokenizer.from_pretrained("microsoft/prophetnet-large-uncased")
        >>> model = EncoderDecoderModel.from_encoder_decoder_pretrained(
        ...     "google-bert/bert-large-uncased", "microsoft/prophetnet-large-uncased"
        ... )

        >>> ARTICLE = (
        ...     "the us state department said wednesday it had received no "
        ...     "formal word from bolivia that it was expelling the us ambassador there "
        ...     "but said the charges made against him are `` baseless ."
        ... )
        >>> input_ids = tokenizer_enc(ARTICLE, return_tensors="pt").input_ids
        >>> labels = tokenizer_dec(
        ...     "us rejects charges against its ambassador in bolivia", return_tensors="pt"
        ... ).input_ids
        >>> outputs = model(input_ids=input_ids, decoder_input_ids=labels[:, :-1], labels=labels[:, 1:])

        >>> loss = outputs.loss
        ```N)
rŽ   r©   rj   ry  rc   r_  r@  r¾   r`  ra  r(   r   rS   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr—   r'   re  s     r"   rh  z0ProphetNetForCausalLM.forward.<locals>.<genexpr>ò  r½  ri  )	r`   ra   rb   rc   ry   rz   r{   r|   rh   )r�   ra  r‚   r£  rˆ   r,   r3   r¸  r¿  rr   r~   rc   ry   rz   r{   r|   rh   )r�   rŽ   r©   rj   ry  rc   r_  rº  r@  r¾   r`  ra  rÓ   r5  rÔ   r2   rÀ  rÁ  ra   rb   r`   rÂ  s                         r"   r§   zProphetNetForCausalLM.forwardš  s’  € ðv &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆð —/‘/×)Ñ)ØØ)Ø"7Ø#9Ø+Ø'ØØ/Ø!5Ø#ð *ó 
ˆð :CÐ9N i§o¢oÐTa×TgÑTgÐhjÐijÐTkÑ#ˆ
�Oà$ Q™ZŸ_™_¨Z¸¿¹×9JÑ9JÈOÐ]_Ó`ÐØŸ™Ð&8Ó9ˆà¢ 1 Ñ%ˆØ04·±×0AÑ0AÀAÒ0E�~¢a¨© eÒ,È4ˆàˆØÐØ×%Ñ% n°fÓ=ˆDáÜÑR¨6°<Ð*@ÔRÓRˆJØ9=Ð9I�D�7˜ZÑ'¨'°!°"¨+Ñ5ÐgÈzÐ\cÐdeÐdfÐ\gÑOgÐgä,ØØØ)Ø '× 7Ñ 7Ø%×3Ñ3Ø$+×$?Ñ$?Ø"×-Ñ-Ø!(×!9Ñ!9Ø!(×!9Ñ!9ô
ð 
r#   c                 óÌ  — |j                  | j                  j                  |j                  d«      |j                  d«      «      j	                  |«      }t        | j                  j                  «      D ]!  }|dkD  r| j                  r n|||d d …d d …f<   Œ# |j                  dd«      j                  «       }t        j                  j                  |j                  d|j                  d«      «      dt        j                  ¬«      }t        j                  j                  ||j                  d«      d¬«      }| j                  j                   dkD  r“|j#                  dd¬	«       }|j%                  |«      j                  d«      }	||	   }|j'                  «       }| j                  j                   |j                  d«      z  }
d
| j                  j                   z
  |z  |
|z  z   }|S rÄ  rÈ  rÏ  s              r"   r¿  z#ProphetNetForCausalLM._compute_loss  rÖ  r#   )NNNNNNNNNNNrÜ  )rl   rm   rn   r®  r   r›   rX  r]  r   r   r   r	   râ   rr   r~   r§   r¿  r®   r¯   s   @r"   rÞ  rÞ  y  s;  ø„ ð >Ø5XñÐð
Ð/õ ò 7ò8ð ð *.Ø.2Ø59Ø6:Ø(,Ø-1Ø&*Ø!%Ø)-Ø,0Ø#'ñdà—<‘< $Ñ&ðdð Ÿ™ tÑ+ðdð  %Ÿ|™|¨dÑ2ð	dð
 !&§¡¨tÑ 3ðdð  ™ðdð —|‘| dÑ*ðdð —‘˜tÑ#ðdð ˜$‘;ðdð   $™;ðdð # T™kðdð ˜D‘[ðdð 
Ð*Ñ	*òdó ðd÷Lr#   rÞ  c                   ó6   ‡ — e Zd ZdZddiZdefˆ fd„Zd„ Zˆ xZS )rà  z„
    This is a wrapper class, so that [`ProphetNetForCausalLM`] can correctly be loaded from pretrained prophetnet
    classes.
    r�  rœ  r�   c                 óÚ   •— t         ‰| �  |«       t        j                  |j                  |j
                  |j                  ¬«      | _        t        |«      | _	        | j                  «        y )NrJ  )rš   r›   r   rK  rL  rœ   r†   rM  rq  r£  rT  r�   s     €r"   r›   z!ProphetNetDecoderWrapper.__init__(  sP   ø€ Ü‰Ñ˜Ô ä!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ(¨Ó0ˆŒð 	�‰Õr#   c                 ó&   —  | j                   |i |¤ŽS r—   )r£  )r�   ÚargsrÓ   s      r"   r§   z ProphetNetDecoderWrapper.forward1  s   € Øˆt�|‰|˜TÐ, VÑ,Ð,r#   )	rl   rm   rn   ro   r®  r   r›   r§   r®   r¯   s   @r"   rà  rà    s*   ø„ ñð 	)Ð*BðÐðÐ/õ ö-r#   rà  )rq  rH  rÞ  r±  r›  r€   r6  ):ro   rŸ  rF   Údataclassesr   r   r   r   Útorch.nnr   Úactivationsr   Úcache_utilsr	   r
   r   Ú
generationr   Úmodeling_layersr   Úmodeling_outputsr   Úmodeling_utilsr   Úutilsr   r   r   r   Úconfiguration_prophetnetr   Ú
get_loggerrl   r|  r   r>   rQ   r\   r_   rt   rx   r~   r€   rK  r”   ÚModuler±   rä   rî   r*  r9  rH  rq  r›  r±  rÞ  rà  Ú__all__r'   r#   r"   ú<module>rø     sˆ  ðñ Yã Û Ý !ã ß Ý å !ß CÑ CÝ )Ý 9Ý /Ý -ß QÓ QÝ 6ð 
ˆ×	Ñ	˜HÓ	%€óQò7ó, ò6Mñ. ðôð
 ô*? ó *?ó óð*?ñZ ðôð ô(? ;ó (?ó óð(?ñV ðôð
 ô#= ;ó #=ó óð#=ñL ðôð
 ô+= ó +=ó óð+=ð\ ô! ó !ó ð!ô8(- R§\¡\ô (-ôVs2˜"Ÿ)™)ô s2ôl˜BŸI™Iô ô.o/ 2§9¡9ô o/ôd	&Ð7ô &ôRDÐ7ô DñN ðôô
c
Ð1ó c
óð
c
ñL ðôô
hGÐ1ó hGóð
hGðV	 ô|
Ð/ó |
ó ð|
ñ~ ðôô
f:Ð)BÀOó f:óð
f:ñR ðôô
]Ð5°ó ]óð
]ô@-Ð8ô -ò.�r#   