Ë
    ÜÓ^j¾  ã                   óü  — d dl Z d dlmZ d dlZd dlmZ d dlmZmZmZ ddl	m
Z ddlmZ ddlmZmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZ ddlmZmZmZmZm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1m2Z2 ddl3m4Z4  e,jj                  e6«      Z7 G d„ dejp                  «      Z9e) G d„ de$«      «       Z: G d„ dejp                  «      Z;	 	 dBdejx                  dejz                  dejz                  dejz                  dejz                  dz  d e>dz  d!e>d"e&e(   fd#„Z? G d$„ d%ejx                  «      Z@ G d&„ d'e«      ZA G d(„ d)e:«      ZB G d*„ d+e«      ZC G d,„ d-e:«      ZDd.ejz                  d/eEfd0„ZFe) G d1„ d2e:«      «       ZG e)d3¬4«       G d5„ d6e:e«      «       ZH G d7„ d8ejx                  «      ZI e)d9¬4«       G d:„ d;e:«      «       ZJ G d<„ d=e:«      ZK e)d>¬4«       G d?„ d@e:e«      «       ZLg dA¢ZMy)Cé    N)ÚCallable)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutputÚSeq2SeqSequenceClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚis_torchdynamo_compilingÚloggingÚtorch_compilable_check)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚPLBartConfigc            
       ó`   ‡ — e Zd ZdZd
dededededz  fˆ fd„Zdej                  fˆ fd	„Z	ˆ xZ
S )ÚPLBartScaledWordEmbeddingz\
    This module overrides nn.Embeddings' forward by multiplying with embeddings scale.
    Únum_embeddingsÚembedding_dimÚpadding_idxÚembed_scaleNc                 ó6   •— t         ‰| �  |||«       || _        y ©N)ÚsuperÚ__init__r,   )Úselfr)   r*   r+   r,   Ú	__class__s        €úu/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/plbart/modeling_plbart.pyr0   z"PLBartScaledWordEmbedding.__init__B   s   ø€ Ü‰Ñ˜¨¸ÔDØ&ˆÕó    Ú	input_idsc                 ó<   •— t         ‰| �  |«      | j                  z  S r.   )r/   Úforwardr,   )r1   r5   r2   s     €r3   r7   z!PLBartScaledWordEmbedding.forwardF   s   ø€ Ü‰w‰˜yÓ)¨D×,<Ñ,<Ñ<Ð<r4   )ç      ð?©Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚintÚfloatr0   ÚtorchÚTensorr7   Ú__classcell__©r2   s   @r3   r(   r(   =   sE   ø„ ññ' sð '¸3ð 'ÈSð 'Ð_dÐgkÑ_kõ 'ð= §¡÷ =ñ =r4   r(   c                   óF   ‡ — e Zd ZU eed<   dZdZddgZdZdZ	dZ
ˆ fd„Zˆ xZS )ÚPLBartPreTrainedModelÚconfigÚmodelTÚPLBartDecoderLayerÚPLBartEncoderLayerc                 ó„   •— t         ‰| �  |«       t        |t        «      r t	        j
                  |j                  «       y y r.   )r/   Ú_init_weightsÚ
isinstanceÚPLBartForConditionalGenerationÚinitÚzeros_Úfinal_logits_bias)r1   Úmoduler2   s     €r3   rK   z#PLBartPreTrainedModel._init_weightsT   s2   ø€ Ü‰Ñ˜fÔ%Ü�fÔ<Ô=Ü�K‰K˜×0Ñ0Õ1ð >r4   )r:   r;   r<   r&   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnrK   rB   rC   s   @r3   rE   rE   J   s<   ø… àÓØÐØ&*Ð#Ø-Ð/CÐDÐØÐØ€NØÐ÷2ð 2r4   rE   c                   óv   ‡ — e Zd ZdZdedefˆ fd„Z	 d
dej                  dedej                  dz  fˆ fd	„Zˆ xZ	S )Ú PLBartLearnedPositionalEmbeddingzN
    This module learns positional embeddings up to a fixed maximum size.
    r)   r*   c                 óN   •— d| _         t        ‰| �	  || j                   z   |«       y )Né   )Úoffsetr/   r0   )r1   r)   r*   r2   s      €r3   r0   z)PLBartLearnedPositionalEmbedding.__init___   s$   ø€ ð ˆŒÜ‰Ñ˜¨$¯+©+Ñ5°}ÕEr4   Nr5   Úpast_key_values_lengthÚposition_idsc                 ó$  •— |€a|j                   dd \  }}t        j                  |||z   t        j                  | j                  j
                  ¬«      j                  |d«      }n|j                  d«      }t        ‰| �%  || j                  z   «      S )z3`input_ids' shape is expected to be [bsz x seqlen].Nr\   )ÚdtypeÚdeviceéÿÿÿÿr   )Úshaper@   ÚarangeÚlongÚweightrb   ÚexpandÚ	unsqueezer/   r7   r]   )r1   r5   r^   r_   ÚbszÚseq_lenr2   s         €r3   r7   z(PLBartLearnedPositionalEmbedding.forwarde   sˆ   ø€ ð
 ÐØ$Ÿ?™?¨2¨AÐ.‰LˆC�Ü Ÿ<™<Ø&Ð(>ÀÑ(HÔPU×PZÑPZÐcg×cnÑcn×cuÑcuôç‰f�S˜"‹oñ ð (×1Ñ1°!Ó4ˆLä‰w‰˜|¨d¯k©kÑ9Ó:Ð:r4   )r   N)
r:   r;   r<   r=   r>   r0   r@   rA   r7   rB   rC   s   @r3   rZ   rZ   Z   sW   ø„ ñðF sð F¸3õ Fð mqñ;ØŸ™ð;Ø?Bð;ØV[×VbÑVbÐeiÑVi÷;ñ ;r4   rZ   rQ   ÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 óœ  — |€|j                  d«      dz  }t        j                  ||j                  dd«      «      |z  }|�||z   }t        j
                  j                  |d¬«      }t        j
                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )Nrc   ç      à¿r\   r   ©Údim©ÚpÚtrainingr%   )
Úsizer@   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxrq   ry   Ú
contiguous)
rQ   rl   rm   rn   ro   rp   rq   rr   Úattn_weightsÚattn_outputs
             r3   Úeager_attention_forwardr‚   u   s»   € ð €Ø—*‘*˜R“. DÑ(ˆô —<‘<  s§}¡}°Q¸Ó':Ó;¸gÑE€LàÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r4   c                   ó  ‡ — e Zd ZdZ	 	 	 	 	 	 ddedededededed	edz  d
edz  fˆ fd„Z	 	 	 dde	j                  de	j                  dz  dedz  de	j                  dz  dee   dee	j                  e	j                  dz  f   fd„Zˆ xZS )ÚPLBartAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNÚ	embed_dimÚ	num_headsrq   Ú
is_decoderÚbiasÚ	is_causalrF   Ú	layer_idxc	                 óŽ  •— t         ‰	| �  «        || _        || _        || _        ||z  | _        || _        | j
                  |z  | j                  k7  rt        d| j                  › d|› d�«      ‚| j
                  dz  | _        || _	        || _
        || _        |€9| j                  r-t        j                  d| j                  j                  › d�«       t!        j"                  |||¬«      | _        t!        j"                  |||¬«      | _        t!        j"                  |||¬«      | _        t!        j"                  |||¬«      | _        y )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).rt   zInstantiating a decoder z¸ without passing `layer_idx` is not recommended and will lead to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.©rˆ   )r/   r0   r…   r†   rq   Úhead_dimrF   Ú
ValueErrorrp   r‡   r‰   rŠ   ÚloggerÚwarning_oncer2   r:   r   ÚLinearÚk_projÚv_projÚq_projÚout_proj)
r1   r…   r†   rq   r‡   rˆ   r‰   rF   rŠ   r2   s
            €r3   r0   zPLBartAttention.__init__”   s$  ø€ ô 	‰ÑÔØ"ˆŒØ"ˆŒØˆŒØ! YÑ.ˆŒØˆŒà�M‰M˜IÑ%¨$¯.©.Ò8ÜØMÈdÏnÉnÐM]Ø$ Y K¨rð3óð ð —}‘} dÑ*ˆŒØ$ˆŒØ"ˆŒØ"ˆŒØÐ §¢Ü×ÑØ*¨4¯>©>×+BÑ+BÐ*Cð D,ð ,ôô —i‘i 	¨9¸4Ô@ˆŒÜ—i‘i 	¨9¸4Ô@ˆŒÜ—i‘i 	¨9¸4Ô@ˆŒÜŸ	™	 )¨Y¸TÔBˆ�r4   Úhidden_statesÚkey_value_statesÚpast_key_valuesro   rr   Úreturnc                 óØ  — |du}|j                   dd }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }	d}
|�St        |t        «      rA|j                  j                  | j                  «      }
|r|j                  }n|j                  }n|}|r|n|}|rK|�I|
rGj                  | j                     j                  }|j                  | j                     j                  }nÑ| j                  |«      }| j!                  |«      }g |j                   dd ¢d‘| j                  ‘­}|j                  |«      j	                  dd«      }|j                  |«      j	                  dd«      }|�Kj#                  ||| j                  «      \  }}|r)t        |t        «      rd|j                  | j                  <   t%        j&                  | j(                  j*                  t,        «      } || |	|||f| j.                  sdn| j0                  | j2                  dœ|¤Ž\  }} |j4                  g |¢d‘­Ž j7                  «       }| j9                  |«      }||fS )	z#Input shape: Batch x Time x ChannelNrc   r%   r\   FTç        )rq   rp   )rd   r�   r”   Úviewr|   rL   r   Ú
is_updatedÚgetrŠ   Úcross_attention_cacheÚself_attention_cacheÚlayersÚkeysÚvaluesr’   r“   Úupdater   Úget_interfacerF   Ú_attn_implementationr‚   ry   rq   rp   Úreshaper   r•   )r1   r–   r—   r˜   ro   rr   Úis_cross_attentionÚinput_shapeÚhidden_shapeÚquery_statesr�   Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚkv_shapeÚattention_interfacer�   r€   s                      r3   r7   zPLBartAttention.forward»   sd  € ð .°TÐ9Ðð $×)Ñ)¨#¨2Ð.ˆà8˜Ð8 bÐ8¨$¯-©-Ñ8ˆð —{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆàˆ
ØÐ&Ü˜/Ô+>Ô?Ø,×7Ñ7×;Ñ;¸D¿N¹NÓK�
Ù%à+:×+PÑ+PÑ(à+:×+OÑ+OÑ(à'6Ð$á-?Ñ)À]ˆÙ /Ð"=Á*à-×4Ñ4°T·^±^ÑD×IÑIˆJØ/×6Ñ6°t·~±~ÑF×MÑM‰LàŸ™ ^Ó4ˆJØŸ;™; ~Ó6ˆLØF˜×-Ñ-¨c¨rÐ2ÐF°BÐF¸¿¹ÑFˆHØ#Ÿ™¨Ó2×<Ñ<¸QÀÓBˆJØ'×,Ñ,¨XÓ6×@Ñ@ÀÀAÓFˆLàÐ*Ø+?×+FÑ+FÀzÐS_Ðae×aoÑaoÓ+pÑ(�
˜Lá%¬*°_ÔFYÔ*ZØAE�O×.Ñ.¨t¯~©~Ñ>ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$·,±,Ø—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—m‘m KÓ0ˆà˜LÐ(Ð(r4   )r›   FTFNN©NNN)r:   r;   r<   r=   r>   r?   Úboolr&   r0   r@   rA   r   r   r   Útupler7   rB   rC   s   @r3   r„   r„   ‘   s  ø„ ÙGð Ø ØØØ&*Ø $ñ%Càð%Cð ð%Cð ð	%Cð
 ð%Cð ð%Cð ð%Cð ˜tÑ#ð%Cð ˜‘:õ%CðT 15Ø(,Ø.2ñH)à—|‘|ðH)ð  Ÿ,™,¨Ñ-ðH)ð  ™ð	H)ð
 Ÿ™ tÑ+ðH)ð Ð-Ñ.ðH)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷H)r4   r„   c                   óŠ   ‡ — e Zd Zd
dededz  fˆ fd„Zdej                  dej                  dee	   dej                  fd	„Zˆ xZS )rI   NrF   rŠ   c                 ój  •— t         ‰| �  «        |j                  | _        t	        | j                  |j
                  |j                  ||¬«      | _        t        j                  | j                  «      | _
        |j                  | _        t        |j                     | _        |j                  | _        t        j                   | j                  |j"                  «      | _        t        j                   |j"                  | j                  «      | _        t        j                  | j                  «      | _        y )N)r…   r†   rq   rF   rŠ   )r/   r0   Úd_modelr…   r„   Úencoder_attention_headsÚattention_dropoutÚ	self_attnr   Ú	LayerNormÚself_attn_layer_normrq   r
   Úactivation_functionÚactivation_fnÚactivation_dropoutr‘   Úencoder_ffn_dimÚfc1Úfc2Úfinal_layer_norm©r1   rF   rŠ   r2   s      €r3   r0   zPLBartEncoderLayer.__init__  sÔ   ø€ Ü‰ÑÔØŸ™ˆŒä(Ø—n‘nØ×4Ñ4Ø×,Ñ,ØØô
ˆŒô %'§L¡L°·±Ó$@ˆÔ!Ø—~‘~ˆŒÜ# F×$>Ñ$>Ñ?ˆÔØ"(×";Ñ";ˆÔÜ—9‘9˜TŸ^™^¨V×-CÑ-CÓDˆŒÜ—9‘9˜V×3Ñ3°T·^±^ÓDˆŒÜ "§¡¨T¯^©^Ó <ˆÕr4   r–   ro   rr   r™   c                 óF  — |} | j                   |fd|i|¤Ž\  }}t        j                  j                  || j                  | j                  ¬«      }||z   }| j                  |«      }|}| j                  | j                  |«      «      }t        j                  j                  || j                  | j                  ¬«      }| j                  |«      }t        j                  j                  || j                  | j                  ¬«      }||z   }| j                  |«      }|j                  t        j                  k(  rht        j                  |«      j                  «       sEt        j                   |j                  «      j"                  dz
  }t        j$                  || |¬«      }|S )Nro   rw   iè  )ÚminÚmax)rº   r   r}   rq   ry   r¼   r¾   rÁ   r¿   rÂ   rÃ   ra   r@   Úfloat16ÚisfiniteÚallÚfinforÇ   Úclamp)r1   r–   ro   rr   ÚresidualÚ_Úclamp_values          r3   r7   zPLBartEncoderLayer.forward  sh  € ð !ˆØ)˜4Ÿ>™>Øñ
à)ð
ð ñ
Ñˆ�qô
 Ÿ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ  =Ñ0ˆØ×1Ñ1°-Ó@ˆà ˆØ×*Ñ*¨4¯8©8°MÓ+BÓCˆÜŸ™×-Ñ-¨m¸t×?VÑ?VÐae×anÑanÐ-ÓoˆØŸ™ Ó/ˆÜŸ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ  =Ñ0ˆØ×-Ñ-¨mÓ<ˆà×Ñ¤%§-¡-Ò/¼¿¹À}Ó8U×8YÑ8YÔ8[ÜŸ+™+ m×&9Ñ&9Ó:×>Ñ>ÀÑEˆKÜ!ŸK™K¨¸K¸<È[ÔYˆMàÐr4   r.   )r:   r;   r<   r&   r>   r0   r@   ÚFloatTensorr   r   rA   r7   rB   rC   s   @r3   rI   rI     s[   ø„ ñ=˜|ð =¸¸d¹
õ =ð&à×(Ñ(ðð ×)Ñ)ðð Ð+Ñ,ð	ð
 
�‰÷r4   rI   c                   óÈ   ‡ — e Zd ZdZeedœZdefˆ fd„Ze	e
e	 	 	 ddej                  dz  dej                  dz  dej                  dz  d	ee   d
ef
d„«       «       «       Zˆ xZS )ÚPLBartEncoderzä
    Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a
    [`PLBartEncoderLayer`].

    Args:
        config: PLBartConfig
        embed_tokens (nn.Embedding): output embedding
    )r–   Ú
attentionsrF   c           	      óˆ  •— t         ‰| �  |«       |j                  | _        |j                  | _        |j
                  }|j                  | _        |j                  | _	        |j                  rt        j                  |«      nd}t        |j                  || j                  |¬«      | _        t!        |j                  |«      | _        t%        j&                  t)        |j*                  «      D �cg c]  }t-        ||¬«      ‘Œ c}«      | _        t%        j0                  |«      | _        d| _        | j7                  «        y c c}w ©Nr8   ©r,   )rŠ   F)r/   r0   rq   Úencoder_layerdropÚ	layerdropr·   Úpad_token_idr+   Úmax_position_embeddingsÚmax_source_positionsÚscale_embeddingÚmathÚsqrtr(   Ú
vocab_sizeÚembed_tokensrZ   Úembed_positionsr   Ú
ModuleListÚrangeÚencoder_layersrI   r¡   r»   Úlayernorm_embeddingÚgradient_checkpointingÚ	post_init)r1   rF   r…   r,   Úir2   s        €r3   r0   zPLBartEncoder.__init__H  sÿ   ø€ Ü‰Ñ˜Ô à—~‘~ˆŒØ×1Ñ1ˆŒà—N‘Nˆ	Ø!×.Ñ.ˆÔØ$*×$BÑ$BˆÔ!Ø.4×.DÒ.D”d—i‘i 	Ô*È#ˆä5Ø×Ñ˜y¨$×*:Ñ*:Èô
ˆÔô  @Ø×*Ñ*Øó 
ˆÔô —m‘mÔV[Ð\b×\qÑ\qÓVrÖ$sÐQRÔ%7¸È!Ö%LÒ$sÓtˆŒÜ#%§<¡<°	Ó#:ˆÔ à&+ˆÔ#à�‰Õùò %ts   Ã-D?Nr5   ro   Úinputs_embedsrr   r™   c                 óX  — |d u |d uz  rt        d«      ‚|€| j                  |«      }| j                  |d d …d d …df   «      }|j                  |j                  «      }||z   }| j                  |«      }t        j                  j                  || j                  | j                  ¬«      }t        | j                  ||¬«      }t        | j                  «      D ]F  \  }}d}	| j                  r&t        j                  g «      }
|
| j                   k  rd}	|	rŒ= |||fi |¤Ž}ŒH t#        |¬«      S )Nz:You must specify exactly one of input_ids or inputs_embedsrc   rw   )rF   ré   ro   FT)Úlast_hidden_state)rŽ   rà   rá   Útorb   rå   r   r}   rq   ry   r   rF   Ú	enumerater¡   r@   ÚrandrØ   r   )r1   r5   ro   ré   rr   Ú	embed_posr–   ÚidxÚencoder_layerÚto_dropÚdropout_probabilitys              r3   r7   zPLBartEncoder.forwardb  s7  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ×-Ñ-¨iÓ8ˆMà×(Ñ(¨²qº!¸R°xÑ)@ÓAˆ	Ø—L‘L ×!5Ñ!5Ó6ˆ	à%¨	Ñ1ˆØ×0Ñ0°Ó?ˆÜŸ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-Ódˆä2Ø—;‘;Ø'Ø)ô
ˆô
 #,¨D¯K©KÓ"8ò 	ÑˆC�àˆGØ�}Š}Ü&+§j¡j°£nÐ#Ø&¨¯©Ò7Ø"�GâÙ -Ø!Ø"ñ!ð ñ!‘ð	ô Ø+ô
ð 	
r4   r²   )r:   r;   r<   r=   rI   r„   Ú_can_record_outputsr&   r0   r"   r$   r   r@   Ú
LongTensorrA   rÐ   r   r   r   r7   rB   rC   s   @r3   rÒ   rÒ   9  s¨   ø„ ñð ,Ø%ñÐð
˜|õ ð4  ØØð .2Ø.2Ø26ñ	*
à×#Ñ# dÑ*ð*
ð Ÿ™ tÑ+ð*
ð ×(Ñ(¨4Ñ/ð	*
ð
 Ð+Ñ,ð*
ð 
ò*
ó ó ó  ô*
r4   rÒ   c                   óì   ‡ — e Zd Zddededz  fˆ fd„Z	 	 	 	 	 ddej                  dej                  dz  dej                  dz  dej                  dz  d	edz  d
e	dz  de
e   dej                  fd„Zˆ xZS )rH   NrF   rŠ   c           	      ó  •— t         ‰| �  «        |j                  | _        t	        | j                  |j
                  |j                  dd||¬«      | _        |j                  | _        t        |j                     | _        |j                  | _        t        j                  | j                  «      | _        t	        | j                  |j
                  |j                  d||¬«      | _        t        j                  | j                  «      | _        t        j$                  | j                  |j&                  «      | _        t        j$                  |j&                  | j                  «      | _        t        j                  | j                  «      | _        y )NT)r…   r†   rq   r‡   r‰   rF   rŠ   )rq   r‡   rF   rŠ   )r/   r0   r·   r…   r„   Údecoder_attention_headsr¹   rº   rq   r
   r½   r¾   r¿   r   r»   r¼   Úencoder_attnÚencoder_attn_layer_normr‘   Údecoder_ffn_dimrÁ   rÂ   rÃ   rÄ   s      €r3   r0   zPLBartDecoderLayer.__init__“  s   ø€ Ü‰ÑÔØŸ™ˆŒä(Ø—n‘nØ×4Ñ4Ø×,Ñ,ØØØØô
ˆŒð —~‘~ˆŒÜ# F×$>Ñ$>Ñ?ˆÔØ"(×";Ñ";ˆÔä$&§L¡L°·±Ó$@ˆÔ!Ü+Ø�N‰NØ×*Ñ*Ø×,Ñ,ØØØô
ˆÔô (*§|¡|°D·N±NÓ'CˆÔ$Ü—9‘9˜TŸ^™^¨V×-CÑ-CÓDˆŒÜ—9‘9˜V×3Ñ3°T·^±^ÓDˆŒÜ "§¡¨T¯^©^Ó <ˆÕr4   r–   ro   Úencoder_hidden_statesÚencoder_attention_maskr˜   Ú	use_cacherr   r™   c                 ó  — |} | j                   |f||dœ|¤Ž\  }}	t        j                  j                  || j                  | j                  ¬«      }||z   }| j                  |«      }|�h|} | j                  |f|||dœ|¤Ž\  }}	t        j                  j                  || j                  | j                  ¬«      }||z   }| j                  |«      }|}| j                  | j                  |«      «      }t        j                  j                  || j                  | j                  ¬«      }| j                  |«      }t        j                  j                  || j                  | j                  ¬«      }||z   }| j                  |«      }|S )N)r˜   ro   rw   )r—   ro   r˜   )rº   r   r}   rq   ry   r¼   rù   rú   r¾   rÁ   r¿   rÂ   rÃ   )
r1   r–   ro   rü   rý   r˜   rþ   rr   rÍ   rÎ   s
             r3   r7   zPLBartDecoderLayer.forward²  s—  € ð !ˆð *˜4Ÿ>™>Øð
à+Ø)ñ
ð ñ	
Ñˆ�qô Ÿ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ  =Ñ0ˆØ×1Ñ1°-Ó@ˆð !Ð,Ø$ˆHà0˜t×0Ñ0Øð à!6Ø5Ø /ñ	 ð
 ñ ÑˆM˜1ô ŸM™M×1Ñ1°-À4Ç<Á<ÐZ^×ZgÑZgÐ1ÓhˆMØ$ }Ñ4ˆMØ ×8Ñ8¸ÓGˆMð !ˆØ×*Ñ*¨4¯8©8°MÓ+BÓCˆÜŸ™×-Ñ-¨m¸t×?VÑ?VÐae×anÑanÐ-ÓoˆØŸ™ Ó/ˆÜŸ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-ÓdˆØ  =Ñ0ˆØ×-Ñ-¨mÓ<ˆàÐr4   r.   )NNNNT)r:   r;   r<   r&   r>   r0   r@   rA   r   r³   r   r   r7   rB   rC   s   @r3   rH   rH   ’  s°   ø„ ñ=˜|ð =¸¸d¹
õ =ðD /3Ø59Ø6:Ø(,Ø!%ñ/à—|‘|ð/ð Ÿ™ tÑ+ð/ð  %Ÿ|™|¨dÑ2ð	/ð
 !&§¡¨tÑ 3ð/ð  ™ð/ð ˜$‘;ð/ð Ð+Ñ,ð/ð 
�‰÷/r4   rH   c                   óF  ‡ — e Zd ZdZe eedd¬«       eedd¬«      dœZdefˆ fd„Z	e
ee	 	 	 	 	 	 	 dd
ej                  d	z  dej                  d	z  dej                   d	z  dej                  d	z  ded	z  dej                   d	z  ded	z  dee   defd„«       «       «       Zˆ xZS )ÚPLBartDecoderzÐ
    Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`PLBartDecoderLayer`]

    Args:
        config: PLBartConfig
        embed_tokens (nn.Embedding): output embedding
    r%   rº   )ÚindexÚ
layer_namerù   )r–   rÓ   Úcross_attentionsrF   c           	      óÀ  •— t         ‰| �  |«       |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  rt        j                  |j                  «      nd}t        |j                  |j                  | j                  |¬«      | _        t!        |j                  |j                  «      | _        t%        j&                  t)        |j*                  «      D �cg c]  }t-        ||¬«      ‘Œ c}«      | _        t%        j0                  |j                  «      | _        d| _        | j7                  «        y c c}w rÕ   )r/   r0   rq   Údecoder_layerdroprØ   rÙ   r+   rÚ   Úmax_target_positionsrÜ   rÝ   rÞ   r·   r(   rß   rà   rZ   rá   r   râ   rã   Údecoder_layersrH   r¡   r»   rå   ræ   rç   )r1   rF   r,   rè   r2   s       €r3   r0   zPLBartDecoder.__init__ó  s  ø€ Ü‰Ñ˜Ô Ø—~‘~ˆŒØ×1Ñ1ˆŒØ!×.Ñ.ˆÔØ$*×$BÑ$BˆÔ!Ø39×3IÒ3I”d—i‘i §¡Ô/Èsˆä5Ø×Ñ˜vŸ~™~¨t×/?Ñ/?È[ô
ˆÔô  @Ø×*Ñ*Ø�N‰Nó 
ˆÔô —m‘mÔV[Ð\b×\qÑ\qÓVrÖ$sÐQRÔ%7¸È!Ö%LÒ$sÓtˆŒä#%§<¡<°·±Ó#?ˆÔ à&+ˆÔ#à�‰Õùò %ts   Ã?ENr5   ro   rü   rý   r˜   ré   rþ   rr   r™   c                 óˆ  — |d u |d uz  rt        d«      ‚|€| j                  |«      }|rd|€b|€| j                  j                  r4t	        t        | j                  ¬«      t        | j                  ¬«      «      nt        | j                  ¬«      }|j                  «       d d \  }	}
|�|j                  «       nd}t        j                  |
|j                  ¬«      |z   }|€1t        «       s'||
z   }t        j                  |	||j                  ¬«      }t        |t        «      r|j                  n|}t        | j                  |||¬«      }t!        | j                  |||¬«      }| j#                  |||¬«      }|j%                  |j                  «      }||z   }| j'                  |«      }t(        j*                  j-                  || j,                  | j.                  ¬	«      }t1        | j2                  «      D ]E  \  }}| j.                  r%t        j4                  g «      }|| j6                  k  rŒ7 ||||f|||d
œ|¤Ž}ŒG t9        ||¬«      S )NzJYou must specify exactly one of decoder_input_ids or decoder_inputs_embeds)rF   rc   r   ©rb   )rF   ré   ro   r˜   )rF   ré   ro   rü   )r_   rw   )rý   r˜   rþ   )rë   r˜   )rŽ   rà   rF   Úis_encoder_decoderr   r   rz   Úget_seq_lengthr@   re   rb   r   ÚonesrL   r    r   r   rá   rì   rå   r   r}   rq   ry   rí   r¡   rî   rØ   r   )r1   r5   ro   rü   rý   r˜   ré   rþ   rr   Ú
batch_sizeÚ
seq_lengthr^   r_   Úmask_seq_lengthÚself_attn_cacheÚ	positionsr–   rð   Údecoder_layerró   s                       r3   r7   zPLBartDecoder.forward  sP  € ð ˜Ð -°tÐ";Ò<ÜÐiÓjÐjàÐ Ø ×-Ñ-¨iÓ8ˆMñ ˜Ð0ð )Ð4¸¿¹×8VÒ8Vô $¤L¸¿¹Ô$DÄlÐZ^×ZeÑZeÔFfÔgä!¨¯©Ô5ð ð "/×!3Ñ!3Ó!5°c°rÐ!:Ñˆ
�JØETÐE` ×!?Ñ!?Ô!AÐfgÐÜ—|‘| J°}×7KÑ7KÔLÐOeÑeˆàÐ!Ô*BÔ*Dà4°zÑAˆOÜ"ŸZ™Z¨
°OÈM×L`ÑL`ÔaˆNô ˜/Ô+>Ô?ð ×0Ò0à ð 	ô ,Ø—;‘;Ø'Ø)Ø+ô	
ˆô ";Ø—;‘;Ø'Ø1Ø"7ô	"
Ðð ×(Ñ(¨Ð4JÐYeÐ(Ófˆ	Ø—L‘L ×!5Ñ!5Ó6ˆ	à%¨	Ñ1ˆØ×0Ñ0°Ó?ˆäŸ™×-Ñ-¨m¸t¿|¹|ÐVZ×VcÑVcÐ-Ódˆä"+¨D¯K©KÓ"8ò 	ÑˆC�à�}Š}Ü&+§j¡j°£nÐ#Ø&¨¯©Ò7Øá)ØØØ%ðð (>Ø /Ø#ñð ñ‰Mð	ô" 9Ø+Ø+ô
ð 	
r4   )NNNNNNN)r:   r;   r<   r=   rH   r#   r„   rô   r&   r0   r"   r$   r   r@   rõ   rA   rÐ   r   r³   r   r   r   r7   rB   rC   s   @r3   r  r  ä  s  ø„ ñð ,Ù$ _¸AÈ+ÔVÙ*¨?À!ÐP^Ô_ñÐð˜|õ ð0  ØØð .2Ø.2Ø:>Ø:>Ø(,Ø26Ø!%ñR
à×#Ñ# dÑ*ðR
ð Ÿ™ tÑ+ðR
ð  %×0Ñ0°4Ñ7ð	R
ð
 !&× 0Ñ 0°4Ñ 7ðR
ð  ™ðR
ð ×(Ñ(¨4Ñ/ðR
ð ˜$‘;ðR
ð Ð+Ñ,ðR
ð 
3òR
ó ó ó  ôR
r4   r  r5   rÙ   c                 óf  — | j                  «       }|€t        d«      ‚|j                  |dk(  |«       |j                  |«      j	                  d¬«      dz
  j                  d«      }|j                  d|«      j                  «       }|dd…dd…f   j                  «       |dd…dd…f<   ||dd…df<   |S )zÏ
    Shift input ids one token to the right, and wrap the last non pad token (the <LID> token) Note that PLBart does not
    have a single `decoder_start_token_id` in contrast to other Bart-like models.
    Nz1self.model.config.pad_token_id has to be defined.iœÿÿÿr%   ru   rc   r   )ÚclonerŽ   Úmasked_fill_ÚneÚsumri   ÚgatherÚsqueeze)r5   rÙ   Úprev_output_tokensÚindex_of_eosÚdecoder_start_tokenss        r3   Úshift_tokens_rightr  c  sÀ   € ð
 #Ÿ™Ó*ÐàÐÜÐLÓMÐMà×#Ñ#Ð$6¸$Ñ$>ÀÔMà&×)Ñ)¨,Ó7×;Ñ;ÀÐ;ÓBÀQÑF×QÑQÐRTÓU€LØ-×4Ñ4°Q¸ÓE×MÑMÓOÐØ 2²1°c°r°c°6Ñ :× @Ñ @Ó BÐ’q˜!™"�uÑØ3Ð’q˜!�tÑàÐr4   c                   óŽ  ‡ — e Zd ZdddœZdefˆ fd„Zd„ Zd„ Zee	e
	 	 	 	 	 	 	 	 	 ddej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  deej                     dz  dedz  dej                  dz  dej                  dz  dedz  dee   deej                     ez  fd„«       «       «       Zˆ xZS )ÚPLBartModelzshared.weight)zencoder.embed_tokens.weightzdecoder.embed_tokens.weightrF   c                 óJ  •— t         ‰| �  |«       |j                  |j                  }}|j                  rt        j                  |j                  «      nd}t        ||j                  ||¬«      | _	        t        |«      | _        t        |«      | _        | j                  «        y )Nr8   rÖ   )r/   r0   rÙ   rß   rÜ   rÝ   rÞ   r·   r(   ÚsharedrÒ   Úencoderr  Údecoderrç   )r1   rF   r+   rß   r,   r2   s        €r3   r0   zPLBartModel.__init__~  sz   ø€ Ü‰Ñ˜Ô à"(×"5Ñ"5°v×7HÑ7H�ZˆØ39×3IÒ3I”d—i‘i §¡Ô/ÈsˆÜ/°
¸F¿N¹NÈKÐepÔqˆŒä$ VÓ,ˆŒÜ$ VÓ,ˆŒà�‰Õr4   c                 ó   — | j                   S r.   )r"  ©r1   s    r3   Úget_input_embeddingsz PLBartModel.get_input_embeddingsŠ  s   € Ø�{‰{Ðr4   c                 ó~   — || _         | j                   | j                  _        | j                   | j                  _        y r.   )r"  r#  rà   r$  ©r1   rn   s     r3   Úset_input_embeddingsz PLBartModel.set_input_embeddings�  s)   € ØˆŒØ$(§K¡Kˆ�‰Ô!Ø$(§K¡Kˆ�‰Õ!r4   Nr5   ro   Údecoder_input_idsÚdecoder_attention_maskÚencoder_outputsr˜   ré   Údecoder_inputs_embedsrþ   rr   r™   c
                 ó
  — |€"|€ t        || j                  j                  «      }|€ | j                  d	|||dœ|
¤Ž}nGt	        |t
        «      s7t        |d   t        |«      dkD  r|d   ndt        |«      dkD  r|d   nd¬«      } | j                  d	|||d   ||||	dœ|
¤Ž}t        |j                  |j                  |j                  |j                  |j                  |j                  |j                  |j                  ¬«      S )
a  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`] or [`PLBartMultiTokenizer`] depending on the checkpoint.
            See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            PLBart uses a specific language id token as the starting token for `decoder_input_ids` generation that
            varies according to source and target language, *e.g.* 50003 for *en_XX*, and 50001 for *java*. If
            `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
            `past_key_values`).

            For translation and summarization training, `decoder_input_ids` should be provided. If no
            `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
            for denoising pre-training following the paper.
        decoder_attention_mask (:
            obj:*torch.LongTensor* of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior:
            generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default.
        N)r5   ro   ré   r   r%   r\   )rë   r–   rÓ   ©r5   ro   rü   rý   r˜   ré   rþ   )rë   r˜   Údecoder_hidden_statesÚdecoder_attentionsr  Úencoder_last_hidden_staterü   Úencoder_attentions© )r  rF   rÙ   r#  rL   r   Úlenr$  r   rë   r˜   r–   rÓ   r  )r1   r5   ro   r+  r,  r-  r˜   ré   r.  rþ   rr   Údecoder_outputss               r3   r7   zPLBartModel.forward’  s3  € ðP Ð$Ð)>Ð)FÜ 2°9¸d¿k¹k×>VÑ>VÓ WÐàÐ"Ø/;¨t¯|©|ð 0Ø#Ø-Ø+ñ0ð ñ	0‰Oô ˜O¬_Ô=Ü-Ø"1°!Ñ"4Ü47¸Ó4HÈ1Ò4L˜o¨aÒ0ÐRVÜ14°_Ó1EÈÒ1I˜?¨1Ò-ÈtôˆOð '˜$Ÿ,™,ð 	
Ø'Ø1Ø"1°!Ñ"4Ø#1Ø+Ø/Øñ	
ð ñ	
ˆô "Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
r4   ©	NNNNNNNNN)r:   r;   r<   Ú_tied_weights_keysr&   r0   r'  r*  r"   r$   r   r@   rõ   rA   ÚlistrÐ   r   r³   r   r   r´   r   r7   rB   rC   s   @r3   r   r   w  sP  ø„ ð (7Ø'6ñÐð

˜|õ 
òò0ð
  ØØð .2Ø26Ø59Ø6:Ø:>Ø(,Ø26Ø:>Ø!%ñJ
à×#Ñ# dÑ*ðJ
ð ×(Ñ(¨4Ñ/ðJ
ð !×+Ñ+¨dÑ2ð	J
ð
 !&§¡¨tÑ 3ðJ
ð ˜e×/Ñ/Ñ0°4Ñ7ðJ
ð  ™ðJ
ð ×(Ñ(¨4Ñ/ðJ
ð  %×0Ñ0°4Ñ7ðJ
ð ˜$‘;ðJ
ð Ð+Ñ,ðJ
ð 
ˆu�|‰|Ñ	Ð1Ñ	1òJ
ó ó ó  ôJ
r4   r   zv
    The PLBART Model with a language modeling head. Can be used for code-to-text, text-to-code and code-to-code.
    )Úcustom_introc                   ó  ‡ — e Zd ZdZdgZddiZdefˆ fd„Z	 dded	edz  d
e	de
j                  fˆ fd„Zdeddfd„Zeee	 	 	 	 	 	 	 	 	 	 ddej$                  dz  dej$                  dz  dej$                  dz  dej&                  dz  deej*                     dz  dedz  dej*                  dz  dej*                  dz  dej&                  dz  de	dz  dee   deej&                     ez  fd„«       «       «       Zdej&                  fd„Zˆ xZS )rM   rG   rP   úlm_head.weightzmodel.shared.weightrF   c                 óx  •— t         ‰| �  |«       t        |«      | _        | j	                  dt        j                  d| j                  j                  j                  f«      «       t        j                  |j                  | j                  j                  j                  d¬«      | _        | j                  «        y )NrP   r%   FrŒ   )r/   r0   r   rG   Úregister_bufferr@   Úzerosr"  r)   r   r‘   r·   Úlm_headrç   ©r1   rF   r2   s     €r3   r0   z'PLBartForConditionalGeneration.__init__î  s~   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø×ÑÐ0´%·+±+¸qÀ$Ç*Á*×BSÑBS×BbÑBbÐ>cÓ2dÔeÜ—y‘y §¡°·±×1BÑ1B×1QÑ1QÐX]Ô^ˆŒà�‰Õr4   NÚnew_num_tokensÚpad_to_multiple_ofÚmean_resizingr™   c                 óz   •— t         ‰| �  |||«      }| j                  |j                  j                  d   «       |S )Nr   )r/   Úresize_token_embeddingsÚ_resize_final_logits_biasrg   rd   )r1   rC  rD  rE  Únew_embeddingsr2   s        €r3   rG  z6PLBartForConditionalGeneration.resize_token_embeddingsö  s?   ø€ ô ™Ñ8¸ÐI[Ð]jÓkˆØ×&Ñ& ~×'<Ñ'<×'BÑ'BÀ1Ñ'EÔFØÐr4   c                 ó6  — | j                   j                  d   }||k  r| j                   d d …d |…f   }nSt        j                  d||z
  f| j                   j                  ¬«      }t        j
                  | j                   |gd¬«      }| j                  d|«       y )Nrc   r%   r
  ru   rP   )rP   rd   r@   r@  rb   Úcatr?  )r1   rC  Úold_num_tokensÚnew_biasÚ
extra_biass        r3   rH  z8PLBartForConditionalGeneration._resize_final_logits_biasý  sŒ   € Ø×/Ñ/×5Ñ5°bÑ9ˆØ˜^Ò+Ø×-Ñ-ªa°°.°Ð.@ÑA‰HäŸ™ a¨¸.Ñ)HÐ%IÐRV×RhÑRh×RoÑRoÔpˆJÜ—y‘y $×"8Ñ"8¸*Ð!EÈ1ÔMˆHØ×ÑÐ0°(Õ;r4   r5   ro   r+  r,  r-  r˜   ré   r.  Úlabelsrþ   rr   c                 óT  — |	�$|€"|€ t        |	| j                  j                  «      } | j                  |f||||||||
dœ|¤Ž}| j	                  |j
                  «      }|| j                  j                  |j                  «      z   }d}|	�Ft        «       } ||j                  d| j                  j                  «      |	j                  d«      «      }t        |||j                  |j                  |j                  |j                   |j"                  |j$                  |j&                  ¬«	      S )a 
  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`] or [`PLBartMultiTokenizer`] depending on the checkpoint.
            See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            PLBart uses a specific language id token as the starting token for `decoder_input_ids` generation that
            varies according to source and target language, *e.g.* 50003 for *en_XX*, and 50001 for *java*. If
            `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
            `past_key_values`).

            For translation and summarization training, `decoder_input_ids` should be provided. If no
            `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
            for denoising pre-training following the paper.
        decoder_attention_mask (:
            obj:*torch.LongTensor* of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior:
            generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example Mask-filling:

        ```python
        >>> from transformers import AutoTokenizer, PLBartForConditionalGeneration

        >>> model = PLBartForConditionalGeneration.from_pretrained("uclanlp/plbart-base")
        >>> tokenizer = AutoTokenizer.from_pretrained("uclanlp/plbart-base")

        >>> # en_XX is the language symbol id <LID> for English
        >>> TXT = "<s> Is 0 the <mask> Fibonacci number ? </s> en_XX"
        >>> input_ids = tokenizer([TXT], add_special_tokens=False, return_tensors="pt").input_ids

        >>> logits = model(input_ids).logits
        >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item()
        >>> probs = logits[0, masked_index].softmax(dim=0)
        >>> values, predictions = probs.topk(5)

        >>> tokenizer.decode(predictions).split()
        ['first', 'same', 'highest', 'result', 'number']
        ```
        N)ro   r+  r-  r,  r˜   ré   r.  rþ   rc   ©	ÚlossÚlogitsr˜   r1  r2  r  r3  rü   r4  )r  rF   rÙ   rG   rA  rë   rP   rì   rb   r   rœ   rß   r   r˜   r1  r2  r  r3  rü   r4  )r1   r5   ro   r+  r,  r-  r˜   ré   r.  rO  rþ   rr   ÚoutputsÚ	lm_logitsÚmasked_lm_lossÚloss_fcts                   r3   r7   z&PLBartForConditionalGeneration.forward  s2  € ð@ ÐØ Ð(Ð-BÐ-JÜ$6°v¸t¿{¹{×?WÑ?WÓ$XÐ!à&0 d§j¡jØð'
à)Ø/Ø+Ø#9Ø+Ø'Ø"7Øñ'
ð ñ'
ˆð —L‘L ×!:Ñ!:Ó;ˆ	Ø × 6Ñ 6× 9Ñ 9¸)×:JÑ:JÓ KÑKˆ	àˆØÐÜ'Ó)ˆHÙ% i§n¡n°R¸¿¹×9OÑ9OÓ&PÐRX×R]ÑR]Ð^`ÓRaÓbˆNäØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ô

ð 
	
r4   c                 óB   — t        || j                  j                  «      S r.   )r  rF   rÙ   )r1   rO  s     r3   Ú%prepare_decoder_input_ids_from_labelszDPLBartForConditionalGeneration.prepare_decoder_input_ids_from_labelsj  s   € Ü! &¨$¯+©+×*BÑ*BÓCÐCr4   )NT)
NNNNNNNNNN)r:   r;   r<   rS   Ú_keys_to_ignore_on_load_missingr9  r&   r0   r>   r³   r   Ú	EmbeddingrG  rH  r"   r$   r   r@   rõ   rA   r:  rÐ   r   r   r   r´   r   r7   rY  rB   rC   s   @r3   rM   rM   â  sÅ  ø„ ð  ÐØ':Ð&;Ð#àÐ/ðÐð˜|õ ð aeñØ!ðØ7:¸T±zðØY]ðà	�‰õð<¸ð <Àó <ð  ØØð .2Ø26Ø59Ø6:Ø:>Ø(,Ø26Ø:>Ø&*Ø!%ñ_
à×#Ñ# dÑ*ð_
ð ×(Ñ(¨4Ñ/ð_
ð !×+Ñ+¨dÑ2ð	_
ð
 !&§¡¨tÑ 3ð_
ð ˜e×/Ñ/Ñ0°4Ñ7ð_
ð  ™ð_
ð ×(Ñ(¨4Ñ/ð_
ð  %×0Ñ0°4Ñ7ð_
ð —‘˜tÑ#ð_
ð ˜$‘;ð_
ð Ð+Ñ,ð_
ð 
ˆu�|‰|Ñ	˜Ñ	.ò_
ó ó ó  ð_
ðBD¸E¿L¹L÷ Dr4   rM   c                   ól   ‡ — e Zd ZdZdedededefˆ fd„Zdej                  dej                  fd	„Z	ˆ xZ
S )
ÚPLBartClassificationHeadz-Head for sentence-level classification tasks.Ú	input_dimÚ	inner_dimÚnum_classesÚpooler_dropoutc                 óÄ   •— t         ‰| �  «        t        j                  ||«      | _        t        j
                  |¬«      | _        t        j                  ||«      | _        y )N)rx   )r/   r0   r   r‘   ÚdenseÚDropoutrq   r•   )r1   r^  r_  r`  ra  r2   s        €r3   r0   z!PLBartClassificationHead.__init__q  sD   ø€ ô 	‰ÑÔÜ—Y‘Y˜y¨)Ó4ˆŒ
Ü—z‘z NÔ3ˆŒÜŸ	™	 )¨[Ó9ˆ�r4   r–   r™   c                 ó¸   — | j                  |«      }| j                  |«      }t        j                  |«      }| j                  |«      }| j	                  |«      }|S r.   )rq   rc  r@   Útanhr•   )r1   r–   s     r3   r7   z PLBartClassificationHead.forward}  sN   € ØŸ™ ]Ó3ˆØŸ
™
 =Ó1ˆÜŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ mÓ4ˆØÐr4   r9   rC   s   @r3   r]  r]  n  sL   ø„ Ù7ð
:àð
:ð ð
:ð ð	
:ð
 õ
:ð U§\¡\ð °e·l±l÷ r4   r]  z‡
    PLBart model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g.
    for GLUE tasks.
    c                   óh  ‡ — e Zd Zdefˆ fd„Zee	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  de
ej                     dz  d	ej                  dz  d
ej                  dz  dej                  dz  dedz  dee   deez  fd„«       «       Zˆ xZS )ÚPLBartForSequenceClassificationrF   c                 óÜ   •— t        ‰| �  |fi |¤Ž t        |«      | _        t	        |j
                  |j
                  |j                  |j                  «      | _        | j                  «        y r.   )
r/   r0   r   rG   r]  r·   Ú
num_labelsÚclassifier_dropoutÚclassification_headrç   )r1   rF   rr   r2   s      €r3   r0   z(PLBartForSequenceClassification.__init__�  sZ   ø€ Ü‰Ñ˜Ñ* 6Ò*Ü  Ó(ˆŒ
Ü#;Ø�N‰NØ�N‰NØ×ÑØ×%Ñ%ó	$
ˆÔ ð 	�‰Õr4   Nr5   ro   r+  r,  r-  ré   r.  rO  rþ   rr   r™   c
                 óä  — |�d}	|€$|�"t        d| j                  j                  › �«      ‚ | j                  |f|||||||	dœ|
¤Ž}|d   }|j	                  | j
                  j                  «      j                  |j                  «      }t        t        j                  |j                  d«      «      j                  «       dk(  d«       ||dd…f   }t        |j                  d   |j                  d   z  dk\  d«       |j                  |j!                  d«      d	|j!                  d	«      «      dd…d	dd…f   }| j#                  |«      }d}|��Ê|j                  |j                  «      }| j
                  j$                  €¡| j
                  j&                  dk(  rd
| j
                  _        nv| j
                  j&                  dkD  rL|j(                  t        j*                  k(  s|j(                  t        j,                  k(  rd| j
                  _        nd| j
                  _        | j
                  j$                  d
k(  rSt/        «       }| j
                  j&                  dk(  r& ||j1                  «       |j1                  «       «      }n– |||«      }nŒ| j
                  j$                  dk(  rGt3        «       } ||j                  d	| j
                  j&                  «      |j                  d	«      «      }n,| j
                  j$                  dk(  rt5        «       } |||«      }t7        |||j8                  |j:                  |j<                  |j>                  |j@                  |jB                  |jD                  ¬«	      S )a©  
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`] or [`PLBartMultiTokenizer`] depending on the checkpoint.
            See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            PLBart uses a specific language id token as the starting token for `decoder_input_ids` generation that
            varies according to source and target language, *e.g.* 50003 for *en_XX*, and 50001 for *java*. If
            `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
            `past_key_values`).

            For translation and summarization training, `decoder_input_ids` should be provided. If no
            `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right
            for denoising pre-training following the paper.
        decoder_attention_mask (:
            obj:*torch.LongTensor* of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior:
            generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NFz8Passing input embeddings is currently not supported for )ro   r+  r,  r-  ré   r.  rþ   r   r%   z7All examples must have the same number of <eos> tokens.z3Each example must contain at least one <eos> token.rc   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationrQ  )#ÚNotImplementedErrorr2   r:   rG   ÚeqrF   Úeos_token_idrì   rb   r!   r@   Úunique_consecutiver  Únumelrd   rœ   rz   rl  Úproblem_typerj  ra   rf   r>   r   r  r   r   r   r˜   r1  r2  r  r3  rü   r4  )r1   r5   ro   r+  r,  r-  ré   r.  rO  rþ   rr   rT  r–   Úeos_maskÚselectedÚsentence_representationrS  rR  rW  s                      r3   r7   z'PLBartForSequenceClassification.forwardš  s	  € ðP ÐØˆIàÐ Ð!:Ü%ØJÈ4Ï>É>×KbÑKbÐJcÐdóð ð '1 d§j¡jØð
'
à)Ø/Ø#9Ø+Ø'Ø"7Øñ
'
ð ñ
'
ˆð   ™
ˆà—<‘< §¡× 8Ñ 8Ó9×<Ñ<¸]×=QÑ=QÓRˆäÜ×$Ñ$ X§\¡\°!£_Ó5×;Ñ;Ó=ÀÑBØEô	
ð ! ª1 Ñ-ˆÜØ�N‰N˜1Ñ ×!4Ñ!4°QÑ!7Ñ7¸1Ñ<ØAô	
ð #+§-¡-°×0BÑ0BÀ1Ó0EÀrÈ=×K]ÑK]Ð^`ÓKaÓ"bÒcdÐfhÒjkÐckÑ"lÐØ×)Ñ)Ð*AÓBˆàˆØÑØ—Y‘Y˜vŸ}™}Ó-ˆFØ�{‰{×'Ñ'Ð/Ø—;‘;×)Ñ)¨QÒ.Ø/;�D—K‘KÕ,Ø—[‘[×+Ñ+¨aÒ/°V·\±\ÄUÇZÁZÒ5OÐSY×S_ÑS_Ôch×clÑclÒSlØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—;‘;×)Ñ)¨QÒ.Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±×0FÑ0FÓ GÈÏÉÐUWËÓY‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ä.ØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ô

ð 
	
r4   r8  )r:   r;   r<   r&   r0   r   r   r@   rõ   rA   r:  rÐ   r³   r   r   r´   r   r7   rB   rC   s   @r3   rh  rh  †  s,  ø„ ð˜|õ ð Øð .2Ø.2Ø59Ø:>Ø:>Ø26Ø:>Ø*.Ø!%ñk
à×#Ñ# dÑ*ðk
ð Ÿ™ tÑ+ðk
ð !×+Ñ+¨dÑ2ð	k
ð
 !&× 0Ñ 0°4Ñ 7ðk
ð ˜e×/Ñ/Ñ0°4Ñ7ðk
ð ×(Ñ(¨4Ñ/ðk
ð  %×0Ñ0°4Ñ7ðk
ð × Ñ  4Ñ'ðk
ð ˜$‘;ðk
ð Ð+Ñ,ðk
ð 
Ð0Ñ	0òk
ó ó ôk
r4   rh  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚPLBartDecoderWrapperz½
    This wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is
    used in combination with the [`EncoderDecoderModel`] framework.
    c                 ód   •— t         ‰| �  |«       t        |«      | _        | j	                  «        y r.   )r/   r0   r  r$  rç   rB  s     €r3   r0   zPLBartDecoderWrapper.__init__  s&   ø€ Ü‰Ñ˜Ô Ü$ VÓ,ˆŒØ�‰Õr4   c                 ó&   —  | j                   |i |¤ŽS r.   )r$  )r1   Úargsrr   s      r3   r7   zPLBartDecoderWrapper.forward  s   € Øˆt�|‰|˜TÐ, VÑ,Ð,r4   )r:   r;   r<   r=   r0   r7   rB   rC   s   @r3   r{  r{  
  s   ø„ ñô
ö
-r4   r{  zw
    PLBART decoder with a language modeling head on top (linear layer with weights tied to the input embeddings).
    c                   ó\  ‡ — e Zd ZddiZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  dedz  de	j                  dz  de	j                  dz  dedz  dee	j                  z  dee   deez  fd„«       «       Zˆ xZS )ÚPLBartForCausalLMr=  z!model.decoder.embed_tokens.weightc                 óâ   •— d|_         d|_        t        ‰| �  |«       t	        |«      | _        t        j                  |j                  |j                  d¬«      | _
        | j                  «        y )NTFrŒ   )r‡   r  r/   r0   r{  rG   r   r‘   Úhidden_sizerß   rA  rç   rB  s     €r3   r0   zPLBartForCausalLM.__init__#  sX   ø€ Ø ˆÔØ$)ˆÔ!Ü‰Ñ˜Ô Ü)¨&Ó1ˆŒ
ä—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Õr4   c                 óB   — | j                   j                  j                  S r.   ©rG   r$  rà   r&  s    r3   r'  z&PLBartForCausalLM.get_input_embeddings.  s   € Ø�z‰z×!Ñ!×.Ñ.Ð.r4   c                 ó:   — || j                   j                  _        y r.   r„  r)  s     r3   r*  z&PLBartForCausalLM.set_input_embeddings1  s   € Ø*/ˆ�
‰
×ÑÕ'r4   Nr5   ro   rü   rý   r˜   ré   rO  rþ   Úlogits_to_keeprr   r™   c
                 ó  —  | j                   j                  d|||||||dœ|
¤Ž}|d   }t        |	t        «      rt	        |	 d«      n|	}| j                  |dd…|dd…f   «      }d}|�a|j                  |j                  «      }t        «       } ||j                  d| j                  j                  «      |j                  d«      «      }t        |||j                  |j                  |j                  |j                   ¬«      S )aF  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
            config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
            (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, PLBartForCausalLM

        >>> tokenizer = AutoTokenizer.from_pretrained("uclanlp/plbart-base")
        >>> model = PLBartForCausalLM.from_pretrained("uclanlp/plbart-base")
        >>> assert model.config.is_decoder, f"{model.__class__} has to be configured as a decoder."
        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> logits = outputs.logits
        >>> expected_shape = [1, inputs.input_ids.shape[-1], model.config.vocab_size]
        >>> list(logits.shape) == expected_shape
        True
        ```r0  r   Nrc   )rR  rS  r˜   r–   rÓ   r  r5  )rG   r$  rL   r>   ÚslicerA  rì   rb   r   rœ   rF   rß   r   r˜   r–   rÓ   r  )r1   r5   ro   rü   rý   r˜   ré   rO  rþ   r†  rr   rT  r–   Úslice_indicesrS  rR  rW  s                    r3   r7   zPLBartForCausalLM.forward4  s  € ðL >P¸T¿Z¹Z×=OÑ=Oð 	>
ØØ)Ø"7Ø#9Ø+Ø'Øñ	>
ð ñ	>
ˆð   ™
ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ—Y‘Y˜vŸ}™}Ó-ˆFÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯K©K×,BÑ,BÓCÀVÇ[Á[ÐQSÃ_ÓUˆDä0ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô
ð 	
r4   )	NNNNNNNNr   )r:   r;   r<   r9  r0   r'  r*  r   r   r@   rõ   rA   rÐ   r   r³   r>   r   r   r´   r   r7   rB   rC   s   @r3   r€  r€    s1  ø„ ð 	Ð=ðÐô	ò/ò0ð Øð .2Ø.2Ø:>Ø;?Ø(,Ø26Ø*.Ø!%Ø-.ñA
à×#Ñ# dÑ*ðA
ð Ÿ™ tÑ+ðA
ð  %×0Ñ0°4Ñ7ð	A
ð
 !&× 1Ñ 1°DÑ 8ðA
ð  ™ðA
ð ×(Ñ(¨4Ñ/ðA
ð × Ñ  4Ñ'ðA
ð ˜$‘;ðA
ð ˜eŸl™lÑ*ðA
ð Ð+Ñ,ðA
ð 
Ð2Ñ	2òA
ó ó ôA
r4   r€  )r€  rM   rh  r   rE   )Nr›   )NrÝ   Úcollections.abcr   r@   r   Útorch.nnr   r   r   Ú r	   rN   Úactivationsr
   Úcache_utilsr   r   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r    r!   Úutils.genericr"   Úutils.output_capturingr#   r$   Úconfiguration_plbartr&   Ú
get_loggerr:   r�   r[  r(   rE   rZ   ÚModulerA   r?   r‚   r„   rI   rÒ   rH   r  r>   r  r   rM   r]  rh  r{  r€  Ú__all__r5  r4   r3   ú<module>r�     sV  ðó* Ý $ã Ý ß AÑ Aå &Ý !ß CÑ CÝ )ß JÝ BÝ 9÷÷ ÷ GÝ &÷÷ õ 8ß EÝ .ð 
ˆ×	Ñ	˜HÓ	%€ô
= §¡ô 
=ð ô2˜Oó 2ó ð2ô; r§|¡|ô ;ðB !Øñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ñ(ó%ô8r)�b—i‘iô r)ôj0Ð3ô 0ôfV
Ð)ô V
ôrOÐ3ô Oôd|
Ð)ô |
ð~ %§,¡,ð ¸có ð( ôg
Ð'ó g
ó ðg
ñT ðôô
DDÐ%:¸Oó DDóð
DDôN˜rŸy™yô ñ0 ðôô{
Ð&;ó {
óð{
ô|-Ð0ô -ñ ðôô
Y
Ð-¨ó Y
óð
Y
òx�r4   