Ë
    ÜÓ^jO! ã                   ó–  — d Z ddlZddlZddlZddlmZ ddlmZmZmZ ddl	m
Z ddlmZ ddlmZmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZmZmZmZmZmZm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z) ddl*m+Z+  e(jX                  e-«      Z. G d„ dej^                  «      Z0 G d„ dej^                  «      Z1 G d„ dej^                  «      Z2 G d„ dej^                  «      Z3 G d„ dej^                  «      Z4 G d„ dej^                  «      Z5 G d„ dej^                  «      Z6 G d„ d e«      Z7 G d!„ d"ej^                  «      Z8e& G d#„ d$e"«      «       Z9 G d%„ d&e9«      Z:e& G d'„ d(e9«      «       Z; e&d)¬*«       G d+„ d,e9e«      «       Z<e& G d-„ d.e9«      «       Z= e&d/¬*«       G d0„ d1e9«      «       Z>e& G d2„ d3e9«      «       Z?e& G d4„ d5e9«      «       Z@g d6¢ZAy)7zPyTorch UMT5 model.é    N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutputÚ#Seq2SeqQuestionAnsweringModelOutputÚSeq2SeqSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)ÚDUMMY_INPUTSÚ
DUMMY_MASKÚauto_docstringÚis_torchdynamo_compilingÚloggingÚtorch_compilable_checké   )Ú
UMT5Configc                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚUMT5LayerNormc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)ze
        Construct a layernorm module in the UMT5 style. No bias and no subtraction of mean.
        N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s      €úq/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/umt5/modeling_umt5.pyr%   zUMT5LayerNorm.__init__7   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕó    c                 óž  — |j                  t        j                  «      j                  d«      j	                  dd¬«      }|t        j
                  || j                  z   «      z  }| j                  j                  t        j                  t        j                  fv r%|j                  | j                  j                  «      }| j                  |z  S )Né   éÿÿÿÿT)Úkeepdim)Útor'   Úfloat32ÚpowÚmeanÚrsqrtr*   r)   ÚdtypeÚfloat16Úbfloat16)r+   Úhidden_statesÚvariances      r/   ÚforwardzUMT5LayerNorm.forward?   s›   € ð !×#Ñ#¤E§M¡MÓ2×6Ñ6°qÓ9×>Ñ>¸rÈ4Ð>ÓPˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆð �;‰;×Ñ¤§¡´·±Ð ?Ñ?Ø)×,Ñ,¨T¯[©[×->Ñ->Ó?ˆMà�{‰{˜]Ñ*Ð*r0   )g�íµ ÷Æ°>)Ú__name__Ú
__module__Ú__qualname__r%   r?   Ú__classcell__©r.   s   @r/   r"   r"   6   s   ø„ õ$ö+r0   r"   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚUMT5DenseActDenseÚconfigc                 ó^  •— t         ‰| �  «        t        j                  |j                  |j
                  d¬«      | _        t        j                  |j
                  |j                  d¬«      | _        t        j                  |j                  «      | _
        t        |j                     | _        y ©NF©Úbias)r$   r%   r   ÚLinearÚd_modelÚd_ffÚwiÚwoÚDropoutÚdropout_rateÚdropoutr	   Údense_act_fnÚact©r+   rG   r.   s     €r/   r%   zUMT5DenseActDense.__init__Q   sn   ø€ Ü‰ÑÔÜ—)‘)˜FŸN™N¨F¯K©K¸eÔDˆŒÜ—)‘)˜FŸK™K¨¯©¸eÔDˆŒÜ—z‘z &×"5Ñ"5Ó6ˆŒÜ˜&×-Ñ-Ñ.ˆ�r0   c                 ó  — | j                  |«      }| j                  |«      }| j                  |«      }t        | j                  j
                  t        j                  «      r�|j                  | j                  j
                  j                  k7  r`| j                  j
                  j                  t        j                  k7  r/|j                  | j                  j
                  j                  «      }| j	                  |«      }|S ©N)rO   rU   rS   Ú
isinstancerP   r)   r'   ÚTensorr:   Úint8r5   ©r+   r=   s     r/   r?   zUMT5DenseActDense.forwardX   sª   € ØŸ™ Ó.ˆØŸ™ Ó/ˆØŸ™ ]Ó3ˆä�t—w‘w—~‘~¤u§|¡|Ô4Ø×#Ñ# t§w¡w§~¡~×';Ñ';Ò;Ø—‘—‘×$Ñ$¬¯
©
Ò2à)×,Ñ,¨T¯W©W¯^©^×-AÑ-AÓBˆMØŸ™ Ó.ˆØÐr0   ©r@   rA   rB   r    r%   r?   rC   rD   s   @r/   rF   rF   P   s   ø„ ð/˜zõ /ör0   rF   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚUMT5DenseGatedActDenserG   c                 óÀ  •— t         ‰| �  «        t        j                  |j                  |j
                  d¬«      | _        t        j                  |j                  |j
                  d¬«      | _        t        j                  |j
                  |j                  d¬«      | _        t        j                  |j                  «      | _        t        |j                     | _        y rI   )r$   r%   r   rL   rM   rN   Úwi_0Úwi_1rP   rQ   rR   rS   r	   rT   rU   rV   s     €r/   r%   zUMT5DenseGatedActDense.__init__h   sŠ   ø€ Ü‰ÑÔÜ—I‘I˜fŸn™n¨f¯k©kÀÔFˆŒ	Ü—I‘I˜fŸn™n¨f¯k©kÀÔFˆŒ	Ü—)‘)˜FŸK™K¨¯©¸eÔDˆŒÜ—z‘z &×"5Ñ"5Ó6ˆŒÜ˜&×-Ñ-Ñ.ˆ�r0   c                 ó,  — | j                  | j                  |«      «      }| j                  |«      }||z  }| j                  |«      }t	        | j
                  j                  t        j                  «      r�|j                  | j
                  j                  j                  k7  r`| j
                  j                  j                  t        j                  k7  r/|j                  | j
                  j                  j                  «      }| j                  |«      }|S rX   )rU   ra   rb   rS   rY   rP   r)   r'   rZ   r:   r[   r5   )r+   r=   Úhidden_geluÚhidden_linears       r/   r?   zUMT5DenseGatedActDense.forwardp   s¿   € Ø—h‘h˜tŸy™y¨Ó7Ó8ˆØŸ	™	 -Ó0ˆØ# mÑ3ˆØŸ™ ]Ó3ˆô �t—w‘w—~‘~¤u§|¡|Ô4Ø×#Ñ# t§w¡w§~¡~×';Ñ';Ò;Ø—‘—‘×$Ñ$¬¯
©
Ò2à)×,Ñ,¨T¯W©W¯^©^×-AÑ-AÓBˆMàŸ™ Ó.ˆØÐr0   r]   rD   s   @r/   r_   r_   g   s   ø„ ð/˜zõ /ör0   r_   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚUMT5LayerFFrG   c                 ó  •— t         ‰| �  «        |j                  rt        |«      | _        nt        |«      | _        t        |j                  |j                  ¬«      | _	        t        j                  |j                  «      | _        y )N©r-   )r$   r%   Úis_gated_actr_   ÚDenseReluDenserF   r"   rM   Úlayer_norm_epsilonÚ
layer_normr   rQ   rR   rS   rV   s     €r/   r%   zUMT5LayerFF.__init__†   s_   ø€ Ü‰ÑÔØ×ÒÜ"8¸Ó"@ˆDÕä"3°FÓ";ˆDÔä'¨¯©¸F×<UÑ<UÔVˆŒÜ—z‘z &×"5Ñ"5Ó6ˆ�r0   c                 ór   — | j                  |«      }| j                  |«      }|| j                  |«      z   }|S rX   )rm   rk   rS   )r+   r=   Úforwarded_statess      r/   r?   zUMT5LayerFF.forward�   s=   € ØŸ?™?¨=Ó9ÐØ×.Ñ.Ð/?Ó@ÐØ%¨¯©Ð5EÓ(FÑFˆØÐr0   r]   rD   s   @r/   rg   rg   …   s   ø„ ð7˜zõ 7ör0   rg   c            
       óä   ‡ — e Zd ZdZddedz  fˆ fd„Zdej                  dej                  fd„Zd„ Z	dd	„Z
	 	 	 dd
ej                  dej                  dz  dedz  dej                  dz  fd„Zˆ xZS )ÚUMT5Attentionz7
    T5's attention using relative_attention_bias.
    NÚ	layer_idxc                 óÞ  •— t         ‰| �  «        |j                  | _        || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _
        |j                  | _        | j                  | j                  z  | _        || _        |€9| j                  r-t        j!                  d| j"                  j$                  › d�«       t'        j(                  | j                  | j                  d¬«      | _        t'        j(                  | j                  | j                  d¬«      | _        t'        j(                  | j                  | j                  d¬«      | _        t'        j(                  | j                  | j                  d¬«      | _        | j                  r0t'        j2                  | j                  | j                  «      | _        y y )NzInstantiating a decoder z³ without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.FrJ   )r$   r%   Ú
is_decoderÚhas_relative_attention_biasÚrelative_attention_num_bucketsÚrelative_attention_max_distancerM   Úd_kvÚkey_value_proj_dimÚ	num_headsÚn_headsrR   rS   Ú	inner_dimrr   ÚloggerÚwarning_oncer.   r@   r   rL   ÚqÚkÚvÚoÚ	EmbeddingÚrelative_attention_bias)r+   rG   ru   rr   r.   s       €r/   r%   zUMT5Attention.__init__œ   sd  ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØ+FˆÔ(Ø.4×.SÑ.SˆÔ+Ø/5×/UÑ/UˆÔ,Ø—~‘~ˆŒØ"(§+¡+ˆÔØ×'Ñ'ˆŒØ×*Ñ*ˆŒØŸ™¨×(?Ñ(?Ñ?ˆŒØ"ˆŒØÐ §¢Ü×ÑØ*¨4¯>©>×+BÑ+BÐ*Cð D,ð ,ôô —‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ>™>¨4¯<©<¸eÔDˆŒà×+Ò+Ü+-¯<©<¸×8[Ñ8[Ð]a×]iÑ]iÓ+jˆDÕ(ð ,r0   Ú
projectionÚreturnc                 ó¤   — |j                  «       d d | j                  | j                  fz   }|j                  |«      j	                  dddd«      }|S )Nr3   r   r2   r   r   )Úsizer{   ry   ÚviewÚpermute)r+   r…   Únew_projection_shapeÚnew_projections       r/   Ú_shapezUMT5Attention._shape·   sQ   € Ø)Ÿ™Ó0°°"Ð5¸¿¹Àt×G^ÑG^Ð8_Ñ_Ðà#Ÿ™Ð)=Ó>×FÑFÀqÈ!ÈQÐPQÓRˆØÐr0   c                 ó   — d}| j                   }| j                  }| j                  sC|dz  }||dkD  j                  t        j
                  «      |z  z  }t	        j                  |«      }n*t	        j                  |t	        j                  |«      «       }|dz  }||k  }t	        j                  |j                  «       |z  «      t        j                  ||z  «      z  }|||z
  z  }||j                  t        j
                  «      z   }t	        j                  |t	        j                  ||dz
  «      «      }|t	        j                  |||«      z  }|S )aÒ  
        Adapted from Mesh Tensorflow:
        https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593

        Translate relative position to a bucket number for relative attention. The relative position is defined as
        memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
        position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
        small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
        positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
        This should allow for more graceful generalization to longer sequences than the model has been trained on

        Args:
            relative_position: an int32 Tensor
            bidirectional: a boolean - whether the attention is bidirectional
            num_buckets: an integer
            max_distance: an integer

        Returns:
            a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
        r   r2   r   )rv   rw   rt   r5   r'   ÚlongÚabsÚminÚ
zeros_likeÚlogÚfloatÚmathÚ	full_likeÚwhere)	r+   Úrelative_positionÚrelative_bucketsÚnum_bucketsÚmax_distanceÚ	max_exactÚis_smallÚ	log_ratioÚrelative_position_if_larges	            r/   Ú_relative_position_bucketz'UMT5Attention._relative_position_bucket½   sA  € ð* ÐØ×9Ñ9ˆØ×;Ñ;ˆØ�ŠØ˜AÑˆKØÐ!2°QÑ!6× :Ñ :¼5¿:¹:Ó FÈÑ TÑTÐÜ %§	¡	Ð*;Ó <Ñä!&§¡Ð+<¼e×>NÑ>NÐO`Ó>aÓ!bÐ bÐð   1Ñ$ˆ	Ø$ yÑ0ˆô —I‘IÐ/×5Ñ5Ó7¸)ÑCÓDÄtÇxÁxÐP\Ð_hÑPhÓGiÑiˆ	Ø ¨yÑ!8Ñ9ˆ	Ø%.°·±¼e¿j¹jÓ1IÑ%IÐ"Ü%*§Y¡YØ&¬¯©Ð8RÐT_ÐbcÑTcÓ(dó&
Ð"ð 	œEŸK™K¨Ð2CÐE_Ó`Ñ`ÐØÐr0   c                 ó–  — |€ | j                   j                  j                  }t        j                  |t        j
                  |¬«      dd…df   |z   }t        j                  |t        j
                  |¬«      ddd…f   }||z
  }| j                  |«      }| j                  |«      }	|	j                  g d¢«      j                  d«      }	|	S )z%Compute binned relative position biasN)r:   Údevice)r2   r   r   r   )	r„   r)   r¢   r'   Úaranger�   r    rŠ   Ú	unsqueeze)
r+   Úquery_lengthÚ
key_lengthr¢   Úpast_seen_tokensÚcontext_positionÚmemory_positionr˜   Úrelative_position_bucketÚvaluess
             r/   Úcompute_biaszUMT5Attention.compute_biasì   s¶   € àˆ>Ø×1Ñ1×8Ñ8×?Ñ?ˆFÜ Ÿ<™<¨¼E¿J¹JÈvÔVÒWXÐZ^ÐW^Ñ_ÐbrÑrÐÜŸ,™, z¼¿¹ÈFÔSÐTXÒZ[ÐT[Ñ\ˆØ+Ð.>Ñ>ÐØ#'×#AÑ#AÐBSÓ#TÐ Ø×-Ñ-Ð.FÓGˆØ—‘¢	Ó*×4Ñ4°QÓ7ˆØˆr0   r=   Úencoder_hidden_statesÚpast_key_valuesÚattention_maskc                 ó^  — |j                   d d \  }}|�|j                  | j                  «      nd}t        |t        j
                  «      r|j                  «       n|}|d u}	| j                  |«      }
|
j                  |d| j                  | j                  «      j                  dd«      }
d}|�Qt        |t        «      rA|j                  j                  | j                  «      }|	r|j                  }n|j                   }n|}|	r|n|}|	rK|�I|rG|j"                  | j                     j$                  }|j"                  | j                     j&                  }nß| j)                  |«      }| j+                  |«      }|j                  |d| j                  | j                  «      j                  dd«      }|j                  |d| j                  | j                  «      j                  dd«      }|�K|j-                  ||| j                  «      \  }}|	r)t        |t        «      rd|j                  | j                  <   t	        j.                  |
|j                  dd«      «      }|j                   d   }| j0                  s;t	        j2                  d| j                  ||f|j4                  |j6                  ¬	«      }n| j9                  |||j4                  |¬
«      }|�||z   }|}||z  }t:        j<                  j?                  |jA                  «       d¬«      jC                  |«      }t:        j<                  jE                  || jD                  | jF                  ¬«      }t	        j.                  ||«      }|j                  dd«      jI                  «       }|j                  ||d«      }| jK                  |«      }||fS )Nr2   r   r3   r   FTr   éþÿÿÿ)r¢   r:   )r¢   r§   ©Údim)ÚpÚtraining)&ÚshapeÚget_seq_lengthrr   rY   r'   rZ   Úcloner   r‰   r{   ry   Ú	transposer   Ú
is_updatedÚgetÚcross_attention_cacheÚself_attention_cacheÚlayersÚkeysr«   r€   r�   ÚupdateÚmatmulru   Úzerosr¢   r:   r¬   r   Ú
functionalÚsoftmaxr”   Útype_asrS   rµ   Ú
contiguousr‚   )r+   r=   r­   r®   r¯   ÚkwargsÚ
batch_sizeÚ
seq_lengthr§   Úis_cross_attentionÚquery_statesrº   Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚscoresr¦   Úposition_biasÚposition_bias_maskedÚattn_weightsÚattn_outputs                         r/   r?   zUMT5Attention.forwardø   sW  € ð "/×!4Ñ!4°R°aÐ!8Ñˆ
�JØM\ÐMh˜?×9Ñ9¸$¿.¹.ÔIÐnoÐä7AÐBRÔTY×T`ÑT`Ô7aÐ+×1Ñ1Ô3ÐgwÐð 3¸$Ð>Ðà—v‘v˜mÓ,ˆØ#×(Ñ(¨°R¸¿¹Àt×G^ÑG^Ó_×iÑiÐjkÐmnÓoˆð ˆ
ØÐ&¬:°oÔGZÔ+[Ø(×3Ñ3×7Ñ7¸¿¹ÓGˆJÙ!à'6×'LÑ'LÑ$à'6×'KÑ'KÑ$à#2Ð á2DÑ.È-ˆÙ /Ð"=Á*à-×4Ñ4°T·^±^ÑD×IÑIˆJØ/×6Ñ6°t·~±~ÑF×MÑM‰LàŸ™ Ó/ˆJØŸ6™6 .Ó1ˆLØ#Ÿ™¨°R¸¿¹Àt×G^ÑG^Ó_×iÑiÐjkÐmnÓoˆJØ'×,Ñ,¨Z¸¸T¿\¹\È4×KbÑKbÓc×mÑmÐnoÐqrÓsˆLàÐ*à+?×+FÑ+FÀzÐS_Ðae×aoÑaoÓ+pÑ(�
˜Lá%¬*°_ÔFYÔ*ZØAE�O×.Ñ.¨t¯~©~Ñ>ô —‘˜l¨J×,@Ñ,@ÀÀAÓ,FÓGˆà×%Ñ% bÑ)ˆ
Ø×/Ò/Ü!ŸK™KØ�D—L‘L *¨jÐ9À&Ç-Á-ÐW]×WcÑWcô‰Mð !×-Ñ-Ø˜J¨v¯}©}ÐO_ð .ó ˆMð Ð%Ø)¨NÑ:ˆMà,ÐØÐ&Ñ&ˆô —}‘}×,Ñ,¨V¯\©\«^ÀÐ,ÓD×LÑLÈVÓTˆÜ—}‘}×,Ñ,¨\¸T¿\¹\ÐTX×TaÑTaÐ,Óbˆä—l‘l <°Ó>ˆà!×+Ñ+¨A¨qÓ1×<Ñ<Ó>ˆØ!×&Ñ& z°:¸rÓBˆà—f‘f˜[Ó)ˆØ˜LÐ(Ð(r0   )FN)Nr   ©NNN)r@   rA   rB   Ú__doc__Úintr%   r'   rZ   r�   r    r¬   r
   r?   rC   rD   s   @r/   rq   rq   —   s–   ø„ ññkÈSÐSWÉZõ kð6 §¡ð °%·,±,ó ò- ó^
ð 6:Ø(,Ø.2ñN)à—|‘|ðN)ð  %Ÿ|™|¨dÑ2ðN)ð  ™ð	N)ð
 Ÿ™ tÑ+÷N)r0   rq   c                   ó8   ‡ — e Zd Zddedz  fˆ fd„Z	 	 dd„Zˆ xZS )ÚUMT5LayerSelfAttentionNrr   c                 óÜ   •— t         ‰| �  «        t        |d|¬«      | _        t	        |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y )NT©ru   rr   ri   )r$   r%   rq   ÚSelfAttentionr"   rM   rl   rm   r   rQ   rR   rS   ©r+   rG   rr   r.   s      €r/   r%   zUMT5LayerSelfAttention.__init__J  sN   ø€ Ü‰ÑÔÜ*¨6ÈtÐ_hÔiˆÔÜ'¨¯©¸F×<UÑ<UÔVˆŒÜ—z‘z &×"5Ñ"5Ó6ˆ�r0   c                 ó�   — | j                  |«      }| j                  |||¬«      }|| j                  |d   «      z   }|f|dd  z   }|S )N©r¯   r®   r   r   )rm   rÜ   rS   )r+   r=   r¯   r®   rÇ   Únormed_hidden_statesÚattention_outputÚoutputss           r/   r?   zUMT5LayerSelfAttention.forwardP  sg   € ð  $Ÿ™¨}Ó=ÐØ×-Ñ-Ø Ø)Ø+ð .ó 
Ðð
 &¨¯©Ð5EÀaÑ5HÓ(IÑIˆØ Ð"Ð%5°a°bÐ%9Ñ9ˆØˆr0   rX   )NN©r@   rA   rB   r×   r%   r?   rC   rD   s   @r/   rÙ   rÙ   I  s    ø„ ñ7¨#°©*õ 7ð Ø÷	r0   rÙ   c                   ó:   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 dd„Zˆ xZS )ÚUMT5LayerCrossAttentionNrr   c                 óÜ   •— t         ‰| �  «        t        |d|¬«      | _        t	        |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y )NFrÛ   ri   )r$   r%   rq   ÚEncDecAttentionr"   rM   rl   rm   r   rQ   rR   rS   rÝ   s      €r/   r%   z UMT5LayerCrossAttention.__init__c  sO   ø€ Ü‰ÑÔÜ,¨VÐQVÐbkÔlˆÔÜ'¨¯©¸F×<UÑ<UÔVˆŒÜ—z‘z &×"5Ñ"5Ó6ˆ�r0   c                 ó’   — | j                  |«      }| j                  ||||¬«      }|| j                  |d   «      z   }|f|dd  z   }	|	S )N©r­   r¯   r®   r   r   )rm   rç   rS   )
r+   r=   r­   r¯   r®   rÇ   rà   rá   Úlayer_outputrâ   s
             r/   r?   zUMT5LayerCrossAttention.forwardi  si   € ð  $Ÿ™¨}Ó=ÐØ×/Ñ/Ø Ø"7Ø)Ø+ð	 0ó 
Ðð % t§|¡|Ð4DÀQÑ4GÓ'HÑHˆØ�/Ð$4°Q°RÐ$8Ñ8ˆØˆr0   rX   rÕ   rã   rD   s   @r/   rå   rå   b  s#   ø„ ñ7¨#°©*õ 7ð #ØØ÷r0   rå   c                   ó@   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 	 	 dd„Zˆ xZS )Ú	UMT5BlockNrr   c                 ón  •— t         ‰| �  «        |j                  | _        t        j                  «       | _        | j
                  j                  t        ||¬«      «       | j                  r&| j
                  j                  t        ||¬«      «       | j
                  j                  t        |«      «       y )N©rr   )
r$   r%   rt   r   Ú
ModuleListÚlayerÚappendrÙ   rå   rg   rÝ   s      €r/   r%   zUMT5Block.__init__~  sz   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒÜ—]‘]“_ˆŒ
Ø�
‰
×ÑÔ0°À9ÔMÔNØ�?Š?Ø�J‰J×ÑÔ5°fÈ	ÔRÔSà�
‰
×Ñœ+ fÓ-Õ.r0   c                 ó~  —  | j                   d   |||¬«      \  }}	|j                  t        j                  k(  r}t        j                  |j                  «      j
                  }
t        j                  t        j                  |«      j                  «       |
dz
  |
«      }t        j                  || |¬«      }d }| j                  xr |d u}|r¶ | j                   d   ||||¬«      \  }}|j                  t        j                  k(  r}t        j                  |j                  «      j
                  }
t        j                  t        j                  |«      j                  «       |
dz
  |
«      }t        j                  || |¬«      } | j                   d   |«      }|j                  t        j                  k(  r}t        j                  |j                  «      j
                  }
t        j                  t        j                  |«      j                  «       |
dz
  |
«      }t        j                  || |¬«      }|f}|r||	|fz  }|S )Nr   rß   iè  )r‘   Úmaxr   ré   r3   )rð   r:   r'   r;   Úfinforó   r—   ÚisinfÚanyÚclamprt   )r+   r=   r¯   r­   Úencoder_attention_maskr®   Ú	use_cacheÚoutput_attentionsrÇ   Úself_attn_weightsÚ	max_dtypeÚclamp_valueÚcross_attn_weightsÚdo_cross_attentionrâ   s                  r/   r?   zUMT5Block.forwardˆ  sì  € ð ,9¨4¯:©:°a©=ØØ)Ø+ô,
Ñ(ˆÐ(ð ×Ñ¤%§-¡-Ò/ÜŸ™ M×$7Ñ$7Ó8×<Ñ<ˆIÜŸ+™+¤e§k¡k°-Ó&@×&DÑ&DÓ&FÈ	ÐTXÑHXÐZcÓdˆKÜ!ŸK™K¨¸K¸<È[ÔYˆMð "ÐØ!Ÿ_™_ÒRÐ1FÈdÐ1RÐÙØ0=°·
±
¸1±ØØ&;Ø5Ø /ô	1Ñ-ˆMÐ-ð ×"Ñ"¤e§m¡mÒ3Ü!ŸK™K¨×(;Ñ(;Ó<×@Ñ@�	Ü#Ÿk™k¬%¯+©+°mÓ*D×*HÑ*HÓ*JÈIÐX\ÑL\Ð^gÓh�Ü %§¡¨MÀ¸|ÐQ\Ô ]�ð '˜Ÿ
™
 2™ }Ó5ˆð ×Ñ¤%§-¡-Ò/ÜŸ™ M×$7Ñ$7Ó8×<Ñ<ˆIÜŸ+™+¤e§k¡k°-Ó&@×&DÑ&DÓ&FÈ	ÐTXÑHXÐZcÓdˆKÜ!ŸK™K¨¸K¸<È[ÔYˆMà Ð"ˆáØÐ)Ð+=Ð>Ñ>ˆGàˆr0   rX   )NNNNFFrã   rD   s   @r/   rì   rì   }  s,   ø„ ñ/¨#°©*õ /ð Ø"Ø#ØØØ÷5r0   rì   c                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚUMT5ClassificationHeadz-Head for sentence-level classification tasks.rG   c                 ó(  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  ¬«      | _        t        j                  |j                  |j                  «      | _
        y )N)r´   )r$   r%   r   rL   rM   ÚdenserQ   Úclassifier_dropoutrS   Ú
num_labelsÚout_projrV   s     €r/   r%   zUMT5ClassificationHead.__init__Ä  sZ   ø€ Ü‰ÑÔÜ—Y‘Y˜vŸ~™~¨v¯~©~Ó>ˆŒ
Ü—z‘z F×$=Ñ$=Ô>ˆŒÜŸ	™	 &§.¡.°&×2CÑ2CÓDˆ�r0   r=   r†   c                 ó¸   — | j                  |«      }| j                  |«      }t        j                  |«      }| j                  |«      }| j	                  |«      }|S rX   )rS   r  r'   Útanhr  r\   s     r/   r?   zUMT5ClassificationHead.forwardÊ  sN   € ØŸ™ ]Ó3ˆØŸ
™
 =Ó1ˆÜŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ mÓ4ˆØÐr0   )
r@   rA   rB   rÖ   r    r%   r'   rZ   r?   rC   rD   s   @r/   r  r  Á  s/   ø„ Ù7ðE˜zõ Eð U§\¡\ð °e·l±l÷ r0   r  c                   ó€   ‡ — e Zd ZU eed<   dZdZdZdgZdgZ	e
d„ «       Z ej                  «       ˆ fd„«       Zd„ Zˆ xZS )	ÚUMT5PreTrainedModelrG   ÚtransformerTrì   rP   c                 óv   — t        j                  t        «      }t        j                  t        «      }|||dœ}|S )N)Údecoder_input_idsÚ	input_idsÚdecoder_attention_mask)r'   Útensorr   r   )r+   r  Ú
input_maskÚdummy_inputss       r/   r  z UMT5PreTrainedModel.dummy_inputsÝ  s6   € ä—L‘L¤Ó.ˆ	Ü—\‘\¤*Ó-ˆ
à!*Ø"Ø&0ñ
ˆð
 Ðr0   c                 ó’  •— t         ‰| �  |«       | j                  j                  }t	        |t
        «      r$t        j                  |j                  |dz  «       y	t	        |t        t        t        t        f«      rýt        j                  |j                  j                  d|dz  ¬«       t        |d«      rE| j                  j                   s/t        j                  |j"                  j                  d|dz  ¬«       t        |d«      rpt        j                  |j$                  j                  d|| j                  j&                  dz  z  ¬«       t        j(                  |j$                  j*                  «       y	y	t	        |t,        «      rft        |d«      rYt        j                  |j.                  j                  d|dz  ¬«       t        j(                  |j.                  j*                  «       y	y	t	        |t0        «      �r9t        j                  |j2                  j                  d|| j                  j&                  dz  z  ¬«       t        |j2                  d«      r?|j2                  j*                  �)t        j(                  |j2                  j*                  «       t        j                  |j4                  j                  d|| j                  j&                  dz  z  ¬«       t        |j4                  d«      rA|j4                  j*                  �*t        j(                  |j4                  j*                  «       y	y	y	t	        |t6        «      �r9t        j                  |j8                  j                  d|| j                  j&                  dz  z  ¬«       t        |j8                  d«      r?|j8                  j*                  �)t        j(                  |j8                  j*                  «       t        j                  |j:                  j                  d|| j                  j<                  dz  z  ¬«       t        |j:                  d«      rA|j:                  j*                  �*t        j(                  |j:                  j*                  «       y	y	y	t	        |t>        «      �rÔt        j                  |j@                  j                  d|| j                  j&                  dz  z  ¬«       t        |j@                  d«      r?|j@                  j*                  �)t        j(                  |j@                  j*                  «       t        j                  |jB                  j                  d|| j                  j&                  dz  z  ¬«       t        |jB                  d«      r?|jB                  j*                  �)t        j(                  |jB                  j*                  «       t        j                  |j:                  j                  d|| j                  j<                  dz  z  ¬«       t        |j:                  d«      rA|j:                  j*                  �*t        j(                  |j:                  j*                  «       y	y	y	t	        |tD        «      �rP| j                  j&                  }| j                  jF                  }| j                  jH                  }t        j                  |jJ                  j                  d|||z  dz  z  ¬«       t        j                  |jL                  j                  d||dz  z  ¬«       t        j                  |jN                  j                  d||dz  z  ¬«       t        j                  |jP                  j                  d|||z  dz  z  ¬«       |jR                  r3t        j                  |jT                  j                  d||dz  z  ¬«       y	y	y	)
zInitialize the weightsç      ð?g        )r8   ÚstdÚlm_headÚ
qa_outputsç      à¿Ú
classifierrK   N)+r$   Ú_init_weightsrG   Úinitializer_factorrY   r"   ÚinitÚ	constant_r)   Ú	UMT5ModelÚUMT5ForConditionalGenerationÚUMT5EncoderModelÚUMT5ForQuestionAnsweringÚnormal_ÚsharedÚhasattrÚtie_word_embeddingsr  r  rM   Úzeros_rK   ÚUMT5ForTokenClassificationr  r  r  r  rF   rO   rP   rN   r_   ra   rb   rq   rx   rz   r   r€   r�   r‚   ru   r„   )r+   ÚmoduleÚfactorrM   ry   r{   r.   s         €r/   r  z!UMT5PreTrainedModel._init_weightsè  sL  ø€ ô 	‰Ñ˜fÔ%Ø—‘×/Ñ/ˆÜ�fœmÔ,Ü�N‰N˜6Ÿ=™=¨&°3©,Õ7ÜØäÜ,Ü Ü(ð	ô
ô �L‰L˜Ÿ™×-Ñ-°C¸VÀc¹\ÕJÜ�v˜yÔ)°$·+±+×2QÒ2QÜ—‘˜VŸ^™^×2Ñ2¸À&È3Á,ÕOÜ�v˜|Ô,Ü—‘˜V×.Ñ.×5Ñ5¸CÀVÐPT×P[ÑP[×PcÑPcÐhlÑOlÑEmÕnÜ—‘˜F×-Ñ-×2Ñ2Õ3ð -ô ˜Ô :Ô;Ü�v˜|Ô,Ü—‘˜V×.Ñ.×5Ñ5¸CÀVÈcÁ\ÕRÜ—‘˜F×-Ñ-×2Ñ2Õ3ð -ô ˜Ô 6Õ7Ü�L‰L˜Ÿ™×,Ñ,°3¸FÀtÇ{Á{×GZÑGZÐ_cÑFcÑ<dÕeÜ�v—|‘| VÔ,°·±×1BÑ1BÐ1NÜ—‘˜FŸL™L×-Ñ-Ô.Ü�L‰L˜Ÿ™×/Ñ/°c¸vÈ$Ï+É+×J]ÑJ]ÐbfÑIfÑ?gÕhÜ�v—‘¨Ô/°F·O±O×4HÑ4HÐ4TÜ—‘˜FŸO™O×0Ñ0Õ1ð 5UÐ/ä˜Ô 1Õ2ô �L‰L˜Ÿ™×)Ñ)°¸ÀDÇKÁK×DWÑDWÐ\`ÑC`Ñ9aÕbÜ�v—y‘y &Ô)¨f¯i©i¯n©nÐ.HÜ—‘˜FŸI™IŸN™NÔ+Ü�L‰L˜Ÿ™×)Ñ)°¸ÀDÇKÁK×DTÑDTÐY]ÑC]Ñ9^Õ_Ü�v—y‘y &Ô)¨f¯i©i¯n©nÐ.HÜ—‘˜FŸI™IŸN™NÕ+ð /IÐ)ä˜Ô 6Õ7Ü�L‰L˜Ÿ™×+Ñ+°#¸6ÀdÇkÁk×FYÑFYÐ^bÑEbÑ;cÕdÜ�v—{‘{ FÔ+°·±×0@Ñ0@Ð0LÜ—‘˜FŸK™K×,Ñ,Ô-Ü�L‰L˜Ÿ™×+Ñ+°#¸6ÀdÇkÁk×FYÑFYÐ^bÑEbÑ;cÕdÜ�v—{‘{ FÔ+°·±×0@Ñ0@Ð0LÜ—‘˜FŸK™K×,Ñ,Ô-Ü�L‰L˜Ÿ™×)Ñ)°¸ÀDÇKÁK×DTÑDTÐY]ÑC]Ñ9^Õ_Ü�v—y‘y &Ô)¨f¯i©i¯n©nÐ.HÜ—‘˜FŸI™IŸN™NÕ+ð /IÐ)ä˜¤Õ.ð —k‘k×)Ñ)ˆGØ!%§¡×!1Ñ!1ÐØ—k‘k×+Ñ+ˆGÜ�L‰L˜Ÿ™Ÿ™¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÕjÜ�L‰L˜Ÿ™Ÿ™¨s¸À'È4Á-Ñ8PÕQÜ�L‰L˜Ÿ™Ÿ™¨s¸À'È4Á-Ñ8PÕQÜ�L‰L˜Ÿ™Ÿ™¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÕjØ×1Ò1Ü—‘˜V×;Ñ;×BÑBÈÐRXÐ]dÐimÑ\mÑRnÖoð 2ð /r0   c                 ó8  — | j                   j                  }| j                   j                  }|€t        d«      ‚|j	                  |j
                  «      }|dd d…f   j                  «       |ddd …f<   ||d<   |€t        d«      ‚|j                  |dk(  |«       |S )NzŽself.model.config.decoder_start_token_id has to be defined. In UMT5 it is usually set to the pad_token_id. See UMT5 docs for more information..r3   r   ).r   z1self.model.config.pad_token_id has to be defined.éœÿÿÿ)rG   Údecoder_start_token_idÚpad_token_idÚ
ValueErrorÚ	new_zerosr¶   r¸   Úmasked_fill_)r+   r  r,  r-  Úshifted_input_idss        r/   Ú_shift_rightz UMT5PreTrainedModel._shift_right,  s°   € Ø!%§¡×!CÑ!CÐØ—{‘{×/Ñ/ˆà!Ð)Üð6óð ð
 &×/Ñ/°	·±Ó@ÐØ%.¨s°C°R°C¨xÑ%8×%>Ñ%>Ó%@Ð˜#˜q™r˜'Ñ"Ø$:Ð˜&Ñ!àÐÜÐPÓQÐQà×&Ñ&Ð'8¸DÑ'@À,ÔOà Ð r0   )r@   rA   rB   r    Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_can_compile_fullgraphÚ_no_split_modulesÚ_keep_in_fp32_modulesÚpropertyr  r'   Úno_gradr  r2  rC   rD   s   @r/   r
  r
  Ó  sc   ø… àÓØ%ÐØ&*Ð#à!ÐØ$˜ÐØ!˜FÐàñó ðð €U‡]�]ƒ_óApó ðApöF!r0   r
  c                   ó@   ‡ — e Zd Zˆ fd„Zd„ Z	 	 	 	 	 	 	 	 	 	 dd„Zˆ xZS )Ú	UMT5Stackc           	      óü  •— t         ‰| �  |«       t        j                  |j                  |j
                  «      | _        |j                  | _        t        j                  t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ c}«      | _        t        |j
                  |j                  ¬«      | _        t        j                   |j"                  «      | _        d| _        | j)                  «        y c c}w )Nrî   ri   F)r$   r%   r   rƒ   Ú
vocab_sizerM   Úembed_tokensrt   rï   ÚrangeÚ
num_layersrì   Úblockr"   rl   Úfinal_layer_normrQ   rR   rS   Úgradient_checkpointingÚ	post_init)r+   rG   Úir.   s      €r/   r%   zUMT5Stack.__init__C  s±   ø€ Ü‰Ñ˜Ô ÜŸL™L¨×):Ñ):¸F¿N¹NÓKˆÔØ ×+Ñ+ˆŒÜ—]‘]ÌEÐRX×RcÑRcÓLdÖ#eÀq¤I¨fÀÖ$BÒ#eÓfˆŒ
Ü -¨f¯n©nÀ&×B[ÑB[Ô \ˆÔÜ—z‘z &×"5Ñ"5Ó6ˆŒð ',ˆÔ#Ø�‰Õùò $fs   Á7C9c                 ó   — || _         y rX   )r?  ©r+   Únew_embeddingss     r/   Úset_input_embeddingszUMT5Stack.set_input_embeddingsO  s
   € Ø*ˆÕr0   c           
      ó~  — |�|n| j                   j                  }|�|n| j                   j                  }|	�|	n| j                   j                  }	|
�|
n| j                   j                  }
|�$|�"| j
                  rdnd}t        d|› d|› d�«      ‚|�&|j                  «       }|j                  d|d   «      }n8|�|j                  «       d d }n"| j
                  rdnd}t        d|› d|› d	�«      ‚| j                  r%| j                  r|rt        j                  d
«       d}|€(| j                  €t        d«      ‚| j                  |«      }|\  }}|du r| j
                  st        d| › d�«      ‚| j
                  rf|rr|€p| j                   j                  r5t        t!        | j                   ¬«      t!        | j                   ¬«      «      }n%t!        | j                   ¬«      }n| j
                  sd }|�|j#                  «       nd}|€1t%        «       s'||z   }t'        j(                  |||j*                  ¬«      }| j
                  rt-        | j                   |||¬«      }n]|�Y|d d …d d d d …f   }|j/                  |j0                  ¬«      }d|z
  t'        j2                  |j0                  «      j4                  z  }nd }| j
                  r|�t7        | j                   |||¬«      }nd }|	rdnd }|rdnd }|r| j
                  rdnd }| j9                  |«      }t;        | j<                  «      D ]C  \  }}|	r||fz   } ||||||||¬«      }|d   }|sŒ%||d   fz  }| j
                  sŒ;||d   fz  }ŒE | j?                  |«      }| j9                  |«      }|	r||fz   }|
stA        d„ |||||fD «       «      S tC        |||||¬«      S )NÚdecoder_Ú zYou cannot specify both zinput_ids and zinputs_embeds at the same timer3   zYou have to specify either zinput_ids or Úinputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fz<You have to initialize the model with valid token embeddingsTz)`use_cache` can only be set to `True` if z is used as a decoder)rG   r   )r¢   )rG   rN  r¯   r®   )r:   r  )rG   rN  r¯   r­   © )rø   r®   rù   rú   r   r2   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wrX   rO  )Ú.0r�   s     r/   ú	<genexpr>z$UMT5Stack.forward.<locals>.<genexpr>Ö  s   è ø€ ò 
àð �=ô ñ
ùs   ‚)Úlast_hidden_stater®   r=   Ú
attentionsÚcross_attentions)"rG   rù   rú   Úoutput_hidden_statesÚreturn_dictrt   r.  rˆ   r‰   rD  rµ   r}   r~   r?  Úis_encoder_decoderr   r   r·   r   r'   r(   r¢   r   r5   r:   rô   r‘   r   rS   Ú	enumeraterB  rC  Útupler   )r+   r  r¯   r­   rø   rN  r®   rù   rú   rV  rW  rÇ   Úerr_msg_prefixÚinput_shaperÈ   rÉ   Úpast_key_values_lengthÚmask_seq_lengthÚcausal_maskÚencoder_extended_attention_maskÚall_hidden_statesÚall_attentionsÚall_cross_attentionsr=   rF  Úlayer_moduleÚlayer_outputss                              r/   r?   zUMT5Stack.forwardR  s   € ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø1BÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐ  ]Ð%>Ø+/¯?ª?™ZÀˆNÜØ*¨>Ð*:¸.ÈÐHXÐXvÐwóð ð Ð"Ø#Ÿ.™.Ó*ˆKØ!Ÿ™ r¨;°r©?Ó;‰IØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰Kà+/¯?ª?™ZÀˆNÜÐ:¸>Ð:JÈ-ÐXfÐWgÐgtÐuÓvÐvà×&Ò&¨4¯=ª=ÙÜ×#Ñ#Øpôð "�	àÐ Ø× Ñ Ð(Ü Ð!_Ó`Ð`Ø ×-Ñ-¨iÓ8ˆMà!,Ñˆ
�Jà˜ÑØ—?’?Ü Ð#LÈTÈFÐRgÐ!hÓiÐið �?Š?Ù˜_Ð4Ø—;‘;×1Ò1Ü&9Ü$¨D¯K©KÔ8¼,ÈdÏkÉkÔ:Zó'‘Oô '3¸$¿+¹+Ô&F‘OØ—’ð #ˆOàETÐE` ×!?Ñ!?Ô!AÐfgÐØÐ!Ô*BÔ*Dà4°zÑAˆOÜ"ŸZ™Z¨
°OÈM×L`ÑL`ÔaˆNà�?Š?Ü,Ø—{‘{Ø+Ø-Ø /ô	‰Kð Ð'Ø(ª¨D°$ºÐ)9Ñ:ˆKØ%Ÿ.™.¨}×/BÑ/B˜.ÓCˆKØ Ñ,´·±¸M×<OÑ<OÓ0P×0TÑ0TÑT‰KàˆKà�?Š?Ð5ÐAÜ.GØ—{‘{Ø+Ø5Ø&;ô	/Ñ+ð /3Ð+á"6™B¸DÐÙ0™°dˆÙ%6¸4¿?º?™rÐPTÐàŸ™ ]Ó3ˆä(¨¯©Ó4ò 	@‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!á(ØØØ%Ø'FØ /Ø#Ø"3ôˆMð *¨!Ñ,ˆMâ Ø =°Ñ#3Ð"5Ñ5�Ø—?“?Ø(¨]¸1Ñ-=Ð,?Ñ?Ñ(ð'	@ð* ×-Ñ-¨mÓ<ˆØŸ™ ]Ó3ˆñ  Ø 1°]Ð4DÑ DÐáÜñ 
ð "Ø#Ø%Ø"Ø(ðô
ó 
ð 
ô 9Ø+Ø+Ø+Ø%Ø1ô
ð 	
r0   )
NNNNNNNNNN)r@   rA   rB   r%   rJ  r?   rC   rD   s   @r/   r<  r<  B  s3   ø„ ô
ò+ð
 ØØ"Ø#ØØØØØ!Ø÷U
r0   r<  c                   ó¨  ‡ — e Zd ZU dZdZeed<   dddœZˆ fd„Zd„ Z	d„ Z
e	 	 	 	 	 	 	 	 	 	 	 	 dd
ej                  d	z  dej                  d	z  dej                  d	z  dej                  d	z  deeej                        d	z  ded	z  dej$                  d	z  dej$                  d	z  ded	z  ded	z  ded	z  ded	z  deej                     ez  fd„«       Zˆ xZS )r  ao  
    Examples:

    ```python
    >>> from transformers import UMT5Model, AutoTokenizer

    >>> model = UMT5Model.from_pretrained("google/umt5-small")
    >>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
    >>> noisy_text = "UN Offizier sagt, dass weiter <extra_id_0> werden muss in Syrien."
    >>> label = "<extra_id_0> verhandelt"
    >>> inputs = tokenizer(inputs, return_tensors="pt")
    >>> labels = tokenizer(label=label, return_tensors="pt")

    >>> outputs = model(input_ids=inputs["input_ids"], decoder_input_ids=labels["input_ids"])
    >>> hidden_states = outputs.last_hidden_state
    ```Úumt5rG   úshared.weight©úencoder.embed_tokens.weightúdecoder.embed_tokens.weightc                 ó‚  •— t         ‰| �  |«       t        j                  |j                  |j
                  «      | _        t        j                  |«      }d|_	        d|_
        t        |«      | _        t        j                  |«      }d|_	        |j                  |_        t        |«      | _        | j!                  «        y ©NFT)r$   r%   r   rƒ   r>  rM   r#  ÚcopyÚdeepcopyrt   rù   r<  ÚencoderÚnum_decoder_layersrA  ÚdecoderrE  ©r+   rG   Úencoder_configÚdecoder_configr.   s       €r/   r%   zUMT5Model.__init__  s’   ø€ Ü‰Ñ˜Ô Ü—l‘l 6×#4Ñ#4°f·n±nÓEˆŒäŸ™ vÓ.ˆØ$)ˆÔ!Ø#(ˆÔ Ü  Ó0ˆŒäŸ™ vÓ.ˆØ$(ˆÔ!Ø$*×$=Ñ$=ˆÔ!Ü  Ó0ˆŒð 	�‰Õr0   c                 ó   — | j                   S rX   ©r#  ©r+   s    r/   Úget_input_embeddingszUMT5Model.get_input_embeddings  ó   € Ø�{‰{Ðr0   c                 ó~   — || _         | j                  j                  |«       | j                  j                  |«       y rX   ©r#  rp  rJ  rr  rH  s     r/   rJ  zUMT5Model.set_input_embeddings  ó-   € Ø$ˆŒØ�‰×)Ñ)¨.Ô9Ø�‰×)Ñ)¨.Õ9r0   Nr  r¯   r  r  Úencoder_outputsr®   rN  Údecoder_inputs_embedsrù   rú   rV  rW  r†   c                 óD  — |	�|	n| j                   j                  }	|�|n| j                   j                  }|€| j                  ||||
||¬«      }nI|rGt	        |t
        «      s7t        |d   t        |«      dkD  r|d   ndt        |«      dkD  r|d   nd¬«      }|d   }| j                  |||||||	|
||¬«
      }|s||z   S t        |j                  |j                  |j                  |j                  |j                  |j                  |j                  |j                  ¬«      S )	ah
  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5
            Training](./umt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, UMT5Model

        >>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
        >>> model = UMT5Model.from_pretrained("google/umt5-small")

        >>> input_ids = tokenizer(
        ...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids  # Batch size 1

        >>> # preprocess: Prepend decoder_input_ids with start token which is pad token for UMT5Model.
        >>> # This is not needed for torch's UMT5ForConditionalGeneration as it does this internally using labels arg.
        >>> decoder_input_ids = model._shift_right(decoder_input_ids)

        >>> # forward pass
        >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        ```N©r  r¯   rN  rú   rV  rW  r   r   r2   ©rS  r=   rT  ©
r  r¯   rN  r®   r­   rø   rù   rú   rV  rW  )rS  r®   Údecoder_hidden_statesÚdecoder_attentionsrU  Úencoder_last_hidden_stater­   Úencoder_attentions)rG   rù   rW  rp  rY   r   Úlenrr  r   rS  r®   r=   rT  rU  )r+   r  r¯   r  r  r~  r®   rN  r  rù   rú   rV  rW  rÇ   r=   Údecoder_outputss                   r/   r?   zUMT5Model.forward  sO  € ðD "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ð%<‘kÀ$Ç+Á+×BYÑBYˆð Ð"Ø"Ÿl™lØ#Ø-Ø+Ø"3Ø%9Ø'ð +ó ‰Oñ ¤¨O¼_Ô!MÜ-Ø"1°!Ñ"4Ü47¸Ó4HÈ1Ò4L˜o¨aÒ0ÐRVÜ14°_Ó1EÈÒ1I˜?¨1Ò-ÈtôˆOð (¨Ñ*ˆð Ÿ,™,Ø'Ø1Ø/Ø+Ø"/Ø#1ØØ/Ø!5Ø#ð 'ó 
ˆñ Ø" _Ñ4Ð4ä!Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
r0   ©NNNNNNNNNNNN)r@   rA   rB   rÖ   Ú
model_typer    r3  Ú_tied_weights_keysr%   ry  rJ  r   r'   Ú
LongTensorÚFloatTensorÚ
BoolTensorrZ  r
   rZ   Úboolr   r?   rC   rD   s   @r/   r  r  ê  ss  ø… ñð" €JØÓà'6Ø'6ñÐô
ò$ò:ð
 ð .2Ø37Ø59Ø:>ØBFØ(,Ø-1Ø59Ø!%Ø)-Ø,0Ø#'ñq
à×#Ñ# dÑ*ðq
ð ×)Ñ)¨DÑ0ðq
ð !×+Ñ+¨dÑ2ð	q
ð
 !&× 0Ñ 0°4Ñ 7ðq
ð ˜u U×%6Ñ%6Ñ7Ñ8¸4Ñ?ðq
ð  ™ðq
ð —|‘| dÑ*ðq
ð  %Ÿ|™|¨dÑ2ðq
ð ˜$‘;ðq
ð   $™;ðq
ð # T™kðq
ð ˜D‘[ðq
ð 
ˆu× Ñ Ñ	!Ð$6Ñ	6òq
ó ôq
r0   r  z<
    UMT5 Model with a `language modeling` head on top.
    )Úcustom_introc                   óÞ  ‡ — e Zd ZdZdZddddœZˆ fd„Zd„ Zd„ Ze		 	 	 	 	 	 	 	 	 	 	 	 	 dd	e
j                  dz  d
e
j                  dz  de
j                  dz  de
j                  dz  deee
j                        dz  dedz  de
j                  dz  de
j                  dz  de
j                  dz  dedz  dedz  dedz  dedz  dee
j                     ez  fd„«       Zde
j                  fd„Zˆ xZS )r  a  
    Examples:

    ```python
    >>> from transformers import UMT5ForConditionalGeneration, AutoTokenizer

    >>> model = UMT5ForConditionalGeneration.from_pretrained("google/umt5-small")
    >>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
    >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien."
    >>> summary = "Weiter Verhandlung in Syrien."
    >>> inputs = tokenizer(article, text_target=summary, return_tensors="pt")

    >>> outputs = model(**inputs)
    >>> loss = outputs.loss
    ```rg  rh  )rj  rk  zlm_head.weightc                 ó  •— t         ‰| �  |«       |j                  | _        t	        j
                  |j                  |j                  «      | _        t        j                  |«      }d|_
        d|_        t        |«      | _        t        j                  |«      }d|_
        |j                  |_        t        |«      | _        t	        j"                  |j                  |j                  d¬«      | _        | j'                  «        y )NFTrJ   )r$   r%   rM   Ú	model_dimr   rƒ   r>  r#  rn  ro  rt   rù   r<  rp  rq  rA  rr  rL   r  rE  rs  s       €r/   r%   z%UMT5ForConditionalGeneration.__init__±  s»   ø€ Ü‰Ñ˜Ô ØŸ™ˆŒä—l‘l 6×#4Ñ#4°f·n±nÓEˆŒäŸ™ vÓ.ˆØ$)ˆÔ!Ø#(ˆÔ Ü  Ó0ˆŒäŸ™ vÓ.ˆØ$(ˆÔ!Ø$*×$=Ñ$=ˆÔ!Ü  Ó0ˆŒä—y‘y §¡°×1BÑ1BÈÔOˆŒð 	�‰Õr0   c                 ó   — | j                   S rX   rw  rx  s    r/   ry  z1UMT5ForConditionalGeneration.get_input_embeddingsÇ  rz  r0   c                 ó~   — || _         | j                  j                  |«       | j                  j                  |«       y rX   r|  rH  s     r/   rJ  z1UMT5ForConditionalGeneration.set_input_embeddingsË  r}  r0   Nr  r¯   r  r  r~  r®   rN  r  Úlabelsrù   rú   rV  rW  r†   c                 óÂ  — |
�|
n| j                   j                  }
|�|n| j                   j                  }|€| j                  ||||||¬«      }nI|rGt	        |t
        «      s7t        |d   t        |«      dkD  r|d   ndt        |«      dkD  r|d   nd¬«      }|d   }|	�|€|€| j                  |	«      }| j                  |||||||
|||¬«
      }|d   }| j                   j                  r|| j                  dz  z  }| j                  |«      }d}|	�^t        d	¬
«      }|	j                  |j                  «      }	 ||j                  d|j!                  d«      «      |	j                  d«      «      }|s|f|dd z   |z   }|�|f|z   S |S t#        |||j$                  |j&                  |j(                  |j*                  |j,                  |j&                  |j(                  ¬«	      S )aˆ  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5
            Training](./umt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
            config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
            labels in `[0, ..., config.vocab_size]`

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, UMT5ForConditionalGeneration

        >>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
        >>> model = UMT5ForConditionalGeneration.from_pretrained("google/umt5-small")

        >>> # training
        >>> input_ids = tokenizer("The <extra_id_0> walks in <extra_id_1> park", return_tensors="pt").input_ids
        >>> labels = tokenizer("<extra_id_0> cute dog <extra_id_1> the <extra_id_2>", return_tensors="pt").input_ids
        >>> outputs = model(input_ids=input_ids, labels=labels)
        >>> loss = outputs.loss
        >>> logits = outputs.logits

        >>> # inference
        >>> input_ids = tokenizer("Studies have shown that <extra_id_0> good for you", return_tensors="pt").input_ids
        >>> outputs = model.generate(input_ids)
        >>> tokenizer.decode(outputs[0], skip_special_tokens=True)
        ```Nr�  r   r   r2   r‚  rƒ  r  r+  ©Úignore_indexr3   ©	ÚlossÚlogitsr®   r„  r…  rU  r†  r­   r‡  )rG   rù   rW  rp  rY   r   rˆ  r2  rr  r%  r”  r  r   r5   r¢   r‰   rˆ   r   r®   r=   rT  rU  rS  )r+   r  r¯   r  r  r~  r®   rN  r  r—  rù   rú   rV  rW  rÇ   r=   r‰  Úsequence_outputÚ	lm_logitsrœ  Úloss_fctÚoutputs                         r/   r?   z$UMT5ForConditionalGeneration.forwardÐ  s  € ðL "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ð%<‘kÀ$Ç+Á+×BYÑBYˆð Ð"à"Ÿl™lØ#Ø-Ø+Ø"3Ø%9Ø'ð +ó ‰Oñ ¤¨O¼_Ô!MÜ-Ø"1°!Ñ"4Ü47¸Ó4HÈ1Ò4L˜o¨aÒ0ÐRVÜ14°_Ó1EÈÒ1I˜?¨1Ò-ÈtôˆOð (¨Ñ*ˆàÐÐ"3Ð";Ð@UÐ@]à $× 1Ñ 1°&Ó 9Ðð Ÿ,™,Ø'Ø1Ø/Ø+Ø"/Ø#1ØØ/Ø!5Ø#ð 'ó 
ˆð *¨!Ñ,ˆà�;‰;×*Ò*ð .°·±ÀÑ1EÑFˆOà—L‘L Ó1ˆ	àˆØÐÜ'°TÔ:ˆHà—Y‘Y˜y×/Ñ/Ó0ˆFÙ˜IŸN™N¨2¨y¯~©~¸bÓ/AÓBÀFÇKÁKÐPRÃOÓTˆDáØ�\ O°A°BÐ$7Ñ7¸/ÑIˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEäØØØ+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô

ð 
	
r0   c                 ó$   — | j                  |«      S rX   )r2  )r+   r—  s     r/   Ú%prepare_decoder_input_ids_from_labelszBUMT5ForConditionalGeneration.prepare_decoder_input_ids_from_labels`  s   € Ø× Ñ  Ó(Ð(r0   ©NNNNNNNNNNNNN)r@   rA   rB   rÖ   r‹  rŒ  r%   ry  rJ  r   r'   r�  rŽ  r�  rZ  rZ   r
   r�  r   r?   r£  rC   rD   s   @r/   r  r  ”  s›  ø„ ñð  €Jà'6Ø'6Ø)ñÐôò,ò:ð
 ð .2Ø37Ø59Ø:>Ø=AØ(,Ø26Ø:>Ø*.Ø!%Ø)-Ø,0Ø#'ñL
à×#Ñ# dÑ*ðL
ð ×)Ñ)¨DÑ0ðL
ð !×+Ñ+¨dÑ2ð	L
ð
 !&× 0Ñ 0°4Ñ 7ðL
ð ˜u U§\¡\Ñ2Ñ3°dÑ:ðL
ð  ™ðL
ð ×(Ñ(¨4Ñ/ðL
ð  %×0Ñ0°4Ñ7ðL
ð × Ñ  4Ñ'ðL
ð ˜$‘;ðL
ð   $™;ðL
ð # T™kðL
ð ˜D‘[ðL
ð  
ˆu× Ñ Ñ	! OÑ	3ò!L
ó ðL
ð^)¸E¿L¹L÷ )r0   r  c                   óö   ‡ — e Zd ZdZdZddiZˆ fd„Zd„ Zd„ Ze		 	 	 	 	 	 dd	e
j                  dz  d
e
j                  dz  de
j                  dz  dedz  dedz  dedz  dee
j                     ez  fd„«       Zˆ xZS )r   aâ  
    Examples:

    ```python
    >>> from transformers import UMT5EncoderModel, AutoTokenizer

    >>> model = UMT5EncoderModel.from_pretrained("google/umt5-small")
    >>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
    >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien."
    >>> input_ids = tokenizer(article, return_tensors="pt").input_ids
    >>> outputs = model(input_ids)
    >>> hidden_state = outputs.last_hidden_state
    ```rg  rj  rh  c                 ó  •— t         ‰| �  |«       t        j                  |j                  |j
                  «      | _        t        j                  |«      }d|_	        d|_
        t        |«      | _        | j                  «        y )NF)r$   r%   r   rƒ   r>  rM   r#  rn  ro  rù   rX  r<  rp  rE  )r+   rG   rt  r.   s      €r/   r%   zUMT5EncoderModel.__init__z  sb   ø€ Ü‰Ñ˜Ô Ü—l‘l 6×#4Ñ#4°f·n±nÓEˆŒäŸ™ vÓ.ˆØ#(ˆÔ Ø,1ˆÔ)Ü  Ó0ˆŒð 	�‰Õr0   c                 ó   — | j                   S rX   rw  rx  s    r/   ry  z%UMT5EncoderModel.get_input_embeddings‡  rz  r0   c                 óH   — || _         | j                  j                  |«       y rX   )r#  rp  rJ  rH  s     r/   rJ  z%UMT5EncoderModel.set_input_embeddings‹  s   € Ø$ˆŒØ�‰×)Ñ)¨.Õ9r0   Nr  r¯   rN  rú   rV  rW  r†   c                 óh   — |�|n| j                   j                  }| j                  ||||||¬«      }|S )aQ  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).

        Example:

        ```python
        >>> from transformers import AutoTokenizer, UMT5EncoderModel

        >>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
        >>> model = UMT5EncoderModel.from_pretrained("google/umt5-small")
        >>> input_ids = tokenizer(
        ...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> outputs = model(input_ids=input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        ```r�  )rG   rW  rp  )	r+   r  r¯   rN  rú   rV  rW  rÇ   r~  s	            r/   r?   zUMT5EncoderModel.forward�  sH   € ðF &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàŸ,™,ØØ)Ø'Ø/Ø!5Ø#ð 'ó 
ˆð Ðr0   )NNNNNN)r@   rA   rB   rÖ   r‹  rŒ  r%   ry  rJ  r   r'   r�  rŽ  r�  rZ  r   r?   rC   rD   s   @r/   r   r   d  sÖ   ø„ ñð €Jð 	& ðÐô
òò:ð ð .2Ø37Ø26Ø)-Ø,0Ø#'ñ,à×#Ñ# dÑ*ð,ð ×)Ñ)¨DÑ0ð,ð ×(Ñ(¨4Ñ/ð	,ð
   $™;ð,ð # T™kð,ð ˜D‘[ð,ð 
ˆu× Ñ Ñ	! OÑ	3ò,ó ô,r0   r   z…
    UMT5 model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE
    tasks.
    c                   ó~  ‡ — e Zd ZdgZdefˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e
ej                     dz  d
ej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  dedz  deez  fd„«       Zˆ xZS )ÚUMT5ForSequenceClassificationúFdecoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weightrG   c                 ó„   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        | j                  «        y rX   )r$   r%   r  r  r  Úclassification_headrE  rV   s     €r/   r%   z&UMT5ForSequenceClassification.__init__Ê  s6   ø€ Ü‰Ñ˜Ô Ü$ VÓ,ˆÔÜ#9¸&Ó#AˆÔ ð 	�‰Õr0   Nr  r¯   r  r  r~  rN  r  r—  rù   rú   rV  rW  r†   c                 óR  — |�|n| j                   j                  }|�d}	|€$|�"t        d| j                  j                  › �«      ‚|€ |€|€t        d«      ‚| j                  |«      }| j                  ||||||||	|
||¬«      }|d   }|j                  | j                   j                  «      j                  |j                  «      }t        t        j                  |j                  d«      «      j!                  «       dk(  d«       |j"                  \  }}}||dd…f   }t        |j"                  d   |z  dk\  d	«       |j%                  |d
|«      dd…d
dd…f   }| j'                  |«      }d}|��Ê|j                  |j                  «      }| j                   j(                  €¡| j                   j*                  dk(  rd| j                   _        nv| j                   j*                  dkD  rL|j,                  t        j.                  k(  s|j,                  t        j0                  k(  rd| j                   _        nd| j                   _        | j                   j(                  dk(  rSt3        «       }| j                   j*                  dk(  r& ||j5                  «       |j5                  «       «      }n– |||«      }nŒ| j                   j(                  dk(  rGt7        «       } ||j%                  d
| j                   j*                  «      |j%                  d
«      «      }n,| j                   j(                  dk(  rt9        «       } |||«      }|s|f|dd z   }|�|f|z   S |S t;        |||j<                  |j>                  |j@                  |jB                  |jD                  |jF                  |jH                  ¬«	      S )a°  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5
            Training](./umt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NFz8Passing input embeddings is currently not supported for ú°If no `decoder_input_ids` or `decoder_inputs_embeds` are passed, `input_ids` cannot be `None`. Please pass either `input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`.)
r¯   r  r  r~  rN  r  rù   rú   rV  rW  r   r   z7All examples must have the same number of <eos> tokens.z3Each example must contain at least one <eos> token.r3   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr›  )%rG   rW  ÚNotImplementedErrorr.   r@   r.  r2  r  ÚeqÚeos_token_idr5   r¢   r   r'   Úunique_consecutiveÚsumÚnumelr¶   r‰   r®  Úproblem_typer  r:   r�   r×   r   Úsqueezer   r   r   r®   r„  r…  rU  r†  r­   r‡  )r+   r  r¯   r  r  r~  rN  r  r—  rù   rú   rV  rW  rÇ   râ   rž  Úeos_maskrÈ   Ú_r,   ÚselectedÚsentence_representationr�  rœ  r   r¡  s                             r/   r?   z%UMT5ForSequenceClassification.forwardÒ  sp  € ð` &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆØÐØˆIàÐ Ð!:Ü%ØJÈ4Ï>É>×KbÑKbÐJcÐdóð ð Ð$Ð)>Ð)FØÐ Ü ðUóð ð
 !%× 1Ñ 1°)Ó <Ðà×"Ñ"ØØ)Ø/Ø#9Ø+Ø'Ø"7ØØ/Ø!5Ø#ð #ó 
ˆð " !™*ˆà—<‘< §¡× 8Ñ 8Ó9×<Ñ<¸_×=SÑ=SÓTˆäÜ×$Ñ$ X§\¡\°!£_Ó5×;Ñ;Ó=ÀÑBØEô	
ð &5×%:Ñ%:Ñ"ˆ
�A�{Ø" 8ªQ ;Ñ/ˆÜØ�N‰N˜1Ñ Ñ+¨qÑ0ØAô	
ð #+§-¡-°
¸BÀÓ"LÊQÐPRÒTUÈXÑ"VÐØ×)Ñ)Ð*AÓBˆàˆØÑØ—Y‘Y˜vŸ}™}Ó-ˆFØ�{‰{×'Ñ'Ð/Ø—;‘;×)Ñ)¨QÒ.Ø/;�D—K‘KÕ,Ø—[‘[×+Ñ+¨aÒ/°V·\±\ÄUÇZÁZÒ5OÐSY×S_ÑS_Ôch×clÑclÒSlØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—;‘;×)Ñ)¨QÒ.Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±×0FÑ0FÓ GÈÏÉÐUWËÓY‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ÙØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä.ØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ô

ð 
	
r0   rŠ  )r@   rA   rB   Ú"_keys_to_ignore_on_load_unexpectedr    r%   r   r'   r�  rZ   ÚlistrŽ  r�  rZ  r   r?   rC   rD   s   @r/   r«  r«  À  sU  ø„ ð +sÐ)sÐ&ð˜zõ ð ð .2Ø.2Ø59Ø:>Ø:>Ø26Ø:>Ø*.Ø!%Ø)-Ø,0Ø#'ñF
à×#Ñ# dÑ*ðF
ð Ÿ™ tÑ+ðF
ð !×+Ñ+¨dÑ2ð	F
ð
 !&× 0Ñ 0°4Ñ 7ðF
ð ˜e×/Ñ/Ñ0°4Ñ7ðF
ð ×(Ñ(¨4Ñ/ðF
ð  %×0Ñ0°4Ñ7ðF
ð × Ñ  4Ñ'ðF
ð ˜$‘;ðF
ð   $™;ðF
ð # T™kðF
ð ˜D‘[ðF
ð 
Ð0Ñ	0òF
ó ôF
r0   r«  c                   ó  ‡ — e Zd ZdgZdefˆ fd„Ze	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	dz  d
e	dz  de	dz  de
ej                     ez  fd„«       Zˆ xZS )r'  r¬  rG   c                 ó,  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j                  «        y rX   )r$   r%   r  r   r  r   rQ   r  rS   rL   r,   r  rE  rV   s     €r/   r%   z#UMT5ForTokenClassification.__init__a  sj   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä+¨FÓ3ˆÔÜ—z‘z &×";Ñ";Ó<ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr0   Nr  r¯   rN  r—  rú   rV  rW  r†   c                 óž  — |�|n| j                   j                  }| j                  ||||||¬«      }	|	d   }
| j                  |
«      }
| j	                  |
«      }d}|�<t        «       } ||j                  d| j                  «      |j                  d«      «      }|s||	dd f}|�|f|z   S |S t        |||	j                  |	j                  ¬«      S )aB  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        N)r¯   rN  rú   rV  rW  r   r3   r2   )rœ  r�  r=   rT  )rG   rW  r  rS   r  r   r‰   r  r   r=   rT  )r+   r  r¯   rN  r—  rú   rV  rW  rÇ   râ   r=   r�  rœ  r   r¡  s                  r/   r?   z"UMT5ForTokenClassification.forwardl  sí   € ð6 &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà×"Ñ"ØØ)Ø'Ø/Ø!5Ø#ð #ó 
ˆð   ™
ˆØŸ™ ]Ó3ˆØ—‘ Ó/ˆàˆØÐÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDáØ˜g a¨˜mÐ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r0   )NNNNNNN)r@   rA   rB   rÀ  r    r%   r   r'   rZ   r�  rZ  r   r?   rC   rD   s   @r/   r'  r'  \  sÐ   ø„ à*rÐ)sÐ&ð	˜zõ 	ð ð *.Ø.2Ø-1Ø&*Ø)-Ø,0Ø#'ñ6
à—<‘< $Ñ&ð6
ð Ÿ™ tÑ+ð6
ð —|‘| dÑ*ð	6
ð
 —‘˜tÑ#ð6
ð   $™;ð6
ð # T™kð6
ð ˜D‘[ð6
ð 
ˆu�|‰|Ñ	Ð4Ñ	4ò6
ó ô6
r0   r'  c                   óÈ  ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  deeej                        dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  dedz  deej                     ez  fd„«       Zˆ xZS )r!  rh  ri  c                 ó$  •— t         ‰| �  |«       |j                  | _        t	        j
                  |j                  |j                  «      | _        t        j                  |«      }d|_
        d|_        t        |«      | _        t        j                  |«      }d|_
        |j                  |_        t        |«      | _        |j"                  | _        t	        j$                  |j                  |j"                  «      | _        | j)                  «        y rm  )r$   r%   rM   r”  r   rƒ   r>  r#  rn  ro  rt   rù   r<  rp  rq  rA  rr  r  rL   r  rE  rs  s       €r/   r%   z!UMT5ForQuestionAnswering.__init__®  sÆ   ø€ Ü‰Ñ˜Ô ØŸ™ˆŒä—l‘l 6×#4Ñ#4°f·n±nÓEˆŒäŸ™ vÓ.ˆØ$)ˆÔ!Ø#(ˆÔ Ü  Ó0ˆŒäŸ™ vÓ.ˆØ$(ˆÔ!Ø$*×$=Ñ$=ˆÔ!Ü  Ó0ˆŒà ×+Ñ+ˆŒÜŸ)™) F§N¡N°F×4EÑ4EÓFˆŒð 	�‰Õr0   c                 ó   — | j                   S rX   rw  rx  s    r/   ry  z-UMT5ForQuestionAnswering.get_input_embeddingsÅ  rz  r0   c                 ó~   — || _         | j                  j                  |«       | j                  j                  |«       y rX   r|  rH  s     r/   rJ  z-UMT5ForQuestionAnswering.set_input_embeddingsÉ  r}  r0   Nr  r¯   r  r  r~  Ústart_positionsÚend_positionsrN  r  rù   rú   rV  rW  r†   c                 ó¾  — |�|n| j                   j                  }|
�|
n| j                   j                  }
|�|�d}
|€ |	€|€t        d«      ‚| j	                  |«      }|
�|
n| j                   j                  }
|�|n| j                   j                  }|€| j                  ||||||¬«      }nI|rGt        |t        «      s7t        |d   t        |«      dkD  r|d   ndt        |«      dkD  r|d   nd¬«      }|d   }| j                  |||	d|||
|||¬	«
      }|d   }| j                  |«      }|j                  dd
¬«      \  }}|j                  d
«      j                  «       }|j                  d
«      j                  «       }d}|�é|�çt        |j                  «       «      dkD  r*|j                  d
«      j                  |j                   «      }t        |j                  «       «      dkD  r*|j                  d
«      j                  |j                   «      }|j                  d«      }|j#                  d|«      }|j#                  d|«      }t%        |¬«      } |||«      } |||«      }||z   dz  }|s||f|dd z   |z   }|�|f|z   S |S t'        ||||j(                  |j*                  |j,                  |j.                  |j0                  |j*                  |j,                  ¬«
      S )a†  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5
            Training](./umt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        NFr°  r�  r   r   r2   r‚  rƒ  r3   r²   r™  )
rœ  Ústart_logitsÚ
end_logitsr®   r„  r…  rU  r†  r­   r‡  )rG   rW  rù   r.  r2  rp  rY   r   rˆ  rr  r  Úsplitr»  rÆ   rˆ   r5   r¢   r÷   r   r   r®   r=   rT  rU  rS  )r+   r  r¯   r  r  r~  rÉ  rÊ  rN  r  rù   rú   rV  rW  rÇ   r=   r‰  rž  r�  rÌ  rÍ  Ú
total_lossÚignored_indexr   Ú
start_lossÚend_lossr¡  s                              r/   r?   z UMT5ForQuestionAnswering.forwardÎ  s0  € ð\ &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆØ!*Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	ØÐ&¨=Ð+DØˆIð
 Ð$Ð)>Ð)FØÐ Ü ðUóð ð
 !%× 1Ñ 1°)Ó <Ðà!*Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ð%<‘kÀ$Ç+Á+×BYÑBYˆð Ð"Ø"Ÿl™lØ#Ø-Ø+Ø"3Ø%9Ø'ð +ó ‰Oñ ¤¨O¼_Ô!MÜ-Ø"1°!Ñ"4Ü47¸Ó4HÈ1Ò4L˜o¨aÒ0ÐRVÜ14°_Ó1EÈÒ1I˜?¨1Ò-ÈtôˆOð (¨Ñ*ˆð Ÿ,™,Ø'Ø1Ø/Ø Ø"/Ø#1ØØ/Ø!5Ø#ð 'ó 
ˆð *¨!Ñ,ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=×"@Ñ"@À×ATÑATÓ"U�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9× <Ñ <¸Z×=NÑ=NÓ O�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJáØ" JÐ/°/À!À"Ð2EÑEÈÑWˆFØ/9Ð/E�Z�M FÑ*ÐQÈ6ÐQä2ØØ%Ø!Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô
ð 	
r0   r¤  )r@   rA   rB   rŒ  r%   ry  rJ  r   r'   r�  rŽ  r�  rZ  rZ   r�  r   r?   rC   rD   s   @r/   r!  r!  §  s…  ø„ ð (7Ø'6ñÐô
ò.ò:ð
 ð .2Ø37Ø59Ø:>Ø=AØ37Ø15Ø26Ø:>Ø!%Ø)-Ø,0Ø#'ñI
à×#Ñ# dÑ*ðI
ð ×)Ñ)¨DÑ0ðI
ð !×+Ñ+¨dÑ2ð	I
ð
 !&× 0Ñ 0°4Ñ 7ðI
ð ˜u U§\¡\Ñ2Ñ3°dÑ:ðI
ð ×)Ñ)¨DÑ0ðI
ð ×'Ñ'¨$Ñ.ðI
ð ×(Ñ(¨4Ñ/ðI
ð  %×0Ñ0°4Ñ7ðI
ð ˜$‘;ðI
ð   $™;ðI
ð # T™kðI
ð ˜D‘[ðI
ð  
ˆu× Ñ Ñ	!Ð$GÑ	Gò!I
ó ôI
r0   r!  )r   r  r!  r«  r'  r  r
  )BrÖ   rn  r•   r'   r   Útorch.nnr   r   r   rM  r   r  Úactivationsr	   Úcache_utilsr
   r   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_utilsr   Úutilsr   r   r   r   r   r   Úconfiguration_umt5r    Ú
get_loggerr@   r}   ÚModuler"   rF   r_   rg   rq   rÙ   rå   rì   r  r
  r<  r  r  r   r«  r'  r!  Ú__all__rO  r0   r/   ú<module>rà     s  ðñ ã Û ã Ý ß AÑ Aå &Ý !ß CÑ CÝ )ß JÝ 9÷÷ ñ õ .÷÷ õ +ð 
ˆ×	Ñ	˜HÓ	%€ô+�B—I‘Iô +ô4˜Ÿ	™	ô ô.˜RŸY™Yô ô<�"—)‘)ô ô$o)�B—I‘Iô o)ôd˜RŸY™Yô ô2˜bŸi™iô ô6@Ð*ô @ôH˜RŸY™Yô ð$ ôk!˜/ó k!ó ðk!ô\e
Ð#ô e
ðP ôf
Ð#ó f
ó ðf
ñR ðôô
H)Ð#6¸ó H)óð
H)ðV ôXÐ*ó Xó ðXñv ðôôS
Ð$7ó S
óðS
ðl ôG
Ð!4ó G
ó ðG
ðT ôp
Ð2ó p
ó ðp
òf�r0   