
    ^jO!                       d Z ddlZddlZddlZddlmZ ddlmZmZmZ ddl	m
Z ddlmZ ddlmZmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZmZmZmZmZmZm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z) ddl*m+Z+  e(jX                  e-      Z. G d dej^                        Z0 G d dej^                        Z1 G d dej^                        Z2 G d dej^                        Z3 G d dej^                        Z4 G d dej^                        Z5 G d dej^                        Z6 G d d e      Z7 G d! d"ej^                        Z8e& G d# d$e"             Z9 G d% d&e9      Z:e& G d' d(e9             Z; e&d)*       G d+ d,e9e             Z<e& G d- d.e9             Z= e&d/*       G d0 d1e9             Z>e& G d2 d3e9             Z?e& G d4 d5e9             Z@g d6ZAy)7zPyTorch UMT5 model.    N)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )initialization)ACT2FN)CacheDynamicCacheEncoderDecoderCache)GenerationMixin)create_bidirectional_maskcreate_causal_mask)GradientCheckpointingLayer)BaseModelOutput)BaseModelOutputWithPastAndCrossAttentionsSeq2SeqLMOutputSeq2SeqModelOutput#Seq2SeqQuestionAnsweringModelOutputSeq2SeqSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)DUMMY_INPUTS
DUMMY_MASKauto_docstringis_torchdynamo_compilingloggingtorch_compilable_check   )
UMT5Configc                   &     e Zd Zd fd	Zd Z xZS )UMT5LayerNormc                     t         |           t        j                  t	        j
                  |            | _        || _        y)ze
        Construct a layernorm module in the UMT5 style. No bias and no subtraction of mean.
        N)super__init__r   	Parametertorchonesweightvariance_epsilon)selfhidden_sizeeps	__class__s      q/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/umt5/modeling_umt5.pyr%   zUMT5LayerNorm.__init__7   s1     	ll5::k#:; #    c                    |j                  t        j                        j                  d      j	                  dd      }|t        j
                  || j                  z         z  }| j                  j                  t        j                  t        j                  fv r%|j                  | j                  j                        }| j                  |z  S )N   T)keepdim)tor'   float32powmeanrsqrtr*   r)   dtypefloat16bfloat16)r+   hidden_statesvariances      r/   forwardzUMT5LayerNorm.forward?   s     !##EMM266q9>>r4>P%Ht?T?T4T(UU ;; ??),,T[[->->?M{{]**r0   )gư>)__name__
__module____qualname__r%   r?   __classcell__r.   s   @r/   r"   r"   6   s    $+r0   r"   c                   *     e Zd Zdef fdZd Z xZS )UMT5DenseActDenseconfigc                 ^   t         |           t        j                  |j                  |j
                  d      | _        t        j                  |j
                  |j                  d      | _        t        j                  |j                        | _
        t        |j                     | _        y NFbias)r$   r%   r   Lineard_modeld_ffwiwoDropoutdropout_ratedropoutr	   dense_act_fnactr+   rG   r.   s     r/   r%   zUMT5DenseActDense.__init__Q   sn    ))FNNFKKeD))FKKeDzz&"5"56&--.r0   c                    | j                  |      }| j                  |      }| j                  |      }t        | j                  j
                  t        j                        r|j                  | j                  j
                  j                  k7  r`| j                  j
                  j                  t        j                  k7  r/|j                  | j                  j
                  j                        }| j	                  |      }|S N)rO   rU   rS   
isinstancerP   r)   r'   Tensorr:   int8r5   r+   r=   s     r/   r?   zUMT5DenseActDense.forwardX   s    ./]3tww~~u||4##tww~~';';;$$

2),,TWW^^-A-ABM.r0   r@   rA   rB   r    r%   r?   rC   rD   s   @r/   rF   rF   P   s    /z /r0   rF   c                   *     e Zd Zdef fdZd Z xZS )UMT5DenseGatedActDenserG   c                    t         |           t        j                  |j                  |j
                  d      | _        t        j                  |j                  |j
                  d      | _        t        j                  |j
                  |j                  d      | _        t        j                  |j                        | _        t        |j                     | _        y rI   )r$   r%   r   rL   rM   rN   wi_0wi_1rP   rQ   rR   rS   r	   rT   rU   rV   s     r/   r%   zUMT5DenseGatedActDense.__init__h   s    IIfnnfkkF	IIfnnfkkF	))FKKeDzz&"5"56&--.r0   c                 ,   | j                  | j                  |            }| j                  |      }||z  }| j                  |      }t	        | j
                  j                  t        j                        r|j                  | j
                  j                  j                  k7  r`| j
                  j                  j                  t        j                  k7  r/|j                  | j
                  j                  j                        }| j                  |      }|S rX   )rU   ra   rb   rS   rY   rP   r)   r'   rZ   r:   r[   r5   )r+   r=   hidden_geluhidden_linears       r/   r?   zUMT5DenseGatedActDense.forwardp   s    hhtyy78		-0#m3]3 tww~~u||4##tww~~';';;$$

2),,TWW^^-A-ABM.r0   r]   rD   s   @r/   r_   r_   g   s    /z /r0   r_   c                   *     e Zd Zdef fdZd Z xZS )UMT5LayerFFrG   c                    t         |           |j                  rt        |      | _        nt        |      | _        t        |j                  |j                        | _	        t        j                  |j                        | _        y )Nr-   )r$   r%   is_gated_actr_   DenseReluDenserF   r"   rM   layer_norm_epsilon
layer_normr   rQ   rR   rS   rV   s     r/   r%   zUMT5LayerFF.__init__   s_    "8"@D"3F";D'F<U<UVzz&"5"56r0   c                 r    | j                  |      }| j                  |      }|| j                  |      z   }|S rX   )rm   rk   rS   )r+   r=   forwarded_statess      r/   r?   zUMT5LayerFF.forward   s=    ??=9../?@%5E(FFr0   r]   rD   s   @r/   rg   rg      s    7z 7r0   rg   c            
            e Zd ZdZddedz  f fdZdej                  dej                  fdZd Z	dd	Z
	 	 	 dd
ej                  dej                  dz  dedz  dej                  dz  fdZ xZS )UMT5Attentionz7
    T5's attention using relative_attention_bias.
    N	layer_idxc                    t         |           |j                  | _        || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _
        |j                  | _        | j                  | j                  z  | _        || _        |9| j                  r-t        j!                  d| j"                  j$                   d       t'        j(                  | j                  | j                  d      | _        t'        j(                  | j                  | j                  d      | _        t'        j(                  | j                  | j                  d      | _        t'        j(                  | j                  | j                  d      | _        | j                  r0t'        j2                  | j                  | j                        | _        y y )NzInstantiating a decoder z without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.FrJ   )r$   r%   
is_decoderhas_relative_attention_biasrelative_attention_num_bucketsrelative_attention_max_distancerM   d_kvkey_value_proj_dim	num_headsn_headsrR   rS   	inner_dimrr   loggerwarning_oncer.   r@   r   rL   qkvo	Embeddingrelative_attention_bias)r+   rG   ru   rr   r.   s       r/   r%   zUMT5Attention.__init__   sd    +++F(.4.S.S+/5/U/U,~~"(++''**(?(??"*4>>+B+B*C D, , 4<<eD4<<eD4<<eD4>>4<<eD+++-<<8[8[]a]i]i+jD( ,r0   
projectionreturnc                     |j                         d d | j                  | j                  fz   }|j                  |      j	                  dddd      }|S )Nr3   r   r2   r   r   )sizer{   ry   viewpermute)r+   r   new_projection_shapenew_projections       r/   _shapezUMT5Attention._shape   sQ    )0"5tG^G^8__#)=>FFq!QPQRr0   c                    d}| j                   }| j                  }| j                  sC|dz  }||dkD  j                  t        j
                        |z  z  }t	        j                  |      }n*t	        j                  |t	        j                  |             }|dz  }||k  }t	        j                  |j                         |z        t        j                  ||z        z  }|||z
  z  }||j                  t        j
                        z   }t	        j                  |t	        j                  ||dz
              }|t	        j                  |||      z  }|S )a  
        Adapted from Mesh Tensorflow:
        https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593

        Translate relative position to a bucket number for relative attention. The relative position is defined as
        memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
        position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
        small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
        positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
        This should allow for more graceful generalization to longer sequences than the model has been trained on

        Args:
            relative_position: an int32 Tensor
            bidirectional: a boolean - whether the attention is bidirectional
            num_buckets: an integer
            max_distance: an integer

        Returns:
            a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
        r   r2   r   )rv   rw   rt   r5   r'   longabsmin
zeros_likelogfloatmath	full_likewhere)	r+   relative_positionrelative_bucketsnum_bucketsmax_distance	max_exactis_small	log_ratiorelative_position_if_larges	            r/   _relative_position_bucketz'UMT5Attention._relative_position_bucket   sA   * 99;;AK!2Q!6 : :5:: F TT %		*; <!&+<e>N>NO`>a!b b  1$	$y0 II/557)CDtxxP\_hPhGii	y!89	%.ejj1I%I"%*YY&8RT_bcTc(d&
" 	EKK2CE_``r0   c                    | | j                   j                  j                  }t        j                  |t        j
                  |      dddf   |z   }t        j                  |t        j
                  |      dddf   }||z
  }| j                  |      }| j                  |      }	|	j                  g d      j                  d      }	|	S )z%Compute binned relative position biasN)r:   device)r2   r   r   r   )	r   r)   r   r'   aranger   r   r   	unsqueeze)
r+   query_length
key_lengthr   past_seen_tokenscontext_positionmemory_positionr   relative_position_bucketvaluess
             r/   compute_biaszUMT5Attention.compute_bias   s    >1188??F <<EJJvVWXZ^W^_brr,,zFSTXZ[T[\+.>>#'#A#ABS#T --.FG	*44Q7r0   r=   encoder_hidden_statespast_key_valuesattention_maskc                 ^   |j                   d d \  }}||j                  | j                        nd}t        |t        j
                        r|j                         n|}|d u}	| j                  |      }
|
j                  |d| j                  | j                        j                  dd      }
d}|Qt        |t              rA|j                  j                  | j                        }|	r|j                  }n|j                   }n|}|	r|n|}|	rK|I|rG|j"                  | j                     j$                  }|j"                  | j                     j&                  }n| j)                  |      }| j+                  |      }|j                  |d| j                  | j                        j                  dd      }|j                  |d| j                  | j                        j                  dd      }|K|j-                  ||| j                        \  }}|	r)t        |t              rd|j                  | j                  <   t	        j.                  |
|j                  dd            }|j                   d   }| j0                  s;t	        j2                  d| j                  ||f|j4                  |j6                  	      }n| j9                  |||j4                  |
      }|||z   }|}||z  }t:        j<                  j?                  |jA                         d      jC                  |      }t:        j<                  jE                  || jD                  | jF                        }t	        j.                  ||      }|j                  dd      jI                         }|j                  ||d      }| jK                  |      }||fS )Nr2   r   r3   r   FTr   )r   r:   )r   r   dim)ptraining)&shapeget_seq_lengthrr   rY   r'   rZ   cloner   r   r{   ry   	transposer   
is_updatedgetcross_attention_cacheself_attention_cachelayerskeysr   r   r   updatematmulru   zerosr   r:   r   r   
functionalsoftmaxr   type_asrS   r   
contiguousr   )r+   r=   r   r   r   kwargs
batch_size
seq_lengthr   is_cross_attentionquery_statesr   curr_past_key_valuescurrent_states
key_statesvalue_statesscoresr   position_biasposition_bias_maskedattn_weightsattn_outputs                         r/   r?   zUMT5Attention.forward   sW    "/!4!4Ra!8
JM\Mh?99$..Ino7ABRTYT`T`7a+113gw 3$>vvm,#((RtG^G^_iijkmno 
&:oGZ+[(3377GJ!'6'L'L$'6'K'K$#2 2D.-/"=*-44T^^DIIJ/66t~~FMML/J66.1L#RtG^G^_iijkmnoJ',,ZT\\4KbKbcmmnoqrsL*+?+F+FzS_aeaoao+p(
L%*_FY*ZAEO..t~~> lJ,@,@A,FG%%b)
//!KKDLL*j9&--W]WcWcM !--Jv}}O_ . M %)N:M,&& }},,V\\^,DLLVT}},,\T\\TXTaTa,bll<>!++Aq1<<>!&&z:rBff[)L((r0   )FN)Nr   NNN)r@   rA   rB   __doc__intr%   r'   rZ   r   r   r   r
   r?   rC   rD   s   @r/   rq   rq      s    kSSWZ k6 %,, - ^
 6:(,.2N)||N)  %||d2N) 	N)
 t+N)r0   rq   c                   8     e Zd Zddedz  f fdZ	 	 ddZ xZS )UMT5LayerSelfAttentionNrr   c                     t         |           t        |d|      | _        t	        |j
                  |j                        | _        t        j                  |j                        | _        y )NTru   rr   ri   )r$   r%   rq   SelfAttentionr"   rM   rl   rm   r   rQ   rR   rS   r+   rG   rr   r.   s      r/   r%   zUMT5LayerSelfAttention.__init__J  sN    *6t_hi'F<U<UVzz&"5"56r0   c                     | j                  |      }| j                  |||      }|| j                  |d         z   }|f|dd  z   }|S )Nr   r   r   r   )rm   r   rS   )r+   r=   r   r   r   normed_hidden_statesattention_outputoutputss           r/   r?   zUMT5LayerSelfAttention.forwardP  sg      $}=-- )+ . 

 &5Ea5H(II "%5ab%99r0   rX   )NNr@   rA   rB   r   r%   r?   rC   rD   s   @r/   r   r   I  s     7#* 7 	r0   r   c                   :     e Zd Zddedz  f fdZ	 	 	 ddZ xZS )UMT5LayerCrossAttentionNrr   c                     t         |           t        |d|      | _        t	        |j
                  |j                        | _        t        j                  |j                        | _        y )NFr   ri   )r$   r%   rq   EncDecAttentionr"   rM   rl   rm   r   rQ   rR   rS   r   s      r/   r%   z UMT5LayerCrossAttention.__init__c  sO    ,VQVbkl'F<U<UVzz&"5"56r0   c                     | j                  |      }| j                  ||||      }|| j                  |d         z   }|f|dd  z   }	|	S )Nr   r   r   r   r   )rm   r   rS   )
r+   r=   r   r   r   r   r   r   layer_outputr   s
             r/   r?   zUMT5LayerCrossAttention.forwardi  si      $}=// "7)+	 0 
 %t||4DQ4G'HH/$4QR$88r0   rX   r   r   rD   s   @r/   r   r   b  s#    7#* 7 #r0   r   c                   @     e Zd Zddedz  f fdZ	 	 	 	 	 	 ddZ xZS )	UMT5BlockNrr   c                 n   t         |           |j                  | _        t        j                         | _        | j
                  j                  t        ||             | j                  r&| j
                  j                  t        ||             | j
                  j                  t        |             y )Nrr   )
r$   r%   rt   r   
ModuleListlayerappendr   r   rg   r   s      r/   r%   zUMT5Block.__init__~  sz     ++]]_


09MN??JJ5f	RS

+f-.r0   c                 ~    | j                   d   |||      \  }}	|j                  t        j                  k(  r}t        j                  |j                        j
                  }
t        j                  t        j                  |      j                         |
dz
  |
      }t        j                  || |      }d }| j                  xr |d u}|r | j                   d   ||||      \  }}|j                  t        j                  k(  r}t        j                  |j                        j
                  }
t        j                  t        j                  |      j                         |
dz
  |
      }t        j                  || |      } | j                   d   |      }|j                  t        j                  k(  r}t        j                  |j                        j
                  }
t        j                  t        j                  |      j                         |
dz
  |
      }t        j                  || |      }|f}|r||	|fz  }|S )Nr   r   i  )r   maxr   r   r3   )r   r:   r'   r;   finfor   r   isinfanyclamprt   )r+   r=   r   r   encoder_attention_maskr   	use_cacheoutput_attentionsr   self_attn_weights	max_dtypeclamp_valuecross_attn_weightsdo_cross_attentionr   s                  r/   r?   zUMT5Block.forward  s    ,94::a=)+,
(( %--/M$7$78<<I++ekk-&@&D&D&F	TXHXZcdK!KKK<[YM "!__R1Fd1R0=

1&;5 /	1-M- ""emm3!KK(;(;<@@	#kk%++m*D*H*H*JIX\L\^gh %M|Q\ ] '

2}5 %--/M$7$78<<I++ekk-&@&D&D&F	TXHXZcdK!KKK<[YM ")+=>>Gr0   rX   )NNNNFFr   rD   s   @r/   r   r   }  s,    /#* / "#5r0   r   c                   `     e Zd ZdZdef fdZdej                  dej                  fdZ xZ	S )UMT5ClassificationHeadz-Head for sentence-level classification tasks.rG   c                 (   t         |           t        j                  |j                  |j                        | _        t        j                  |j                        | _        t        j                  |j                  |j                        | _
        y )N)r   )r$   r%   r   rL   rM   denserQ   classifier_dropoutrS   
num_labelsout_projrV   s     r/   r%   zUMT5ClassificationHead.__init__  sZ    YYv~~v~~>
zzF$=$=>		&..&2C2CDr0   r=   r   c                     | j                  |      }| j                  |      }t        j                  |      }| j                  |      }| j	                  |      }|S rX   )rS   r  r'   tanhr  r\   s     r/   r?   zUMT5ClassificationHead.forward  sN    ]3

=1

=1]3m4r0   )
r@   rA   rB   r   r    r%   r'   rZ   r?   rC   rD   s   @r/   r  r    s/    7Ez EU\\ ell r0   r  c                        e Zd ZU eed<   dZdZdZdgZdgZ	e
d        Z ej                          fd       Zd Z xZS )	UMT5PreTrainedModelrG   transformerTr   rP   c                 v    t        j                  t              }t        j                  t              }|||d}|S )N)decoder_input_ids	input_idsdecoder_attention_mask)r'   tensorr   r   )r+   r  
input_maskdummy_inputss       r/   r  z UMT5PreTrainedModel.dummy_inputs  s6    LL.	\\*-
!*"&0

 r0   c                    t         |   |       | j                  j                  }t	        |t
              r$t        j                  |j                  |dz         y	t	        |t        t        t        t        f      rt        j                  |j                  j                  d|dz         t        |d      rE| j                  j                   s/t        j                  |j"                  j                  d|dz         t        |d      rpt        j                  |j$                  j                  d|| j                  j&                  dz  z         t        j(                  |j$                  j*                         y	y	t	        |t,              rft        |d      rYt        j                  |j.                  j                  d|dz         t        j(                  |j.                  j*                         y	y	t	        |t0              r9t        j                  |j2                  j                  d|| j                  j&                  dz  z         t        |j2                  d      r?|j2                  j*                  )t        j(                  |j2                  j*                         t        j                  |j4                  j                  d|| j                  j&                  dz  z         t        |j4                  d      rA|j4                  j*                  *t        j(                  |j4                  j*                         y	y	y	t	        |t6              r9t        j                  |j8                  j                  d|| j                  j&                  dz  z         t        |j8                  d      r?|j8                  j*                  )t        j(                  |j8                  j*                         t        j                  |j:                  j                  d|| j                  j<                  dz  z         t        |j:                  d      rA|j:                  j*                  *t        j(                  |j:                  j*                         y	y	y	t	        |t>              rt        j                  |j@                  j                  d|| j                  j&                  dz  z         t        |j@                  d      r?|j@                  j*                  )t        j(                  |j@                  j*                         t        j                  |jB                  j                  d|| j                  j&                  dz  z         t        |jB                  d      r?|jB                  j*                  )t        j(                  |jB                  j*                         t        j                  |j:                  j                  d|| j                  j<                  dz  z         t        |j:                  d      rA|j:                  j*                  *t        j(                  |j:                  j*                         y	y	y	t	        |tD              rP| j                  j&                  }| j                  jF                  }| j                  jH                  }t        j                  |jJ                  j                  d|||z  dz  z         t        j                  |jL                  j                  d||dz  z         t        j                  |jN                  j                  d||dz  z         t        j                  |jP                  j                  d|||z  dz  z         |jR                  r3t        j                  |jT                  j                  d||dz  z         y	y	y	)
zInitialize the weights      ?g        )r8   stdlm_head
qa_outputs      
classifierrK   N)+r$   _init_weightsrG   initializer_factorrY   r"   init	constant_r)   	UMT5ModelUMT5ForConditionalGenerationUMT5EncoderModelUMT5ForQuestionAnsweringnormal_sharedhasattrtie_word_embeddingsr  r  rM   zeros_rK   UMT5ForTokenClassificationr  r  r  r  rF   rO   rP   rN   r_   ra   rb   rq   rx   rz   r   r   r   r   ru   r   )r+   modulefactorrM   ry   r{   r.   s         r/   r  z!UMT5PreTrainedModel._init_weights  sL    	f%//fm,NN6==&3,7, (	
 LL--CVc\Jvy)$++2Q2QV^^22&3,Ov|,V..55CVPTP[P[PcPchlOlEmnF--223 -  :;v|,V..55CVc\RF--223 -  67LL,,3Ft{{GZGZ_cFc<dev||V,1B1B1NFLL--.LL//cv$++J]J]bfIf?ghv/FOO4H4H4TFOO001 5U/ 12 LL))DKKDWDW\`C`9abvyy&)fiinn.HFIINN+LL))DKKDTDTY]C]9^_vyy&)fiinn.HFIINN+ /I) 67LL++#6dkkFYFY^bEb;cdv{{F+0@0@0LFKK,,-LL++#6dkkFYFY^bEb;cdv{{F+0@0@0LFKK,,-LL))DKKDTDTY]C]9^_vyy&)fiinn.HFIINN+ /I). kk))G!%!1!1kk++GLLs7M_C_dhBh8ijLLs'4-8PQLLs'4-8PQLLs7M_C_dhBh8ij11V;;BBRX]dim\mRno 2 /r0   c                 8   | j                   j                  }| j                   j                  }|t        d      |j	                  |j
                        }|dd df   j                         |ddd f<   ||d<   |t        d      |j                  |dk(  |       |S )Nzself.model.config.decoder_start_token_id has to be defined. In UMT5 it is usually set to the pad_token_id. See UMT5 docs for more information..r3   r   ).r   z1self.model.config.pad_token_id has to be defined.)rG   decoder_start_token_idpad_token_id
ValueError	new_zerosr   r   masked_fill_)r+   r  r,  r-  shifted_input_idss        r/   _shift_rightz UMT5PreTrainedModel._shift_right,  s    !%!C!C{{//!)6 
 &//	@%.sCRCx%8%>%>%@#qr'"$:&!PQQ&&'8D'@,O  r0   )r@   rA   rB   r    __annotations__base_model_prefixsupports_gradient_checkpointing_can_compile_fullgraph_no_split_modules_keep_in_fp32_modulespropertyr  r'   no_gradr  r2  rC   rD   s   @r/   r
  r
    sc    %&*#!$!F  U]]_Ap ApF!r0   r
  c                   @     e Zd Z fdZd Z	 	 	 	 	 	 	 	 	 	 ddZ xZS )	UMT5Stackc           	         t         |   |       t        j                  |j                  |j
                        | _        |j                  | _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        t        |j
                  |j                        | _        t        j                   |j"                        | _        d| _        | j)                          y c c}w )Nr   ri   F)r$   r%   r   r   
vocab_sizerM   embed_tokensrt   r   range
num_layersr   blockr"   rl   final_layer_normrQ   rR   rS   gradient_checkpointing	post_init)r+   rG   ir.   s      r/   r%   zUMT5Stack.__init__C  s     LL):):FNNK ++]]ERXRcRcLd#eqIf$B#ef
 -fnn&B[B[ \zz&"5"56 ',# $fs   7C9c                     || _         y rX   )r?  r+   new_embeddingss     r/   set_input_embeddingszUMT5Stack.set_input_embeddingsO  s
    *r0   c           
      ~   ||n| j                   j                  }||n| j                   j                  }|	|	n| j                   j                  }	|
|
n| j                   j                  }
|$|"| j
                  rdnd}t        d| d| d      |&|j                         }|j                  d|d         }n8||j                         d d }n"| j
                  rdnd}t        d| d| d	      | j                  r%| j                  r|rt        j                  d
       d}|(| j                  t        d      | j                  |      }|\  }}|du r| j
                  st        d|  d      | j
                  rf|rr|p| j                   j                  r5t        t!        | j                         t!        | j                               }n%t!        | j                         }n| j
                  sd }||j#                         nd}|1t%               s'||z   }t'        j(                  |||j*                        }| j
                  rt-        | j                   |||      }n]|Y|d d d d d d f   }|j/                  |j0                        }d|z
  t'        j2                  |j0                        j4                  z  }nd }| j
                  r|t7        | j                   |||      }nd }|	rdnd }|rdnd }|r| j
                  rdnd }| j9                  |      }t;        | j<                        D ]C  \  }}|	r||fz   } ||||||||      }|d   }|s%||d   fz  }| j
                  s;||d   fz  }E | j?                  |      }| j9                  |      }|	r||fz   }|
stA        d |||||fD              S tC        |||||      S )Ndecoder_ zYou cannot specify both zinput_ids and zinputs_embeds at the same timer3   zYou have to specify either zinput_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fz<You have to initialize the model with valid token embeddingsTz)`use_cache` can only be set to `True` if z is used as a decoder)rG   r   )r   )rG   rN  r   r   )r:   r  )rG   rN  r   r    )r   r   r   r   r   r2   c              3   $   K   | ]  }|| 
 y wrX   rO  ).0r   s     r/   	<genexpr>z$UMT5Stack.forward.<locals>.<genexpr>  s      
 = 
s   )last_hidden_stater   r=   
attentionscross_attentions)"rG   r   r   output_hidden_statesreturn_dictrt   r.  r   r   rD  r   r}   r~   r?  is_encoder_decoderr   r   r   r   r'   r(   r   r   r5   r:   r   r   r   rS   	enumeraterB  rC  tupler   )r+   r  r   r   r   rN  r   r   r   rV  rW  r   err_msg_prefixinput_shaper   r   past_key_values_lengthmask_seq_lengthcausal_maskencoder_extended_attention_maskall_hidden_statesall_attentionsall_cross_attentionsr=   rF  layer_modulelayer_outputss                              r/   r?   zUMT5Stack.forwardR  s     "+!6IDKK<Q<Q	1B1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++BYBY ]%>+/??ZN*>*:.HXXvw  "#..*K!r;r?;I&',,.s3K+/??ZN:>:J-XfWggtuvv&&4==##p "	   ( !_`` --i8M!,
J?? #LTFRg!hii ??_4;;11&9$DKK8,dkk:Z'O '3$++&FO #OETE`!?!?!Afg!*B*D4zAO"ZZ
OML`L`aN??,{{+- /	K '(D$)9:K%..}/B/B.CK,M<O<O0P0T0TTKK??5A.G{{+5&;	/+ /3+"6BD0d%64??rPT]3(4 	@OA|#$58H$H!(%'F /#"3M *!,M =#3"55??(]1-=,??('	@* --m<]3   1]4D D 
 "#%"(
 
 
 9+++%1
 	
r0   )
NNNNNNNNNN)r@   rA   rB   r%   rJ  r?   rC   rD   s   @r/   r<  r<  B  s3    
+
 "#!U
r0   r<  c                       e Zd ZU dZdZeed<   dddZ fdZd Z	d Z
e	 	 	 	 	 	 	 	 	 	 	 	 dd
ej                  d	z  dej                  d	z  dej                  d	z  dej                  d	z  deeej                        d	z  ded	z  dej$                  d	z  dej$                  d	z  ded	z  ded	z  ded	z  ded	z  deej                     ez  fd       Z xZS )r  ao  
    Examples:

    ```python
    >>> from transformers import UMT5Model, AutoTokenizer

    >>> model = UMT5Model.from_pretrained("google/umt5-small")
    >>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
    >>> noisy_text = "UN Offizier sagt, dass weiter <extra_id_0> werden muss in Syrien."
    >>> label = "<extra_id_0> verhandelt"
    >>> inputs = tokenizer(inputs, return_tensors="pt")
    >>> labels = tokenizer(label=label, return_tensors="pt")

    >>> outputs = model(input_ids=inputs["input_ids"], decoder_input_ids=labels["input_ids"])
    >>> hidden_states = outputs.last_hidden_state
    ```umt5rG   shared.weightencoder.embed_tokens.weightdecoder.embed_tokens.weightc                    t         |   |       t        j                  |j                  |j
                        | _        t        j                  |      }d|_	        d|_
        t        |      | _        t        j                  |      }d|_	        |j                  |_        t        |      | _        | j!                          y NFT)r$   r%   r   r   r>  rM   r#  copydeepcopyrt   r   r<  encodernum_decoder_layersrA  decoderrE  r+   rG   encoder_configdecoder_configr.   s       r/   r%   zUMT5Model.__init__  s     ll6#4#4fnnEv.$)!#(  0v.$(!$*$=$=! 0 	r0   c                     | j                   S rX   r#  r+   s    r/   get_input_embeddingszUMT5Model.get_input_embeddings      {{r0   c                 ~    || _         | j                  j                  |       | j                  j                  |       y rX   r#  rp  rJ  rr  rH  s     r/   rJ  zUMT5Model.set_input_embeddings  -    $)).9)).9r0   Nr  r   r  r  encoder_outputsr   rN  decoder_inputs_embedsr   r   rV  rW  r   c                 D   |	|	n| j                   j                  }	||n| j                   j                  }|| j                  ||||
||      }nI|rGt	        |t
              s7t        |d   t        |      dkD  r|d   ndt        |      dkD  r|d   nd      }|d   }| j                  |||||||	|
||
      }|s||z   S t        |j                  |j                  |j                  |j                  |j                  |j                  |j                  |j                        S )	ah
  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5
            Training](./umt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, UMT5Model

        >>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
        >>> model = UMT5Model.from_pretrained("google/umt5-small")

        >>> input_ids = tokenizer(
        ...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids  # Batch size 1

        >>> # preprocess: Prepend decoder_input_ids with start token which is pad token for UMT5Model.
        >>> # This is not needed for torch's UMT5ForConditionalGeneration as it does this internally using labels arg.
        >>> decoder_input_ids = model._shift_right(decoder_input_ids)

        >>> # forward pass
        >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        ```Nr  r   rN  r   rV  rW  r   r   r2   rS  r=   rT  
r  r   rN  r   r   r   r   r   rV  rW  )rS  r   decoder_hidden_statesdecoder_attentionsrU  encoder_last_hidden_stater   encoder_attentions)rG   r   rW  rp  rY   r   lenrr  r   rS  r   r=   rT  rU  )r+   r  r   r  r  r~  r   rN  r  r   r   rV  rW  r   r=   decoder_outputss                   r/   r?   zUMT5Model.forward  sO   D "+!6IDKK<Q<Q	%0%<k$++BYBY ""ll#-+"3%9' + O O_!M-"1!"4474H14Loa0RV14_1E1I?1-tO (* ,,'1/+"/#1/!5# ' 
 "_44!-??+;;"1"?"?.99,==&5&G&G"1"?"?.99	
 		
r0   NNNNNNNNNNNN)r@   rA   rB   r   
model_typer    r3  _tied_weights_keysr%   ry  rJ  r   r'   
LongTensorFloatTensor
BoolTensorrZ  r
   rZ   boolr   r?   rC   rD   s   @r/   r  r    ss   " J'6'6
$:
  .23759:>BF(,-159!%)-,0#'q
##d*q
 ))D0q
 !++d2	q

 !& 0 04 7q
 uU%6%6784?q
 q
 ||d*q
  %||d2q
 $;q
  $;q
 #Tkq
 D[q
 
u  	!$6	6q
 q
r0   r  z<
    UMT5 Model with a `language modeling` head on top.
    )custom_introc                       e Zd ZdZdZddddZ fdZd Zd Ze		 	 	 	 	 	 	 	 	 	 	 	 	 dd	e
j                  dz  d
e
j                  dz  de
j                  dz  de
j                  dz  deee
j                        dz  dedz  de
j                  dz  de
j                  dz  de
j                  dz  dedz  dedz  dedz  dedz  dee
j                     ez  fd       Zde
j                  fdZ xZS )r  a  
    Examples:

    ```python
    >>> from transformers import UMT5ForConditionalGeneration, AutoTokenizer

    >>> model = UMT5ForConditionalGeneration.from_pretrained("google/umt5-small")
    >>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
    >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien."
    >>> summary = "Weiter Verhandlung in Syrien."
    >>> inputs = tokenizer(article, text_target=summary, return_tensors="pt")

    >>> outputs = model(**inputs)
    >>> loss = outputs.loss
    ```rg  rh  )rj  rk  zlm_head.weightc                    t         |   |       |j                  | _        t	        j
                  |j                  |j                        | _        t        j                  |      }d|_
        d|_        t        |      | _        t        j                  |      }d|_
        |j                  |_        t        |      | _        t	        j"                  |j                  |j                  d      | _        | j'                          y )NFTrJ   )r$   r%   rM   	model_dimr   r   r>  r#  rn  ro  rt   r   r<  rp  rq  rA  rr  rL   r  rE  rs  s       r/   r%   z%UMT5ForConditionalGeneration.__init__  s     ll6#4#4fnnEv.$)!#(  0v.$(!$*$=$=! 0yy1B1BO 	r0   c                     | j                   S rX   rw  rx  s    r/   ry  z1UMT5ForConditionalGeneration.get_input_embeddings  rz  r0   c                 ~    || _         | j                  j                  |       | j                  j                  |       y rX   r|  rH  s     r/   rJ  z1UMT5ForConditionalGeneration.set_input_embeddings  r}  r0   Nr  r   r  r  r~  r   rN  r  labelsr   r   rV  rW  r   c                    |
|
n| j                   j                  }
||n| j                   j                  }|| j                  ||||||      }nI|rGt	        |t
              s7t        |d   t        |      dkD  r|d   ndt        |      dkD  r|d   nd      }|d   }|	||| j                  |	      }| j                  |||||||
|||
      }|d   }| j                   j                  r|| j                  dz  z  }| j                  |      }d}|	^t        d	
      }|	j                  |j                        }	 ||j                  d|j!                  d            |	j                  d            }|s|f|dd z   |z   }||f|z   S |S t#        |||j$                  |j&                  |j(                  |j*                  |j,                  |j&                  |j(                  	      S )a  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5
            Training](./umt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
            config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
            labels in `[0, ..., config.vocab_size]`

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, UMT5ForConditionalGeneration

        >>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
        >>> model = UMT5ForConditionalGeneration.from_pretrained("google/umt5-small")

        >>> # training
        >>> input_ids = tokenizer("The <extra_id_0> walks in <extra_id_1> park", return_tensors="pt").input_ids
        >>> labels = tokenizer("<extra_id_0> cute dog <extra_id_1> the <extra_id_2>", return_tensors="pt").input_ids
        >>> outputs = model(input_ids=input_ids, labels=labels)
        >>> loss = outputs.loss
        >>> logits = outputs.logits

        >>> # inference
        >>> input_ids = tokenizer("Studies have shown that <extra_id_0> good for you", return_tensors="pt").input_ids
        >>> outputs = model.generate(input_ids)
        >>> tokenizer.decode(outputs[0], skip_special_tokens=True)
        ```Nr  r   r   r2   r  r  r  r+  ignore_indexr3   	losslogitsr   r  r  rU  r  r   r  )rG   r   rW  rp  rY   r   r  r2  rr  r%  r  r  r   r5   r   r   r   r   r   r=   rT  rU  rS  )r+   r  r   r  r  r~  r   rN  r  r  r   r   rV  rW  r   r=   r  sequence_output	lm_logitsr  loss_fctoutputs                         r/   r?   z$UMT5ForConditionalGeneration.forward  s   L "+!6IDKK<Q<Q	%0%<k$++BYBY ""ll#-+"3%9' + O O_!M-"1!"4474H14Loa0RV14_1E1I?1-tO (*"3";@U@] $ 1 1& 9 ,,'1/+"/#1/!5# ' 
 *!,;;** .1EFOLL1	'T:HYYy//0FINN2y~~b/ABFKKPROTD\OAB$77/IF)-)9TGf$EvE+;;"1"?"?.99,==&5&G&G"1"?"?.99

 
	
r0   c                 $    | j                  |      S rX   )r2  )r+   r  s     r/   %prepare_decoder_input_ids_from_labelszBUMT5ForConditionalGeneration.prepare_decoder_input_ids_from_labels`  s      ((r0   NNNNNNNNNNNNN)r@   rA   rB   r   r  r  r%   ry  rJ  r   r'   r  r  r  rZ  rZ   r
   r  r   r?   r  rC   rD   s   @r/   r  r    s     J'6'6),:
  .23759:>=A(,26:>*.!%)-,0#'L
##d*L
 ))D0L
 !++d2	L

 !& 0 04 7L
 uU\\23d:L
 L
 ((4/L
  %0047L
   4'L
 $;L
  $;L
 #TkL
 D[L
  
u  	!O	3!L
 L
^)ELL )r0   r  c                        e Zd ZdZdZddiZ fdZd Zd Ze		 	 	 	 	 	 dd	e
j                  dz  d
e
j                  dz  de
j                  dz  dedz  dedz  dedz  dee
j                     ez  fd       Z xZS )r   a  
    Examples:

    ```python
    >>> from transformers import UMT5EncoderModel, AutoTokenizer

    >>> model = UMT5EncoderModel.from_pretrained("google/umt5-small")
    >>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
    >>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien."
    >>> input_ids = tokenizer(article, return_tensors="pt").input_ids
    >>> outputs = model(input_ids)
    >>> hidden_state = outputs.last_hidden_state
    ```rg  rj  rh  c                    t         |   |       t        j                  |j                  |j
                        | _        t        j                  |      }d|_	        d|_
        t        |      | _        | j                          y )NF)r$   r%   r   r   r>  rM   r#  rn  ro  r   rX  r<  rp  rE  )r+   rG   rt  r.   s      r/   r%   zUMT5EncoderModel.__init__z  sb     ll6#4#4fnnEv.#( ,1) 0 	r0   c                     | j                   S rX   rw  rx  s    r/   ry  z%UMT5EncoderModel.get_input_embeddings  rz  r0   c                 H    || _         | j                  j                  |       y rX   )r#  rp  rJ  rH  s     r/   rJ  z%UMT5EncoderModel.set_input_embeddings  s    $)).9r0   Nr  r   rN  r   rV  rW  r   c                 h    ||n| j                   j                  }| j                  ||||||      }|S )aQ  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).

        Example:

        ```python
        >>> from transformers import AutoTokenizer, UMT5EncoderModel

        >>> tokenizer = AutoTokenizer.from_pretrained("google/umt5-small")
        >>> model = UMT5EncoderModel.from_pretrained("google/umt5-small")
        >>> input_ids = tokenizer(
        ...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> outputs = model(input_ids=input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        ```r  )rG   rW  rp  )	r+   r  r   rN  r   rV  rW  r   r~  s	            r/   r?   zUMT5EncoderModel.forward  sH    F &1%<k$++BYBY,,)'/!5# ' 
 r0   )NNNNNN)r@   rA   rB   r   r  r  r%   ry  rJ  r   r'   r  r  r  rZ  r   r?   rC   rD   s   @r/   r   r   d  s     J 	&
:  .23726)-,0#',##d*, ))D0, ((4/	,
  $;, #Tk, D[, 
u  	!O	3, ,r0   r   z
    UMT5 model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE
    tasks.
    c                   ~    e Zd ZdgZdef fdZe	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e
ej                     dz  d
ej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  dedz  deez  fd       Z xZS )UMT5ForSequenceClassificationFdecoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weightrG   c                     t         |   |       t        |      | _        t	        |      | _        | j                          y rX   )r$   r%   r  r  r  classification_headrE  rV   s     r/   r%   z&UMT5ForSequenceClassification.__init__  s6     $V,#9&#A  	r0   Nr  r   r  r  r~  rN  r  r  r   r   rV  rW  r   c                 R   ||n| j                   j                  }|d}	|$|"t        d| j                  j                         | ||t        d      | j                  |      }| j                  ||||||||	|
||      }|d   }|j                  | j                   j                        j                  |j                        }t        t        j                  |j                  d            j!                         dk(  d       |j"                  \  }}}||ddf   }t        |j"                  d   |z  dk\  d	       |j%                  |d
|      ddd
ddf   }| j'                  |      }d}||j                  |j                        }| j                   j(                  | j                   j*                  dk(  rd| j                   _        nv| j                   j*                  dkD  rL|j,                  t        j.                  k(  s|j,                  t        j0                  k(  rd| j                   _        nd| j                   _        | j                   j(                  dk(  rSt3               }| j                   j*                  dk(  r& ||j5                         |j5                               }n |||      }n| j                   j(                  dk(  rGt7               } ||j%                  d
| j                   j*                        |j%                  d
            }n,| j                   j(                  dk(  rt9               } |||      }|s|f|dd z   }||f|z   S |S t;        |||j<                  |j>                  |j@                  |jB                  |jD                  |jF                  |jH                  	      S )a  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5
            Training](./umt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NFz8Passing input embeddings is currently not supported for If no `decoder_input_ids` or `decoder_inputs_embeds` are passed, `input_ids` cannot be `None`. Please pass either `input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`.)
r   r  r  r~  rN  r  r   r   rV  rW  r   r   z7All examples must have the same number of <eos> tokens.z3Each example must contain at least one <eos> token.r3   
regressionsingle_label_classificationmulti_label_classificationr  )%rG   rW  NotImplementedErrorr.   r@   r.  r2  r  eqeos_token_idr5   r   r   r'   unique_consecutivesumnumelr   r   r  problem_typer  r:   r   r   r   squeezer   r   r   r   r  r  rU  r  r   r  )r+   r  r   r  r  r~  rN  r  r  r   r   rV  rW  r   r   r  eos_maskr   _r,   selectedsentence_representationr  r  r  r  s                             r/   r?   z%UMT5ForSequenceClassification.forward  sp   ` &1%<k$++BYBYI!:%J4>>KbKbJcd  $)>)F  U 
 !% 1 1) <"")/#9+'"7/!5# # 
 "!*<< 8 89<<_=S=ST$$X\\!_5;;=BE	
 &5%:%:"
A{"8Q;/NN1+q0A	
 #+--
B"LQPRTUX"V))*ABYYv}}-F{{''/;;))Q./;DKK,[[++a/V\\UZZ5OSYS_S_chclclSl/LDKK,/KDKK,{{''<7"9;;))Q.#FNN$4fnn6FGD#FF3D))-JJ+-B0F0F GUWY))-II,./Y,F)-)9TGf$EvE.#33")"?"?&99$55&-&G&G")"?"?&99

 
	
r0   r  )r@   rA   rB   "_keys_to_ignore_on_load_unexpectedr    r%   r   r'   r  rZ   listr  r  rZ  r   r?   rC   rD   s   @r/   r  r    sU    +s)s&z   .2.259:>:>26:>*.!%)-,0#'F
##d*F
 t+F
 !++d2	F

 !& 0 04 7F
 e//047F
 ((4/F
  %0047F
   4'F
 $;F
  $;F
 #TkF
 D[F
 
0	0F
 F
r0   r  c                       e Zd ZdgZdef fdZe	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	dz  d
e	dz  de	dz  de
ej                     ez  fd       Z xZS )r'  r  rG   c                 ,   t         |   |       |j                  | _        t        |      | _        t        j                  |j                        | _        t        j                  |j                  |j                        | _        | j                          y rX   )r$   r%   r  r   r  r   rQ   r  rS   rL   r,   r  rE  rV   s     r/   r%   z#UMT5ForTokenClassification.__init__a  sj      +++F3zz&";";<))F$6$68I8IJ 	r0   Nr  r   rN  r  r   rV  rW  r   c                    ||n| j                   j                  }| j                  ||||||      }	|	d   }
| j                  |
      }
| j	                  |
      }d}|<t               } ||j                  d| j                        |j                  d            }|s||	dd f}||f|z   S |S t        |||	j                  |	j                        S )aB  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so you
            should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        N)r   rN  r   rV  rW  r   r3   r2   )r  r  r=   rT  )rG   rW  r  rS   r  r   r   r  r   r=   rT  )r+   r  r   rN  r  r   rV  rW  r   r   r=   r  r  r  r  s                  r/   r?   z"UMT5ForTokenClassification.forwardl  s    6 &1%<k$++BYBY"")'/!5# # 
  
]3/')HFKKDOO<fkk"oNDgam,F)-)9TGf$EvE$!//))	
 	
r0   )NNNNNNN)r@   rA   rB   r  r    r%   r   r'   rZ   r  rZ  r   r?   rC   rD   s   @r/   r'  r'  \  s    *r)s&	z 	  *..2-1&*)-,0#'6
<<$&6
 t+6
 ||d*	6

 t#6
  $;6
 #Tk6
 D[6
 
u||	4	46
 6
r0   r'  c                       e Zd ZdddZ fdZd Zd Ze	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  deeej                        dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  dedz  deej                     ez  fd       Z xZS )r!  rh  ri  c                 $   t         |   |       |j                  | _        t	        j
                  |j                  |j                        | _        t        j                  |      }d|_
        d|_        t        |      | _        t        j                  |      }d|_
        |j                  |_        t        |      | _        |j"                  | _        t	        j$                  |j                  |j"                        | _        | j)                          y rm  )r$   r%   rM   r  r   r   r>  r#  rn  ro  rt   r   r<  rp  rq  rA  rr  r  rL   r  rE  rs  s       r/   r%   z!UMT5ForQuestionAnswering.__init__  s     ll6#4#4fnnEv.$)!#(  0v.$(!$*$=$=! 0 ++))FNNF4E4EF 	r0   c                     | j                   S rX   rw  rx  s    r/   ry  z-UMT5ForQuestionAnswering.get_input_embeddings  rz  r0   c                 ~    || _         | j                  j                  |       | j                  j                  |       y rX   r|  rH  s     r/   rJ  z-UMT5ForQuestionAnswering.set_input_embeddings  r}  r0   Nr  r   r  r  r~  start_positionsend_positionsrN  r  r   r   rV  rW  r   c                    ||n| j                   j                  }|
|
n| j                   j                  }
||d}
| |	|t        d      | j	                  |      }|
|
n| j                   j                  }
||n| j                   j                  }|| j                  ||||||      }nI|rGt        |t              s7t        |d   t        |      dkD  r|d   ndt        |      dkD  r|d   nd      }|d   }| j                  |||	d|||
|||	
      }|d   }| j                  |      }|j                  dd
      \  }}|j                  d
      j                         }|j                  d
      j                         }d}||t        |j                               dkD  r*|j                  d
      j                  |j                         }t        |j                               dkD  r*|j                  d
      j                  |j                         }|j                  d      }|j#                  d|      }|j#                  d|      }t%        |      } |||      } |||      }||z   dz  }|s||f|dd z   |z   }||f|z   S |S t'        ||||j(                  |j*                  |j,                  |j.                  |j0                  |j*                  |j,                  
      S )a  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. UMT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [UMT5 Training](./umt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            UMT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
            is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [UMT5
            Training](./umt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        NFr  r  r   r   r2   r  r  r3   r   r  )
r  start_logits
end_logitsr   r  r  rU  r  r   r  )rG   rW  r   r.  r2  rp  rY   r   r  rr  r  splitr  r   r   r5   r   r   r   r   r   r=   rT  rU  rS  )r+   r  r   r  r  r~  r  r  rN  r  r   r   rV  rW  r   r=   r  r  r  r  r  
total_lossignored_indexr  
start_lossend_lossr  s                              r/   r?   z UMT5ForQuestionAnswering.forward  s0   \ &1%<k$++BYBY!*!6IDKK<Q<Q	&=+DI
 $)>)F  U 
 !% 1 1) <!*!6IDKK<Q<Q	%0%<k$++BYBY ""ll#-+"3%9' + O O_!M-"1!"4474H14Loa0RV14_1E1I?1-tO (* ,,'1/ "/#1/!5# ' 
 *!,1#)<<r<#: j#++B/::<''+668

&=+D?'')*Q."1"9"9""="@"@ATAT"U=%%'(1, - 5 5b 9 < <Z=N=N O(--a0M-33A}EO)//=AM']CH!,@J
M:H$x/14J"J//!"2EEWF/9/EZMF*Q6Q2%!+;;"1"?"?.99,==&5&G&G"1"?"?.99
 	
r0   r  )r@   rA   rB   r  r%   ry  rJ  r   r'   r  r  r  rZ  rZ   r  r   r?   rC   rD   s   @r/   r!  r!    s    (7'6
.:
  .23759:>=A371526:>!%)-,0#'I
##d*I
 ))D0I
 !++d2	I

 !& 0 04 7I
 uU\\23d:I
 ))D0I
 ''$.I
 ((4/I
  %0047I
 $;I
  $;I
 #TkI
 D[I
  
u  	!$G	G!I
 I
r0   r!  )r   r  r!  r  r'  r  r
  )Br   rn  r   r'   r   torch.nnr   r   r   rM  r   r  activationsr	   cache_utilsr
   r   r   
generationr   masking_utilsr   r   modeling_layersr   modeling_outputsr   r   r   r   r   r   r   modeling_utilsr   utilsr   r   r   r   r   r   configuration_umt5r    
get_loggerr@   r}   Moduler"   rF   r_   rg   rq   r   r   r   r  r
  r<  r  r  r   r  r'  r!  __all__rO  r0   r/   <module>r     s        A A & ! C C ) J 9   .  + 
		H	%+BII +4		 .RYY <")) $o)BII o)dRYY 2bii 6@* @HRYY $ k!/ k! k!\e
# e
P f
# f
 f
R 
H)#6 H)
H)V X* X Xv S
$7 S
S
l G
!4 G
 G
T p
2 p
 p
fr0   