Ë
    ÜÓ^jéG ã                   ó  — d Z ddlZddlZddlmZ ddlZddlmZ ddlmZ ddl	m
Z ddlmZ dd	lmZmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZmZmZ ddlmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z'  e%jP                  e)«      Z*dNdejV                  de,de,de,dejV                  f
d„Z-dejV                  de,de,dejV                  fd„Z.dNdejV                  de,de,de,dejV                  f
d„Z/de,dejV                  fd„Z0dejV                  de,dejV                  fd„Z1dejV                  de,d ejd                  dejV                  fd!„Z3dejV                  d"e,de4ejV                  ejV                  f   fd#„Z5dejV                  d"e,dejV                  fd$„Z6d%ejV                  d&ejV                  d'e,dejV                  fd(„Z7 G d)„ d*ejp                  «      Z9 G d+„ d,ejp                  «      Z: G d-„ d.ejp                  «      Z; G d/„ d0ejp                  «      Z< G d1„ d2ejp                  «      Z= G d3„ d4ejp                  «      Z> G d5„ d6ejp                  «      Z? G d7„ d8ejp                  «      Z@ G d9„ d:ejp                  «      ZA G d;„ d<ejp                  «      ZB G d=„ d>ejp                  «      ZC G d?„ d@e«      ZDe# G dA„ dBe«      «       ZE G dC„ dDeE«      ZFe# G dE„ dFeE«      «       ZG e#dG¬H«       G dI„ dJeEe«      «       ZHe# G dK„ dLeE«      «       ZIg dM¢ZJy)OzPyTorch LongT5 model.é    N)ÚAny)Únn)ÚCrossEntropyLossé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutput)ÚPreTrainedModel)ÚDUMMY_INPUTSÚ
DUMMY_MASKÚauto_docstringÚis_torchdynamo_compilingÚloggingé   )ÚLongT5ConfigÚxÚ	block_lenÚdimÚ	pad_valueÚreturnc                 ót  — | j                   |    |z  }t        | j                   «      sCt        | j                   «      }||xx   |z  cc<   t        j                  || j
                  ¬«      S dg| j                  z  }d|f||<   t        |ddd…   d«      }t        j                  j                  | |d|¬«      } | S )	zHPad a tensor so that a sequence length will be a multiple of `block_len`©Údtype©r   r   r   Néÿÿÿÿ© Úconstant©ÚpadÚmodeÚvalue)ÚshapeÚallÚlistÚtorchÚzerosr#   ÚndimÚsumr   Ú
functionalr)   )r   r   r   r   Úpad_lenÚ	new_shaper)   s          úu/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/longt5/modeling_longt5.pyÚ_pad_to_multipler7   5   s¤   € à�w‰w�s‰|ˆm˜iÑ'€Gäˆq�w‰wŒ<Ü˜Ÿ™“Mˆ	Ø�#‹˜'Ñ!‹Ü�{‰{˜9¨A¯G©GÔ4Ð4àˆ(�Q—V‘VÑ
€CØ�7ˆ|€Cˆ�HÜ
ˆc‘$�B�$‰i˜Ó
€CÜ
�‰×Ñ˜! ¨:¸YÐÓG€AØ€Hó    c                 ó>  — | j                   |   |z  dk7  rt        | ||d¬«      } | j                   |   |z  }| j                   d| ||fz   | j                   |dz   d z   }d|v r,t        j                  || j                  | j
                  ¬«      S | j                  |«      S )zÅSplit an input tensor into blocks of a given `block_len` along the given `dim`. If the dimension length
    is not a multiple of `block_len`, it will be padded first with selected `pad_value`.
    r   )r   Nr   ©r#   Údevice)r,   r7   r/   Úemptyr#   r;   Úreshape)r   r   r   Ú
num_blocksÚoutput_shapes        r6   Ú_split_into_blocksr@   E   sš   € ð
 	‡w�wˆs�|�iÑ 1Ò$Ü˜Q 	¨3¸!Ô<ˆØ—‘˜‘ Ñ*€JØ—7‘7˜4˜C�= J°	Ð#:Ñ:¸Q¿W¹WÀcÈAÁgÀ[Ð=QÑQ€LàˆLÑÜ�{‰{˜<¨q¯w©w¸q¿x¹xÔHÐHØ�9‰9�\Ó"Ð"r8   Ú	block_dimÚsequence_dimc                 óœ  — | j                   |   }dg| j                  z  }d||<   t        |ddd…   d«      }t        j                  j                  | |d|¬«      } g }t        d«      D ]M  }t        d	d«      g| j                  z  }t        |||z   «      ||<   t        |«      }|j                  | |   «       ŒO t        j                  ||¬
«      S )zžConcatenate three consecutive blocks for each input block for local attentiont.

    For more information, see: https://huggingface.co/papers/2112.07916.
    r$   )r   r   Nr%   r&   r'   r(   r   r   ©r   )r,   r1   r2   r   r3   r)   ÚrangeÚsliceÚtupleÚappendr/   Úcat)	r   rA   rB   r   r>   r)   Úblocks_listÚiÚindicess	            r6   Ú_concatenate_3_blocksrM   T   sÐ   € ð
 —‘˜Ñ#€Jàˆ(�Q—V‘VÑ
€CØ€Cˆ	�NÜ
ˆc‘$�B�$‰i˜Ó
€Cä
�‰×Ñ˜! ¨:¸YÐÓG€Aà&(€KÜ�1‹Xò 'ˆô ˜˜D“>Ð" Q§V¡VÑ+ˆÜ" 1 a¨*¡nÓ5ˆ�	ÑÜ˜“.ˆØ×Ñ˜1˜W™:Õ&ð'ô �9‰9�[ lÔ3Ð3r8   c                 ó¨   — t        j                  d| z  t         j                  ¬«      }|| |   }|j                  d«      |j                  d«      z
  }|S )z:Makes 3-blocked relative position ids for local attention.r   r"   r   r   )r/   ÚarangeÚint32Ú	unsqueeze)r   Úposition_idsÚcenter_position_idsÚrelative_position_idss       r6   Ú"_make_3block_relative_position_idsrU   m   sR   € ä—<‘<  I¡´U·[±[ÔA€LØ& y°)°Ð<Ðà(×2Ñ2°1Ó5Ð8K×8UÑ8UÐVWÓ8XÑXÐØ Ð r8   Úlocal_attention_maskc                 óÄ   — t        |«      }t        j                  |«      |k  }|dddd…dd…f   }|j                  | j                  «      }t        j
                  | |«      S )znMask local attention mask to enforce that tokens are not allowed to attend tokens farther than ``local_radius.N)rU   r/   ÚabsÚtor;   Úlogical_and)rV   r   rT   Úlocality_masks       r6   Ú_mask_local_attention_maskr\   v   s_   € ä>¸yÓIÐÜ—I‘IÐ3Ó4°yÑ@€MØ! $¨ªa²Ð"2Ñ3€MØ!×$Ñ$Ð%9×%@Ñ%@ÓA€MÜ×ÑÐ1°=ÓAÐAr8   Úattention_maskr;   c                 ó  — t        | |d¬«      }t        |dd¬«      }|j                  d«      }|j                  d«      }t        j                  ||«      }t        ||«      }|j                  d«      j                  |«      S )z;Prepare attention mask to be applied for a local attention.r   rD   é   ©rA   rB   r%   éþÿÿÿ)r@   rM   rQ   r/   rZ   r\   rY   )r]   r   r;   Ú_blocked_attention_maskÚ_3blocked_attention_maskrV   s         r6   Ú_get_local_attention_maskrd      s‡   € ô 1°ÀÐPQÔRÐä4Ð5LÐXYÐhiÔjÐà5×?Ñ?ÀÓCÐØ7×AÑAÀ"ÓEÐä ×,Ñ,Ð-DÐF^Ó_ÐÜ5Ð6JÈIÓVÐà×)Ñ)¨!Ó,×/Ñ/°Ó7Ð7r8   Úglobal_block_sizec                 ó¨  ‡‡— | j                   dd \  }Šdt        j                  dt        j                  fˆˆfd„}t        j                  | | j                  ¬«      ‰z  }t        j
                  |d¬«      |z
  }t        j                  | d	k7  d
d«      j                  | j                  «      }t        j                  ||z   d
z
  «      j                  | j                  «      }t        j                  d|j                  |j                  ¬«      }t        j                  ||kD  ||«      }|| z  | dz
  z   } ||«      }‰‰z  }|dkD  rBt        j                  |d¬«      j                  j                  |d«      j                  dd«      }	n-t        j                  |d|j                  |j                  ¬«      }	t        j
                  t        j                   ||«      d¬«      dz
  }
|
j#                  | j                  «      }
t        j                  |
|	k  dd«      }
|j                  t        j$                  «      |
j                  t        j$                  «      fS )a  Obtain the "fixed block" global id corresponding to each input token.

    This implementation is a simplified version of the original Flaxformr implementation adopted from:
    https://github.com/google/flaxformer/blob/main/flaxformer/architectures/longt5/long_attention.py.

    In our scenario, as we use this strategy only for a decoder, orphan tokens, i.e. those tokens which do not make for
    the whole fixed block, are assigned to the preceding block.

    Padding tokens from the original sequence are represented by -1.
    Nr_   Ú	block_idsr    c                 óX  •— t        j                  ‰«      ‰z  ‰dz
  k(  }|j                  | j                  «      }t        j                  || dk\  «      }|j                  d«      j                  d«      j                  | j                  «      dz
  }t        j                  | |k  | |«      } | S )Nr   r   r%   )
r/   rO   rY   r;   rZ   r2   rQ   Útyper#   Úwhere)rg   Ú
block_endsÚtrue_block_endsÚfull_blocksre   Úseq_lens       €€r6   Úhandle_orphan_tokensz:_make_global_fixed_block_ids.<locals>.handle_orphan_tokensž   sš   ø€ Ü—l‘l 7Ó+Ð.?Ñ?ÐDUÐXYÑDYÑYˆ
Ø—]‘] 9×#3Ñ#3Ó4ˆ
Ü×+Ñ+¨J¸	ÀQ¹ÓGˆØ%×)Ñ)¨"Ó-×7Ñ7¸Ó;×@Ñ@ÀÇÁÓQÐTUÑUˆÜ—K‘K 	¨KÑ 7¸ÀKÓPˆ	ØÐr8   ©r;   r   )Úaxisç        ç      ð?g     @�Àr%   r:   r   rD   )r,   r/   ÚTensorÚ	ones_liker;   Úcumsumrj   ri   r#   ÚfloorÚtensorÚmaxÚvaluesÚrepeatÚ	transposer0   ÚonesrY   Úint)r]   re   Ú
batch_sizero   Úfixed_block_maskÚmaskÚglobal_block_idsÚ_global_block_ids_lower_boundÚnum_globalsÚ_sequence_block_ids_maxÚglobal_segment_idsrn   s    `         @r6   Ú_make_global_fixed_block_idsr‡   �   s  ù€ ð )×.Ñ.¨r°Ð2Ñ€J�ð¬¯©ð ¼¿¹ö ô —‘ ~¸n×>SÑ>SÔTÐWhÑhÐÜ—|‘|Ð$4¸1Ô=Ð@PÑPÐÜ�;‰;�~¨Ñ,¨c°7Ó;×@Ñ@À×AUÑAUÓV€DÜ—{‘{ 4Ð*:Ñ#:¸SÑ#@ÓA×FÑFÀ~×G[ÑG[Ó\ÐÜ$)§L¡L°Ð;K×;QÑ;QÐZj×ZqÑZqÔ$rÐ!Ü—{‘{ØÐ8Ñ8Ð:JÐLióÐð )¨>Ñ9¸nÈqÑ>PÑQÐá+Ð,<Ó=ÐØÐ.Ñ.€Kà�Q‚Ü"'§)¡)Ð,<À"Ô"E×"LÑ"L×"SÑ"SÐT_ÐabÓ"c×"mÑ"mÐnoÐqrÓ"sÑä"'§+¡+Ø˜Ð!1×!7Ñ!7Ð@P×@WÑ@Wô#
Ðô Ÿ™¤e§j¡j°¸[Ó&IÈrÔRÐUVÑVÐØ+×.Ñ.¨~×/DÑ/DÓEÐÜŸ™Ð%7Ð;RÑ%RÐTUÐWXÓYÐØ× Ñ ¤§¡Ó+Ð-?×-DÑ-DÄUÇYÁYÓ-OÐOÐOr8   c                 óÎ   — t        | |«      \  }}|j                  d   }t        j                  ||j                  ¬«      }||d   z
  }|j                  t        j                  «      S )zBCreate the relative position tensor for local -> global attention.r%   rp   ©.N)r‡   r,   r/   rO   r;   ri   Úint64)r]   re   rg   r†   Úglobal_seq_lenÚglobal_positionsÚside_relative_positions          r6   Ú _make_side_relative_position_idsrŽ   À   sa   € ä$@ÀÐQbÓ$cÑ!€IÐ!Ø'×-Ñ-¨bÑ1€NÜ—|‘| N¸9×;KÑ;KÔLÐØ-°	¸)Ñ0DÑDÐØ!×&Ñ&¤u§{¡{Ó3Ð3r8   Úhidden_statesrg   r‹   c                 óx  — |j                  |dk\  t        j                  ||j                  |j                  ¬«      «      }t
        j                  j                  |j                  t        j                  «      |dz   «      dd…dd…dd…f   }t        j                  d| |j                  | j                  «      «      S )zFCompute individual block aggregates by summing over individual blocks.r   r:   r   Nr%   z...nd,...ng->...gd)rj   r/   rx   r#   r;   r   r3   Úone_hotri   rŠ   Úeinsum)r�   rg   r‹   Úone_hot_block_idss       r6   Ú_create_global_aggregatesr”   É   s›   € ð
 —‘Ø�Q‰œŸ™ ^¸9¿?¹?ÐS\×ScÑScÔdó€Iô Ÿ™×-Ñ-¨i¯n©n¼U¿[¹[Ó.IÈ>Ð\]ÑK]Ó^Ò_`ÒbcÐehÐfhÐehÐ_hÑiÐÜ�<‰<Ð,¨mÐ=N×=SÑ=SÐTa×TgÑTgÓ=hÓiÐir8   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚLongT5LayerNormc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)zg
        Construct a layernorm module in the LongT5 style. No bias and no subtraction of mean.
        N)ÚsuperÚ__init__r   Ú	Parameterr/   r}   ÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s      €r6   r™   zLongT5LayerNorm.__init__×   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕr8   c                 óž  — |j                  t        j                  «      j                  d«      j	                  dd¬«      }|t        j
                  || j                  z   «      z  }| j                  j                  t        j                  t        j                  fv r%|j                  | j                  j                  «      }| j                  |z  S )Nr_   r%   T)Úkeepdim)rY   r/   Úfloat32ÚpowÚmeanÚrsqrtrœ   r›   r#   Úfloat16Úbfloat16)r�   r�   Úvariances      r6   ÚforwardzLongT5LayerNorm.forwardß   s›   € ð !×#Ñ#¤E§M¡MÓ2×6Ñ6°qÓ9×>Ñ>¸rÈ4Ð>ÓPˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆð �;‰;×Ñ¤§¡´·±Ð ?Ñ?Ø)×,Ñ,¨T¯[©[×->Ñ->Ó?ˆMà�{‰{˜]Ñ*Ð*r8   )g�íµ ÷Æ°>)Ú__name__Ú
__module__Ú__qualname__r™   rª   Ú__classcell__©r    s   @r6   r–   r–   Ö   s   ø„ õ$ö+r8   r–   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚLongT5DenseActDenseÚconfigc                 ó^  •— t         ‰| �  «        t        j                  |j                  |j
                  d¬«      | _        t        j                  |j
                  |j                  d¬«      | _        t        j                  |j                  «      | _
        t        |j                     | _        y ©NF©Úbias)r˜   r™   r   ÚLinearÚd_modelÚd_ffÚwiÚwoÚDropoutÚdropout_rateÚdropoutr   Údense_act_fnÚact©r�   r²   r    s     €r6   r™   zLongT5DenseActDense.__init__ñ   sn   ø€ Ü‰ÑÔÜ—)‘)˜FŸN™N¨F¯K©K¸eÔDˆŒÜ—)‘)˜FŸK™K¨¯©¸eÔDˆŒÜ—z‘z &×"5Ñ"5Ó6ˆŒÜ˜&×-Ñ-Ñ.ˆ�r8   c                 ó  — | j                  |«      }| j                  |«      }| j                  |«      }t        | j                  j
                  t        j                  «      r�|j                  | j                  j
                  j                  k7  r`| j                  j
                  j                  t        j                  k7  r/|j                  | j                  j
                  j                  «      }| j	                  |«      }|S ©N)rº   rÀ   r¾   Ú
isinstancer»   r›   r/   rt   r#   Úint8rY   )r�   r�   s     r6   rª   zLongT5DenseActDense.forwardø   sª   € ØŸ™ Ó.ˆØŸ™ Ó/ˆØŸ™ ]Ó3ˆä�t—w‘w—~‘~¤u§|¡|Ô4Ø×#Ñ# t§w¡w§~¡~×';Ñ';Ò;Ø—‘—‘×$Ñ$¬¯
©
Ò2à)×,Ñ,¨T¯W©W¯^©^×-AÑ-AÓBˆMØŸ™ Ó.ˆØÐr8   ©r«   r¬   r­   r   r™   rª   r®   r¯   s   @r6   r±   r±   ð   s   ø„ ð/˜|õ /ör8   r±   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚLongT5DenseGatedActDenser²   c                 óÀ  •— t         ‰| �  «        t        j                  |j                  |j
                  d¬«      | _        t        j                  |j                  |j
                  d¬«      | _        t        j                  |j
                  |j                  d¬«      | _        t        j                  |j                  «      | _        t        |j                     | _        y r´   )r˜   r™   r   r·   r¸   r¹   Úwi_0Úwi_1r»   r¼   r½   r¾   r   r¿   rÀ   rÁ   s     €r6   r™   z!LongT5DenseGatedActDense.__init__  sŠ   ø€ Ü‰ÑÔÜ—I‘I˜fŸn™n¨f¯k©kÀÔFˆŒ	Ü—I‘I˜fŸn™n¨f¯k©kÀÔFˆŒ	Ü—)‘)˜FŸK™K¨¯©¸eÔDˆŒÜ—z‘z &×"5Ñ"5Ó6ˆŒÜ˜&×-Ñ-Ñ.ˆ�r8   c                 ó¶   — | j                  | j                  |«      «      }| j                  |«      }||z  }| j                  |«      }| j	                  |«      }|S rÃ   )rÀ   rÊ   rË   r¾   r»   )r�   r�   Úhidden_geluÚhidden_linears       r6   rª   z LongT5DenseGatedActDense.forward  sS   € Ø—h‘h˜tŸy™y¨Ó7Ó8ˆØŸ	™	 -Ó0ˆØ# mÑ3ˆØŸ™ ]Ó3ˆØŸ™ Ó.ˆØÐr8   rÆ   r¯   s   @r6   rÈ   rÈ     s   ø„ ð/˜|õ /ör8   rÈ   c                   ó*   ‡ — e Zd Zdefˆ fd„Zd„ Zˆ xZS )ÚLongT5LayerFFr²   c                 ó  •— t         ‰| �  «        |j                  rt        |«      | _        nt        |«      | _        t        |j                  |j                  ¬«      | _	        t        j                  |j                  «      | _        y )N©rŸ   )r˜   r™   Úis_gated_actrÈ   ÚDenseReluDenser±   r–   r¸   Úlayer_norm_epsilonÚ
layer_normr   r¼   r½   r¾   rÁ   s     €r6   r™   zLongT5LayerFF.__init__  s_   ø€ Ü‰ÑÔØ×ÒÜ":¸6Ó"BˆDÕä"5°fÓ"=ˆDÔä)¨&¯.©.¸f×>WÑ>WÔXˆŒÜ—z‘z &×"5Ñ"5Ó6ˆ�r8   c                 ór   — | j                  |«      }| j                  |«      }|| j                  |«      z   }|S rÃ   )rÖ   rÔ   r¾   )r�   r�   Úforwarded_statess      r6   rª   zLongT5LayerFF.forward$  s=   € ØŸ?™?¨=Ó9ÐØ×.Ñ.Ð/?Ó@ÐØ%¨¯©Ð5EÓ(FÑFˆØÐr8   rÆ   r¯   s   @r6   rÐ   rÐ     s   ø„ ð7˜|õ 7ör8   rÐ   c                   ó`   ‡ — e Zd Z	 	 ddededz  fˆ fd„Zed	d„«       Zd
d„Z	 	 	 	 	 dd„Z	ˆ xZ
S )ÚLongT5AttentionNr²   Ú	layer_idxc                 óê  •— t         ‰| �  «        |j                  | _        || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _
        |j                  | _        | j                  | j                  z  | _        || _        |€9| j                  r-t        j!                  d| j"                  j$                  › d�«       t'        j(                  | j                  | j                  d¬«      | _        t'        j(                  | j                  | j                  d¬«      | _        t'        j(                  | j                  | j                  d¬«      | _        t'        j(                  | j                  | j                  d¬«      | _        | j                  r/t'        j2                  | j                  | j                  «      | _        d| _        y )NzInstantiating a decoder z³ without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.Frµ   )r˜   r™   Ú
is_decoderÚhas_relative_attention_biasÚrelative_attention_num_bucketsÚrelative_attention_max_distancer¸   Úd_kvÚkey_value_proj_dimÚ	num_headsÚn_headsr½   r¾   Ú	inner_dimrÛ   ÚloggerÚwarning_oncer    r«   r   r·   ÚqÚkÚvÚoÚ	EmbeddingÚrelative_attention_biasÚgradient_checkpointing©r�   r²   rÞ   rÛ   r    s       €r6   r™   zLongT5Attention.__init__-  si  ø€ ô 	‰ÑÔØ ×+Ñ+ˆŒØ+FˆÔ(Ø.4×.SÑ.SˆÔ+Ø/5×/UÑ/UˆÔ,Ø—~‘~ˆŒØ"(§+¡+ˆÔØ×'Ñ'ˆŒØ×*Ñ*ˆŒØŸ™¨×(?Ñ(?Ñ?ˆŒØ"ˆŒØÐ §¢Ü×ÑØ*¨4¯>©>×+BÑ+BÐ*Cð D,ð ,ôô —‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ>™>¨4¯<©<¸eÔDˆŒà×+Ò+Ü+-¯<©<¸×8[Ñ8[Ð]a×]iÑ]iÓ+jˆDÔ(à&+ˆÕ#r8   c                 óT  — d}|rC|dz  }|| dkD  j                  t        j                  «      |z  z  }t        j                  | «      } n*t        j                  | t        j
                  | «      «       } |dz  }| |k  }|t        j                  | j                  «       |z  «      t        j                  ||z  «      z  ||z
  z  j                  t        j                  «      z   }t        j                  |t        j                  ||dz
  «      «      }|t        j                  || |«      z  }|S ©aÒ  
        Adapted from Mesh Tensorflow:
        https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593

        Translate relative position to a bucket number for relative attention. The relative position is defined as
        memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
        position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
        small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
        positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
        This should allow for more graceful generalization to longer sequences than the model has been trained on

        Args:
            relative_position: an int32 Tensor
            bidirectional: a boolean - whether the attention is bidirectional
            num_buckets: an integer
            max_distance: an integer

        Returns:
            a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
        r   r_   r   ©rY   r/   ÚlongrX   ÚminÚ
zeros_likeÚlogÚfloatÚmathÚ	full_likerj   ©Úrelative_positionÚbidirectionalÚnum_bucketsÚmax_distanceÚrelative_bucketsÚ	max_exactÚis_smallÚrelative_position_if_larges           r6   Ú_relative_position_bucketz)LongT5Attention._relative_position_bucketO  s(  € ð, ÐÙØ˜AÑˆKØÐ!2°QÑ!6× :Ñ :¼5¿:¹:Ó FÈÑ TÑTÐÜ %§	¡	Ð*;Ó <Ñä!&§¡Ð+<¼e×>NÑ>NÐO`Ó>aÓ!bÐ bÐð   1Ñ$ˆ	Ø$ yÑ0ˆð &/Ü�I‰IÐ'×-Ñ-Ó/°)Ñ;Ó<Ü�h‰h�| iÑ/Ó0ñ1à˜YÑ&ñ(÷ ‰"ŒU�Z‰Z‹.ñ	&Ð"ô
 &+§Y¡YØ&¬¯©Ð8RÐT_ÐbcÑTcÓ(dó&
Ð"ð 	œEŸK™K¨Ð2CÐE_Ó`Ñ`ÐØÐr8   c                 óÜ  — |€ | j                   j                  j                  }t        j                  |t        j
                  |¬«      dd…df   |z   }t        j                  |t        j
                  |¬«      ddd…f   }||z
  }| j                  || j                   | j                  | j                  ¬«      }| j                  |«      }	|	j                  g d¢«      j                  d«      }	|	S )ú%Compute binned relative position biasNr:   ©rü   rý   rþ   ©r_   r   r   r   )rí   r›   r;   r/   rO   ró   r  rÝ   rß   rà   ÚpermuterQ   )
r�   Úquery_lengthÚ
key_lengthr;   Úpast_seen_tokensÚcontext_positionÚmemory_positionrû   Úrelative_position_bucketrz   s
             r6   Úcompute_biaszLongT5Attention.compute_bias  sÜ   € àˆ>Ø×1Ñ1×8Ñ8×?Ñ?ˆFÜ Ÿ<™<¨¼E¿J¹JÈvÔVÒWXÐZ^ÐW^Ñ_ÐbrÑrÐÜŸ,™, z¼¿¹ÈFÔSÐTXÒZ[ÐT[Ñ\ˆØ+Ð.>Ñ>ÐØ#'×#AÑ#AØØ#Ÿ™Ð.Ø×;Ñ;Ø×=Ñ=ð	 $Bó $
Ð ð ×-Ñ-Ð.FÓGˆØ—‘¢	Ó*×4Ñ4°QÓ7ˆØˆr8   c                 óÊ  — |j                   dd }g |¢d‘| j                  ‘­}	|�|j                  | j                  «      nd}
t	        |
t
        j                  «      r|
j                  «       n|
}
|du}| j                  |«      j                  |	«      j                  dd«      }d}t	        |t        «      rA|j                  j                  | j                  «      }|r|j                  }n|j                  }n|}|r|n|}|rK|�I|rG|j                   | j                     j"                  }|j                   | j                     j$                  }nÍg |j                   dd ¢d‘| j                  ‘­}| j'                  |«      j                  |«      j                  dd«      }| j)                  |«      j                  |«      j                  dd«      }|�K|j+                  ||| j                  «      \  }}|r)t	        |t        «      rd|j                  | j                  <   t        j,                  ||j                  dd«      «      }|€Â|j                   d	   }| j.                  s`t        j0                  d|j                   d   |d   |f|j2                  |j4                  ¬
«      }| j6                  r6| j8                  r*d|_        n"| j=                  |d   ||j2                  |
¬«      }|�#|dd…dd…dd…d|j                   d	   …f   }||z   }|}||z  }t>        j@                  jC                  |jE                  «       d¬«      jG                  |«      }t>        j@                  jI                  || jH                  | j8                  ¬«      }t        j,                  ||«      }|j                  dd«      jK                  «       } |jL                  g |¢d‘­Ž }| jO                  |«      }||f}|r||fz   }|S )z€
        Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states).
        Nr%   r   r   r_   FTr   ra   ©r;   r#   )r;   r  rD   ©ÚpÚtraining)(r,   râ   Úget_seq_lengthrÛ   rÄ   r/   rt   Úclonerè   Úviewr|   r   Ú
is_updatedÚgetÚcross_attention_cacheÚself_attention_cacheÚlayersÚkeysrz   ré   rê   ÚupdateÚmatmulrÞ   r0   r;   r#   rî   r  Úrequires_gradr  r   r3   Úsoftmaxr÷   Útype_asr¾   Ú
contiguousr=   rë   )r�   r�   r�   Úkey_value_statesÚposition_biasÚpast_key_valuesÚoutput_attentionsÚkwargsÚinput_shapeÚhidden_shaper  Úis_cross_attentionÚquery_statesr  Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚkv_shapeÚscoresr
  Úcausal_maskÚposition_bias_maskedÚattn_weightsÚattn_outputÚoutputss                             r6   rª   zLongT5Attention.forward�  s·  € ð $×)Ñ)¨#¨2Ð.ˆØB˜ÐB bÐB¨$×*AÑ*AÑBˆØM\ÐMh˜?×9Ñ9¸$¿.¹.ÔIÐnoÐä7AÐBRÔTY×T`ÑT`Ô7aÐ+×1Ñ1Ô3ÐgwÐð .°TÐ9Ðà—v‘v˜mÓ,×1Ñ1°,Ó?×IÑIÈ!ÈQÓOˆð ˆ
Ü�oÔ':Ô;Ø(×3Ñ3×7Ñ7¸¿¹ÓGˆJÙ!à'6×'LÑ'LÑ$à'6×'KÑ'KÑ$à#2Ð á-?Ñ)À]ˆÙ /Ð"=Á*à-×4Ñ4°T·^±^ÑD×IÑIˆJØ/×6Ñ6°t·~±~ÑF×MÑM‰LàP˜×-Ñ-¨c¨rÐ2ÐP°BÐP¸×8OÑ8OÑPˆHØŸ™ Ó/×4Ñ4°XÓ>×HÑHÈÈAÓNˆJØŸ6™6 .Ó1×6Ñ6°xÓ@×JÑJÈ1ÈaÓPˆLàÐ*Ø+?×+FÑ+FÀzÐS_Ðae×aoÑaoÓ+pÑ(�
˜Lá%¬*°_ÔFYÔ*ZØAE�O×.Ñ.¨t¯~©~Ñ>ô —‘˜l¨J×,@Ñ,@ÀÀAÓ,FÓGˆàÐ Ø#×)Ñ)¨"Ñ-ˆJØ×3Ò3Ü %§¡Ø˜×*Ñ*¨1Ñ-¨{¸1©~¸zÐJÐSY×S`ÑS`Ðhn×htÑhtô!�ð ×.Ò.°4·=²=Ø26�MÕ/à $× 1Ñ 1Ø ‘N J°v·}±}ÐWgð !2ó !�ð ÐØ"¢1¢aªÐ,B¨j×.>Ñ.>¸rÑ.BÐ,BÐ#BÑC�Ø -°Ñ ;�à,ÐØÐ&Ñ&ˆô —}‘}×,Ñ,¨V¯\©\«^ÀÐ,ÓD×LÑLÈVÓTˆÜ—}‘}×,Ñ,¨\¸T¿\¹\ÐTX×TaÑTaÐ,Óbˆä—l‘l <°Ó>ˆà!×+Ñ+¨A¨qÓ1×<Ñ<Ó>ˆØ)�k×)Ñ)Ð;¨;Ð;¸Ò;ˆØ—f‘f˜[Ó)ˆà Ð.ˆáØ  Ñ/ˆGØˆr8   ©FN©Té    é€   )Nr   )NNNNF)r«   r¬   r­   r   r~   r™   Ústaticmethodr  r  rª   r®   r¯   s   @r6   rÚ   rÚ   ,  sX   ø„ ð %*Ø $ñ	 ,àð ,ð ˜‘:õ	 ,ðD ò- ó ð- ó^ð( ØØØØ÷[r8   rÚ   c                   óZ   ‡ — e Zd Zd
dededdfˆ fd„Zedd„«       Zdefd„Z		 	 	 dd	„Z
ˆ xZS )ÚLongT5LocalAttentionr²   rÞ   r    Nc                 ó°  •— t         ‰| �  «        |j                  | _        || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _
        |j                  | _        | j                  dz   | _        |j                  | _        | j                  | j                  z  | _        t!        j"                  | j                  | j                  d¬«      | _        t!        j"                  | j                  | j                  d¬«      | _        t!        j"                  | j                  | j                  d¬«      | _        t!        j"                  | j                  | j                  d¬«      | _        | j                  r/t!        j,                  | j                  | j                  «      | _        d| _        y )Nr   Frµ   )r˜   r™   rÝ   rÞ   rß   rà   r¸   rá   râ   rã   rä   Úlocal_radiusr   r½   r¾   rå   r   r·   rè   ré   rê   rë   rì   rí   rî   ©r�   r²   rÞ   r    s      €r6   r™   zLongT5LocalAttention.__init__ï  sG  ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØ+FˆÔ(Ø.4×.SÑ.SˆÔ+Ø/5×/UÑ/UˆÔ,Ø—~‘~ˆŒØ"(§+¡+ˆÔØ×'Ñ'ˆŒØ"×/Ñ/ˆÔØ×*Ñ*¨QÑ.ˆŒØ×*Ñ*ˆŒØŸ™¨×(?Ñ(?Ñ?ˆŒä—‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ>™>¨4¯<©<¸eÔDˆŒà×+Ò+Ü+-¯<©<¸×8[Ñ8[Ð]a×]iÑ]iÓ+jˆDÔ(à&+ˆÕ#r8   c                 óT  — d}|rC|dz  }|| dkD  j                  t        j                  «      |z  z  }t        j                  | «      } n*t        j                  | t        j
                  | «      «       } |dz  }| |k  }|t        j                  | j                  «       |z  «      t        j                  ||z  «      z  ||z
  z  j                  t        j                  «      z   }t        j                  |t        j                  ||dz
  «      «      }|t        j                  || |«      z  }|S rñ   rò   rú   s           r6   r  z.LongT5LocalAttention._relative_position_bucket  ó(  € ð. ÐÙØ˜AÑˆKØÐ!2°QÑ!6× :Ñ :¼5¿:¹:Ó FÈÑ TÑTÐÜ %§	¡	Ð*;Ó <Ñä!&§¡Ð+<¼e×>NÑ>NÐO`Ó>aÓ!bÐ bÐð   1Ñ$ˆ	Ø$ yÑ0ˆð &/Ü�I‰IÐ'×-Ñ-Ó/°)Ñ;Ó<Ü�h‰h�| iÑ/Ó0ñ1à˜YÑ&ñ(÷ ‰"ŒU�Z‰Z‹.ñ	&Ð"ô
 &+§Y¡YØ&¬¯©Ð8RÐT_ÐbcÑTcÓ(dó&
Ð"ð 	œEŸK™K¨Ð2CÐE_Ó`Ñ`ÐØÐr8   Úblock_lengthc                 ó  — | j                   j                  j                  j                  dk7  r | j                   j                  j                  nd}t	        j
                  d|z  t        j                  |¬«      }|||  }|ddd…f   |dd…df   z
  }| j                  || j                   | j                  | j                  ¬«      }| j                  |«      }|j                  g d¢«      j                  d«      j                  d«      }|S ©r  ÚmetaNr   r:   r  r  r   ©rí   r›   r;   ri   r/   rO   ró   r  rÝ   rß   rà   r  rQ   ©r�   rD  Útarget_devicer  r  rû   r  rz   s           r6   r  z!LongT5LocalAttention.compute_bias8  ó  € ð ×+Ñ+×2Ñ2×9Ñ9×>Ñ>À&ÒHð ×(Ñ(×/Ñ/×6Ò6àð 	ô
  Ÿ,™, q¨<Ñ'7¼u¿z¹zÐR_Ô`ˆØ*¨<¸¸ÐFÐð ,¨D²!¨GÑ4Ð7GÊÈ4ÈÑ7PÑPÐØ#'×#AÑ#AØØ#Ÿ™Ð.Ø×;Ñ;Ø×=Ñ=ð	 $Bó $
Ð ð ×-Ñ-Ð.FÓGˆà—‘¢	Ó*×4Ñ4°QÓ7×AÑAÀ!ÓDˆØˆr8   c                 ó  ‡ ‡— |j                   d d \  Š}ˆˆ fd„}ˆˆ fd„} |‰ j                  |«      «      } |‰ j                  |«      «      }	 |‰ j                  |«      «      }
t	        |‰ j
                  d¬«      }t	        |	‰ j
                  d¬«      }	t	        |
‰ j
                  d¬«      }
t        |	dd¬«      }	t        |
dd¬«      }
t        j                  d||	«      }|€Ê‰ j                  srt        j                  dd‰ j                  ‰ j
                  d‰ j
                  z  f|j                  |j                  ¬	«      }‰ j                  r/‰ j                  r#d
|_        n‰ j#                  ‰ j
                  «      }|�/t        j$                  |dkD  dd«      }||j'                  dd«      z   }||z  }t(        j*                  j-                  |j/                  «       d¬«      j1                  |«      }t(        j*                  j3                  |‰ j2                  ‰ j                  ¬«      }|j5                  |
j                  «      } |t        j                  d||
«      «      }|d d …d |…d d …f   }‰ j7                  |«      }||f}|r||fz   }|S )Nr_   c                 óT   •— | j                  ‰d‰j                  ‰j                  «      S ©Ú
projectionr%   ©r  rä   râ   ©Ústatesr   r�   s    €€r6   r,   z+LongT5LocalAttention.forward.<locals>.shapeY  ó"   ø€ à—;‘;˜z¨2¨t¯|©|¸T×=TÑ=TÓUÐUr8   c                 óZ   •— | j                  «       j                  ‰d‰j                  «      S ©r=   r%   ©r#  r  rå   rQ  s    €€r6   Úunshapez-LongT5LocalAttention.forward.<locals>.unshape]  ó%   ø€ à×$Ñ$Ó&×+Ñ+¨J¸¸D¿N¹NÓKÐKr8   r   rD   r`   ú...qhd,...khd->...hqkr   r  Tr   rr   ç    _ Âr%   r  ú...hqk,...khd->...qhd)r,   rè   ré   rê   r@   r   rM   r/   r’   rÞ   r0   rä   r;   r#   rî   r  r   r  rj   r|   r   r3   r!  r÷   r"  r¾   ri   rë   )r�   r�   r�   r%  r'  Ú
seq_lengthr,   rW  r,  r/  r0  r2  r5  r6  r7  r   s   `              @r6   rª   zLongT5LocalAttention.forwardP  sQ  ù€ ð "/×!4Ñ!4°R°aÐ!8Ñˆ
�Jõ	Võ	Lñ
 ˜TŸV™V MÓ2Ó3ˆÙ˜4Ÿ6™6 -Ó0Ó1ˆ
Ù˜TŸV™V MÓ2Ó3ˆô *¨,¸¿¹ÈAÔNˆÜ'¨
°D·N±NÈÔJˆ
Ü)¨,¸¿¹ÈAÔNˆô +¨:ÀÐQRÔSˆ
Ü,¨\ÀQÐUVÔWˆô —‘Ø# \°:ó
ˆð Ð à×3Ò3Ü %§¡Ø˜˜4Ÿ<™<¨¯©¸¸T¿^¹^Ñ9KÐLÐU[×UbÑUbÐjp×jvÑjvô!�ð ×.Ò.°4·=²=Ø26�MÕ/à $× 1Ñ 1°$·.±.Ó A�àÐä—{‘{ 4¨!¡8¨S°%Ó8�à -°·±¸qÀ!Ó0DÑ D�à�-Ñˆä—}‘}×,Ñ,¨V¯\©\«^ÀÐ,ÓD×LÑLÈVÓTˆä—}‘}×,Ñ,¨\¸T¿\¹\ÐTX×TaÑTaÐ,Óbˆà#×(Ñ(¨×);Ñ);Ó<ˆÙœeŸl™lÐ+BÀLÐR^Ó_Ó`ˆØ!¢! [ j [²!Ð"3Ñ4ˆØ—f‘f˜[Ó)ˆð Øð
ˆñ
 Ø  Ñ/ˆGØˆr8   ©Fr9  ©NNF)r«   r¬   r­   r   Úboolr™   r<  r  r~   r  rª   r®   r¯   s   @r6   r>  r>  î  sP   ø„ ñ,˜|ð ,È$ð ,Ð[_õ ,ð0 ò- ó ð- ð^¨ó ð6 ØØ÷Gr8   r>  c                   óª   ‡ — e Zd Zddededdfˆ fd„Zedd„«       Zdefd„Z	d	e
j                  d
e
j                  de
j                  fd„Z	 	 	 dd„Zˆ xZS )ÚLongT5TransientGlobalAttentionr²   rÞ   r    Nc                 ó†  •— t         ‰| �  «        |j                  | _        || _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _
        |j                  | _        | j                  dz   | _        |j                  | _        |j                  | _        | j                  | j                  z  | _        t#        j$                  | j                  | j                   d¬«      | _        t#        j$                  | j                  | j                   d¬«      | _        t#        j$                  | j                  | j                   d¬«      | _        t#        j$                  | j                   | j                  d¬«      | _        | j                  r/t#        j.                  | j                  | j                  «      | _        | j                  r/t#        j.                  | j                  | j                  «      | _        t5        |j                  |j6                  ¬«      | _        y )Nr   Frµ   rÒ   )r˜   r™   rÝ   rÞ   rß   rà   r¸   rá   râ   rã   rä   r@  r   re   r½   r¾   rå   r   r·   rè   ré   rê   rë   rì   rí   Úglobal_relative_attention_biasr–   rÕ   Úglobal_input_layer_normrA  s      €r6   r™   z'LongT5TransientGlobalAttention.__init__›  s‘  ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØ+FˆÔ(Ø.4×.SÑ.SˆÔ+Ø/5×/UÑ/UˆÔ,Ø—~‘~ˆŒØ"(§+¡+ˆÔØ×'Ñ'ˆŒØ"×/Ñ/ˆÔØ×*Ñ*¨QÑ.ˆŒØ!'×!9Ñ!9ˆÔØ×*Ñ*ˆŒØŸ™¨×(?Ñ(?Ñ?ˆŒä—‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ<™<¨¯©¸eÔDˆŒÜ—‘˜4Ÿ>™>¨4¯<©<¸eÔDˆŒà×+Ò+Ü+-¯<©<¸×8[Ñ8[Ð]a×]iÑ]iÓ+jˆDÔ(ð ×+Ò+Ü24·,±,¸t×?bÑ?bÐdh×dpÑdpÓ2qˆDÔ/Ü'6°v·~±~È6×KdÑKdÔ'eˆÕ$r8   c                 óT  — d}|rC|dz  }|| dkD  j                  t        j                  «      |z  z  }t        j                  | «      } n*t        j                  | t        j
                  | «      «       } |dz  }| |k  }|t        j                  | j                  «       |z  «      t        j                  ||z  «      z  ||z
  z  j                  t        j                  «      z   }t        j                  |t        j                  ||dz
  «      «      }|t        j                  || |«      z  }|S rñ   rò   rú   s           r6   r  z8LongT5TransientGlobalAttention._relative_position_bucket·  rC  r8   rD  c                 ó  — | j                   j                  j                  j                  dk7  r | j                   j                  j                  nd}t	        j
                  d|z  t        j                  |¬«      }|||  }|ddd…f   |dd…df   z
  }| j                  || j                   | j                  | j                  ¬«      }| j                  |«      }|j                  g d¢«      j                  d«      j                  d«      }|S rF  rH  rI  s           r6   r  z+LongT5TransientGlobalAttention.compute_biasè  rK  r8   r�   r†   c                 óv  — t        j                  |d   |d d …d d d …f   «      d d …d df   }t        j                  |dkD  dd«      }t        || j                  «      }| j                  || j                   | j                  | j                  ¬«      }| j                  |«      }|j                  g d¢«      }||z   }|S )Nr‰   .r   rr   rZ  r  )r   r   r   r_   )r/   Úeqrj   rŽ   re   r  rÝ   rß   rà   rc  r  )r�   r�   r†   Úside_attention_maskÚattention_side_biasr�   Úside_relative_position_bucketÚ	side_biass           r6   Úcompute_side_biasz0LongT5TransientGlobalAttention.compute_side_bias   sË   € ä#Ÿh™h t¨I¡Ð8JÊ1ÈdÒTUÈ:Ñ8VÓWÒXYÐ[_ÐadÐXdÑeÐÜ#Ÿk™kÐ*=ÀÑ*AÀ3ÈÓNÐä!AÀ$È×H^ÑH^Ó!_ÐØ(,×(FÑ(FØ"Ø#Ÿ™Ð.Ø×;Ñ;Ø×=Ñ=ð	 )Gó )
Ð%ð ×7Ñ7Ð8UÓVˆ	ð ×%Ñ%¢lÓ3ˆ	à1°IÑ=ÐØ"Ð"r8   c                 ó6	  ‡ ‡— |j                   d d \  Š}ˆˆ fd„}ˆˆ fd„}t        |�|n!t        j                  |j                   d d «      ‰ j                  «      \  }}	|	j                   d   }
t        |||
«      }‰ j                  |«      } |‰ j                  |«      «      } |‰ j                  |«      «      } |‰ j                  |«      «      } |‰ j                  |«      «      } |‰ j                  |«      «      }t        |‰ j                  d¬«      }t        |‰ j                  d¬«      }t        |‰ j                  d¬«      }t        |dd¬«      }t        |dd¬«      }dg|j                  dz   z  }|j                   d   |d<   |j                  d«      j                  |«      }|j                  d«      j                  |«      }t        j                   ||gd¬«      }t        j                   ||gd¬«      }t        j"                  d||«      }|�<t%        |‰ j                  |j&                  «      }t        j(                  |d	kD  d
d«      }nd }|�€j‰ j*                  srt        j,                  dd‰ j.                  ‰ j                  d‰ j                  z  f|j&                  |j0                  ¬«      }‰ j2                  r/‰ j4                  r#d|_        n‰ j9                  ‰ j                  «      }|�||j;                  dd«      z   }|j=                  |j0                  «      }|€t        j                  ‰|«      }‰ j?                  ||	«      }t        |‰ j                  d¬«      j;                  dd«      }|j=                  |j0                  «      jA                  |j&                  «      }t        j                   ||gd¬«      }||z  }tB        jD                  jG                  |jI                  «       d¬«      jK                  |«      }tB        jD                  jM                  |‰ jL                  ‰ j4                  ¬«      }|j=                  |j0                  «      } |t        j"                  d||«      «      }|d d …d |…d d …f   }‰ jO                  |«      }||f}|r||fz   }|S )Nr_   c                 óT   •— | j                  ‰d‰j                  ‰j                  «      S rN  rP  rQ  s    €€r6   r,   z5LongT5TransientGlobalAttention.forward.<locals>.shape  rS  r8   c                 óZ   •— | j                  «       j                  ‰d‰j                  «      S rU  rV  rQ  s    €€r6   rW  z7LongT5TransientGlobalAttention.forward.<locals>.unshape"  rX  r8   r%   r   rD   r`   rY  r   rr   rZ  r   r  Tra   r  r[  )(r,   r‡   r/   r}   re   r”   rd  rè   ré   rê   r@   r   rM   r1   rQ   r{   rI   r’   rd   r;   rj   rÞ   r0   rä   r#   rî   r  r   r  r|   ri   rm  rY   r   r3   r!  r÷   r"  r¾   rë   )r�   r�   r�   r%  r'  r\  r,   rW  rg   r†   Ú_global_seq_lenÚglobal_inputsr,  r/  r0  Úside_key_statesÚside_value_statesÚrepsr2  rV   Úside_position_biasr5  r6  r7  r   s   `                       @r6   rª   z&LongT5TransientGlobalAttention.forward  s0  ù€ ð "/×!4Ñ!4°R°aÐ!8Ñˆ
�Jõ	Võ	Lô )EØÐ$‰D¬%¯*©*°]×5HÑ5HÈÈ"Ð5MÓ*NØ×"Ñ"ó)
Ñ%ˆ	Ð%ð
 -×2Ñ2°2Ñ6ˆÜ1°-ÀÈOÓ\ˆØ×4Ñ4°]ÓCˆñ ˜TŸV™V MÓ2Ó3ˆÙ˜4Ÿ6™6 -Ó0Ó1ˆ
Ù˜TŸV™V MÓ2Ó3ˆá §¡ }Ó 5Ó6ˆÙ! $§&¡&¨Ó"7Ó8Ðô *¨,¸¿¹ÈAÔNˆÜ'¨
°D·N±NÈÔJˆ
Ü)¨,¸¿¹ÈAÔNˆô +¨:ÀÐQRÔSˆ
Ü,¨\ÀQÐUVÔWˆð ˆs�o×*Ñ*¨QÑ.Ñ/ˆØ×"Ñ" 1Ñ%ˆˆQ‰Ø)×3Ñ3°AÓ6×=Ñ=¸dÓCˆØ-×7Ñ7¸Ó:×AÑAÀ$ÓGÐô —Y‘Y 
¨OÐ<À!ÔDˆ
Ü—y‘y ,Ð0AÐ!BÈÔJˆô —‘Ð5°|ÀZÓPˆàÐä#<¸TÀ4Ç>Á>ÐS`×SgÑSgÓ#hÐ ä#(§;¡;Ð/CÀaÑ/GÈÈeÓ#TÑ à#'Ð àÑ à×3Ò3Ü %§¡Ø˜˜4Ÿ<™<¨¯©¸¸T¿^¹^Ñ9KÐLØ!Ÿ=™=Ø Ÿ,™,ô!�ð
 ×.Ò.°4·=²=Ø26�MÕ/à $× 1Ñ 1°$·.±.Ó A�à#Ð/à -Ð0D×0NÑ0NÈqÐRSÓ0TÑ T�Ø)×.Ñ.¨v¯|©|Ó<ˆMð ˆ|Ü—z‘z *¨jÓ9�à!%×!7Ñ!7¸Ð>PÓ!QÐä!3Ð4FÈÏÉÐ\^Ô!_×!iÑ!iÐjkÐmnÓ!oÐØ!3×!8Ñ!8¸¿¹Ó!F×!IÑ!IÈ&Ï-É-Ó!XÐä!ŸI™I }Ð6HÐ&IÈrÔRˆMà�-Ñˆä—}‘}×,Ñ,¨V¯\©\«^ÀÐ,ÓD×LÑLÈVÓTˆÜ—}‘}×,Ñ,¨\¸T¿\¹\ÐTX×TaÑTaÐ,Óbˆà#×(Ñ(¨×);Ñ);Ó<ˆÙœeŸl™lÐ+BÀLÐR^Ó_Ó`ˆØ!¢! [ j [²!Ð"3Ñ4ˆØ—f‘f˜[Ó)ˆà Ð.ˆáØ  Ñ/ˆGØˆr8   r]  r9  r^  )r«   r¬   r­   r   r_  r™   r<  r  r~   r  r/   rt   rm  rª   r®   r¯   s   @r6   ra  ra  š  s}   ø„ ñf˜|ð fÈ$ð fÐ[_õ fð8 ò- ó ð- ð^¨ó ð0# e§l¡lð #ÈÏÉð #ÐY^×YeÑYeó #ð0 ØØ÷qr8   ra  c                   ó>   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 	 dd„Zˆ xZS )ÚLongT5LayerSelfAttentionNrÛ   c                 óÜ   •— t         ‰| �  «        t        |||¬«      | _        t	        |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y )N©rÞ   rÛ   rÒ   )r˜   r™   rÚ   ÚSelfAttentionr–   r¸   rÕ   rÖ   r   r¼   r½   r¾   rï   s       €r6   r™   z!LongT5LayerSelfAttention.__init__‹  sT   ø€ Ü‰ÑÔÜ,ØÐ0KÐW`ô
ˆÔô *¨&¯.©.¸f×>WÑ>WÔXˆŒÜ—z‘z &×"5Ñ"5Ó6ˆ�r8   c                 ó–   — | j                  |«      }| j                  ||||||¬«      }	|| j                  |	d   «      z   }|f|	dd  z   }
|
S )N)r�   r%  r&  Ú	use_cacher'  r   r   )rÖ   r{  r¾   )r�   r�   r]   r%  r&  r}  r'  r(  Únormed_hidden_statesÚattention_outputr7  s              r6   rª   z LongT5LayerSelfAttention.forward“  sp   € ð  $Ÿ™¨}Ó=ÐØ×-Ñ-Ø ØØ'Ø+ØØ/ð .ó 
Ðð &¨¯©Ð5EÀaÑ5HÓ(IÑIˆØ Ð"Ð%5°a°bÐ%9Ñ9ˆØˆr8   r8  )NNNFF©r«   r¬   r­   r~   r™   rª   r®   r¯   s   @r6   rx  rx  Š  s*   ø„ ñ7ÈSÐSWÉZõ 7ð ØØØØ÷r8   rx  c                   óD   ‡ — e Zd ZdZddedz  fˆ fd„Z	 	 	 ddefd„Zˆ xZS )	ÚLongT5LayerLocalSelfAttentionz$Local self attention used in encoderNrÛ   c                 óÚ   •— t         ‰| �  «        t        ||¬«      | _        t	        |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y ©N)rÞ   rÒ   )r˜   r™   r>  ÚLocalSelfAttentionr–   r¸   rÕ   rÖ   r   r¼   r½   r¾   rï   s       €r6   r™   z&LongT5LayerLocalSelfAttention.__init__®  sL   ø€ Ü‰ÑÔÜ"6°vÐ[vÔ"wˆÔÜ)¨&¯.©.¸f×>WÑ>WÔXˆŒÜ—z‘z &×"5Ñ"5Ó6ˆ�r8   r(  c                 ó’   — | j                  |«      }| j                  ||||¬«      }|| j                  |d   «      z   }|f|dd  z   }|S ©N)r�   r%  r'  r   r   )rÖ   r…  r¾   ©	r�   r�   r]   r%  r'  r(  r~  r  r7  s	            r6   rª   z%LongT5LayerLocalSelfAttention.forward´  sj   € ð  $Ÿ™¨}Ó=ÐØ×2Ñ2Ø ØØ'Ø/ð	 3ó 
Ðð &¨¯©Ð5EÀaÑ5HÓ(IÑIˆØ Ð"Ð%5°a°bÐ%9Ñ9ˆØˆr8   r8  r^  ©	r«   r¬   r­   Ú__doc__r~   r™   r   rª   r®   r¯   s   @r6   r‚  r‚  «  s1   ø„ Ù.ñ7ÈSÐSWÉZõ 7ð ØØñð ÷r8   r‚  c                   óD   ‡ — e Zd ZdZddedz  fˆ fd„Z	 	 	 ddefd„Zˆ xZS )	Ú'LongT5LayerTransientGlobalSelfAttentionz/Transient-Global self attention used in encoderNrÛ   c                 óÚ   •— t         ‰| �  «        t        ||¬«      | _        t	        |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y r„  )r˜   r™   ra  ÚTransientGlobalSelfAttentionr–   r¸   rÕ   rÖ   r   r¼   r½   r¾   rï   s       €r6   r™   z0LongT5LayerTransientGlobalSelfAttention.__init__Ë  sQ   ø€ Ü‰ÑÔÜ,JØÐ0Kô-
ˆÔ)ô *¨&¯.©.¸f×>WÑ>WÔXˆŒÜ—z‘z &×"5Ñ"5Ó6ˆ�r8   r(  c                 ó’   — | j                  |«      }| j                  ||||¬«      }|| j                  |d   «      z   }|f|dd  z   }|S r‡  )rÖ   rŽ  r¾   rˆ  s	            r6   rª   z/LongT5LayerTransientGlobalSelfAttention.forwardÓ  sj   € ð  $Ÿ™¨}Ó=ÐØ×<Ñ<Ø ØØ'Ø/ð	 =ó 
Ðð &¨¯©Ð5EÀaÑ5HÓ(IÑIˆØ Ð"Ð%5°a°bÐ%9Ñ9ˆØˆr8   r8  r^  r‰  r¯   s   @r6   rŒ  rŒ  È  s1   ø„ Ù9ñ7ÈSÐSWÉZõ 7ð ØØñð ÷r8   rŒ  c                   ó<   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 dd„Zˆ xZS )ÚLongT5LayerCrossAttentionNrÛ   c                 óÜ   •— t         ‰| �  «        t        |d|¬«      | _        t	        |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y )NFrz  rÒ   )r˜   r™   rÚ   ÚEncDecAttentionr–   r¸   rÕ   rÖ   r   r¼   r½   r¾   )r�   r²   rÛ   r    s      €r6   r™   z"LongT5LayerCrossAttention.__init__é  sO   ø€ Ü‰ÑÔÜ.¨vÐSXÐdmÔnˆÔÜ)¨&¯.©.¸f×>WÑ>WÔXˆŒÜ—z‘z &×"5Ñ"5Ó6ˆ�r8   c                 ó–   — | j                  |«      }| j                  ||||||¬«      }	|| j                  |	d   «      z   }
|
f|	dd  z   }|S )N)r�   r$  r%  r&  r'  r   r   )rÖ   r“  r¾   )r�   r�   r$  r]   r%  r&  r'  r(  r~  r  Úlayer_outputr7  s               r6   rª   z!LongT5LayerCrossAttention.forwardï  so   € ð  $Ÿ™¨}Ó=ÐØ×/Ñ/Ø ØØ-Ø'Ø+Ø/ð 0ó 
Ðð % t§|¡|Ð4DÀQÑ4GÓ'HÑHˆØ�/Ð$4°Q°RÐ$8Ñ8ˆØˆr8   rÃ   )NNNFr€  r¯   s   @r6   r‘  r‘  è  s&   ø„ ñ7¨#°©*õ 7ð ØØØ÷r8   r‘  c                   óF   ‡ — e Zd Zddedz  fˆ fd„Z	 	 	 	 	 	 	 	 	 dd„Zˆ xZS )ÚLongT5BlockNrÛ   c                 ó  •— t         ‰| �  «        |j                  | _        |j                  rt        }nE|j                  dk(  rt
        }n/|j                  dk(  rt        }nt        d|j                  › d�«      ‚t        j                  «       | _
        | j                  j                   ||||¬«      «       | j                  r&| j                  j                  t        ||¬«      «       | j                  j                  t        |«      «       y )NÚlocalztransient-globalzjFor encoder attention mechanism, either `local` or `transient-global` attention type is expected, but got ú.rz  )rÛ   )r˜   r™   rÝ   rx  Úencoder_attention_typer‚  rŒ  Ú
ValueErrorr   Ú
ModuleListÚlayerrH   r‘  rÐ   )r�   r²   rÞ   rÛ   Úattention_layerr    s        €r6   r™   zLongT5Block.__init__  sÞ   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØ×ÒÜ6‰OØ×*Ñ*¨gÒ5Ü;‰OØ×*Ñ*Ð.@Ò@ÜE‰OäðØ!×8Ñ8Ð9¸ð<óð ô —]‘]“_ˆŒ
Ø�
‰
×ÑÙ˜FÐ@[ÐgpÔqô	
ð �?Š?Ø�J‰J×ÑÔ7¸È)ÔTÔUà�
‰
×Ñœ-¨Ó/Õ0r8   c                 ó  —  | j                   d   ||||||	¬«      }|d   }|dd  }|j                  t        j                  k(  rht        j                  |«      j                  «       rEt        j                  |j                  «      j                  dz
  }t        j                  || |¬«      }| j                  xr |d u}|r­ | j                   d   ||||||	¬«      }|d   }|j                  t        j                  k(  rht        j                  |«      j                  «       rEt        j                  |j                  «      j                  dz
  }t        j                  || |¬«      }||dd  z   } | j                   d   |«      }|j                  t        j                  k(  rht        j                  |«      j                  «       rEt        j                  |j                  «      j                  dz
  }t        j                  || |¬«      }|f|z   S )Nr   )r]   r%  r&  r}  r'  r   iè  )rô   ry   )r$  r]   r%  r&  r'  r%   )
rž  r#   r/   r§   ÚisinfÚanyÚfinfory   ÚclamprÝ   )r�   r�   r]   r%  Úencoder_hidden_statesÚencoder_attention_maskÚencoder_decoder_position_biasr&  r}  r'  Úreturn_dictr(  Úself_attention_outputsÚattention_outputsÚclamp_valueÚdo_cross_attentionÚcross_attention_outputss                    r6   rª   zLongT5Block.forward  sß  € ð "/ §¡¨A¡ØØ)Ø'Ø+ØØ/ô"
Ðð /¨qÑ1ˆØ2°1°2Ð6Ðð ×Ñ¤%§-¡-Ò/´E·K±KÀÓ4N×4RÑ4RÔ4TÜŸ+™+ m×&9Ñ&9Ó:×>Ñ>ÀÑEˆKÜ!ŸK™K¨¸K¸<È[ÔYˆMà!Ÿ_™_ÒRÐ1FÈdÐ1RÐÙØ&3 d§j¡j°¡mØØ!6Ø5Ø;Ø /Ø"3ô'Ð#ð 4°AÑ6ˆMð ×"Ñ"¤e§m¡mÒ3¼¿¹ÀMÓ8R×8VÑ8VÔ8XÜ#Ÿk™k¨-×*=Ñ*=Ó>×BÑBÀTÑI�Ü %§¡¨MÀ¸|ÐQ\Ô ]�ð !2Ð4KÈAÈBÐ4OÑ OÐð '˜Ÿ
™
 2™ }Ó5ˆð ×Ñ¤%§-¡-Ò/´E·K±KÀÓ4N×4RÑ4RÔ4TÜŸ+™+ m×&9Ñ&9Ó:×>Ñ>ÀÑEˆKÜ!ŸK™K¨¸K¸<È[ÔYˆMð ÐÐ0Ñ0ð	
r8   r8  )	NNNNNNFFTr€  r¯   s   @r6   r—  r—    s6   ø„ ñ1ÈSÐSWÉZõ 1ð4 ØØ"Ø#Ø&*ØØØØ÷<
r8   r—  c                   óz   ‡ — e Zd ZU eed<   dZdZdgZdZe	d„ «       Z
 ej                  «       ˆ fd„«       Zd„ Zˆ xZS )	ÚLongT5PreTrainedModelr²   ÚtransformerTr—  Fc                 óv   — t        j                  t        «      }t        j                  t        «      }|||dœ}|S )N)Údecoder_input_idsÚ	input_idsÚdecoder_attention_mask)r/   rx   r   r   )r�   r³  Ú
input_maskÚdummy_inputss       r6   r¶  z"LongT5PreTrainedModel.dummy_inputsg  s8   € ô —L‘L¤Ó.ˆ	Ü—\‘\¤*Ó-ˆ
à!*Ø"Ø&0ñ
ˆð
 Ðr8   c                 ó°  •— t         ‰| �  |«       | j                  j                  }t	        |t
        «      r$t        j                  |j                  |dz  «       yt	        |t        t        t        f«      rƒt        j                  |j                  j                  d|dz  ¬«       t        |d«      rG| j                  j                  s0t        j                  |j                   j                  d|dz  ¬«       yyyt	        |t"        «      �r9t        j                  |j$                  j                  d|| j                  j&                  dz  z  ¬«       t        |j$                  d«      r?|j$                  j(                  �)t        j*                  |j$                  j(                  «       t        j                  |j,                  j                  d|| j                  j.                  dz  z  ¬«       t        |j,                  d«      rA|j,                  j(                  �*t        j*                  |j,                  j(                  «       yyyt	        |t0        «      �rÔt        j                  |j2                  j                  d|| j                  j&                  dz  z  ¬«       t        |j2                  d«      r?|j2                  j(                  �)t        j*                  |j2                  j(                  «       t        j                  |j4                  j                  d|| j                  j&                  dz  z  ¬«       t        |j4                  d«      r?|j4                  j(                  �)t        j*                  |j4                  j(                  «       t        j                  |j,                  j                  d|| j                  j.                  dz  z  ¬«       t        |j,                  d«      rA|j,                  j(                  �*t        j*                  |j,                  j(                  «       yyyt	        |t6        t8        t:        f«      �r“| j                  j&                  }| j                  j<                  }| j                  j>                  }t        j                  |j@                  j                  d|||z  dz  z  ¬«       t        j                  |jB                  j                  d||dz  z  ¬«       t        j                  |jD                  j                  d||dz  z  ¬«       t        j                  |jF                  j                  d|||z  dz  z  ¬«       |jH                  rvt        j                  |jJ                  j                  d||dz  z  ¬«       t	        |t:        «      r3t        j                  |jL                  j                  d||dz  z  ¬«       yyyy)zInitialize the weightsrs   rr   )r¥   ÚstdÚlm_headç      à¿r¶   N)'r˜   Ú_init_weightsr²   Úinitializer_factorrÄ   r–   ÚinitÚ	constant_r›   ÚLongT5ModelÚLongT5ForConditionalGenerationÚLongT5EncoderModelÚnormal_ÚsharedÚhasattrÚtie_word_embeddingsr¹  r±   rº   r¸   r¶   Úzeros_r»   r¹   rÈ   rÊ   rË   rÚ   r>  ra  rá   rã   rè   ré   rê   rë   rÞ   rí   rc  )r�   ÚmoduleÚfactorr¸   râ   rä   r    s         €r6   r»  z#LongT5PreTrainedModel._init_weightss  s  ø€ ô 	‰Ñ˜fÔ%Ø—‘×/Ñ/ˆÜ�fœoÔ.Ü�N‰N˜6Ÿ=™=¨&°3©,Õ7Ü˜¤Ô.LÔN`Ð aÔbÜ�L‰L˜Ÿ™×-Ñ-°C¸VÀc¹\ÕJÜ�v˜yÔ)°$·+±+×2QÒ2QÜ—‘˜VŸ^™^×2Ñ2¸À&È3Á,ÖOð 3RÐ)ä˜Ô 3Õ4Ü�L‰L˜Ÿ™×)Ñ)°¸ÀDÇKÁK×DWÑDWÐ\`ÑC`Ñ9aÕbÜ�v—y‘y &Ô)¨f¯i©i¯n©nÐ.HÜ—‘˜FŸI™IŸN™NÔ+Ü�L‰L˜Ÿ™×)Ñ)°¸ÀDÇKÁK×DTÑDTÐY]ÑC]Ñ9^Õ_Ü�v—y‘y &Ô)¨f¯i©i¯n©nÐ.HÜ—‘˜FŸI™IŸN™NÕ+ð /IÐ)ä˜Ô 8Õ9Ü�L‰L˜Ÿ™×+Ñ+°#¸6ÀdÇkÁk×FYÑFYÐ^bÑEbÑ;cÕdÜ�v—{‘{ FÔ+°·±×0@Ñ0@Ð0LÜ—‘˜FŸK™K×,Ñ,Ô-Ü�L‰L˜Ÿ™×+Ñ+°#¸6ÀdÇkÁk×FYÑFYÐ^bÑEbÑ;cÕdÜ�v—{‘{ FÔ+°·±×0@Ñ0@Ð0LÜ—‘˜FŸK™K×,Ñ,Ô-Ü�L‰L˜Ÿ™×)Ñ)°¸ÀDÇKÁK×DTÑDTÐY]ÑC]Ñ9^Õ_Ü�v—y‘y &Ô)¨f¯i©i¯n©nÐ.HÜ—‘˜FŸI™IŸN™NÕ+ð /IÐ)ä˜¤Ô2FÔHfÐ gÕhØ—k‘k×)Ñ)ˆGØ!%§¡×!1Ñ!1ÐØ—k‘k×+Ñ+ˆGÜ�L‰L˜Ÿ™Ÿ™¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÕjÜ�L‰L˜Ÿ™Ÿ™¨s¸À'È4Á-Ñ8PÕQÜ�L‰L˜Ÿ™Ÿ™¨s¸À'È4Á-Ñ8PÕQÜ�L‰L˜Ÿ™Ÿ™¨s¸À7ÐM_ÑC_ÐdhÑBhÑ8iÕjØ×1Ò1Ü—‘˜V×;Ñ;×BÑBÈÐRXÐ]dÐimÑ\mÑRnÕoÜ˜fÔ&DÔEÜ—L‘LØ×=Ñ=×DÑDÈ3ÐTZÐ_fÐkoÑ^oÑTpöð Fð 2ð ir8   c                 ó8  — | j                   j                  }| j                   j                  }|€t        d«      ‚|j	                  |j
                  «      }|dd d…f   j                  «       |ddd …f<   ||d<   |€t        d«      ‚|j                  |dk(  |«       |S )Nz’self.model.config.decoder_start_token_id has to be defined. In LongT5 it is usually set to the pad_token_id. See LongT5 docs for more information..r%   r   ).r   z1self.model.config.pad_token_id has to be defined.éœÿÿÿ)r²   Údecoder_start_token_idÚpad_token_idrœ  Ú	new_zerosr,   r  Úmasked_fill_)r�   r³  rË  rÌ  Úshifted_input_idss        r6   Ú_shift_rightz"LongT5PreTrainedModel._shift_rightŸ  s°   € Ø!%§¡×!CÑ!CÐØ—{‘{×/Ñ/ˆà!Ð)Üð8óð ð
 &×/Ñ/°	·±Ó@ÐØ%.¨s°C°R°C¨xÑ%8×%>Ñ%>Ó%@Ð˜#˜q™r˜'Ñ"Ø$:Ð˜&Ñ!àÐÜÐPÓQÐQà×&Ñ&Ð'8¸DÑ'@À,ÔOà Ð r8   )r«   r¬   r­   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_can_compile_fullgraphÚpropertyr¶  r/   Úno_gradr»  rÐ  r®   r¯   s   @r6   r¯  r¯  ^  sW   ø… àÓØ%ÐØ&*Ð#Ø&˜Ðà"Ðàñó ðð €U‡]�]ƒ_ó(ó ð(öV!r8   r¯  c                   ó@   ‡ — e Zd Zˆ fd„Zd„ Z	 	 	 	 	 	 	 	 	 	 dd„Zˆ xZS )ÚLongT5Stackc                 ó`  •— t         ‰| �  |«       t        j                  |j                  |j
                  «      | _        |j                  | _        |j                  | _        | j                  dz   | _	        t        j                  t        |j                  «      D �cg c]  }t        |t        |dk(  «      |¬«      ‘Œ c}«      | _        t!        |j
                  |j"                  ¬«      | _        t        j&                  |j(                  «      | _        d| _        | j/                  «        y c c}w )Nr   r   rz  rÒ   F)r˜   r™   r   rì   Ú
vocab_sizer¸   Úembed_tokensrÝ   r@  r   r�  rE   Ú
num_layersr—  r_  Úblockr–   rÕ   Úfinal_layer_normr¼   r½   r¾   rî   Ú	post_init)r�   r²   rK   r    s      €r6   r™   zLongT5Stack.__init__¶  sé   ø€ Ü‰Ñ˜Ô äŸL™L¨×):Ñ):¸F¿N¹NÓKˆÔØ ×+Ñ+ˆŒà"×/Ñ/ˆÔØ×*Ñ*¨QÑ.ˆŒä—]‘]ô ˜v×0Ñ0Ó1öàô ˜FÄÀQÈ!ÁVÃÐXYÖZòó
ˆŒ
ô !0°·±ÀF×D]ÑD]Ô ^ˆÔÜ—z‘z &×"5Ñ"5Ó6ˆŒà&+ˆÔ#ð 	�‰Õùòs   Â!D+c                 ó   — || _         y rÃ   )rÜ  ©r�   Únew_embeddingss     r6   Úset_input_embeddingsz LongT5Stack.set_input_embeddingsÎ  s
   € Ø*ˆÕr8   c                 ó   — |�|n| j                   j                  }|�|n| j                   j                  }|	�|	n| j                   j                  }	|
�|
n| j                   j                  }
|�$|�"| j
                  rdnd}t        d|› d|› d�«      ‚|�&|j                  «       }|j                  d|d   «      }n8|�|j                  «       d d }n"| j
                  rdnd}t        d|› d|› d	�«      ‚| j                  r%| j                  r|rt        j                  d
«       d}|€$| j                  €J d«       ‚| j                  |«      }|\  }}| j
                  rf|rr|€p| j                   j                  r5t        t!        | j                   ¬«      t!        | j                   ¬«      «      }n%t!        | j                   ¬«      }n| j
                  sd }|�|j#                  «       nd}|€1t%        «       s'||z   }t'        j(                  |||j*                  ¬«      }| j
                  rt-        | j                   |||¬«      }n=| j                   j.                  dk(  r"t1        || j2                  |j*                  «      }n|}|�t5        | j                   |||¬«      }|	rdnd }|rdnd }|r| j
                  rdnd }d }d }| j7                  |«      }t9        | j:                  «      D ]b  \  }}|	r||fz   } |||||||||||
¬«
      }|d   }|d   }| j
                  r|�	||rdnd   }|sŒD||d   fz   }| j
                  sŒZ||d   fz   }Œd | j=                  |«      }| j7                  |«      }|	r||fz   }|
st?        d„ |||||fD «       «      S tA        |||||¬«      S )NÚdecoder_Ú zYou cannot specify both zinput_ids and zinputs_embeds at the same timer%   zYou have to specify either zinput_ids or Úinputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Fz<You have to initialize the model with valid token embeddings)r²   r   rp   )r²   rè  r]   r&  r™  )r²   rè  r]   r¥  r&   )r&  r}  r'  r¨  r   r   r_   é   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wrÃ   r&   )Ú.0rê   s     r6   ú	<genexpr>z&LongT5Stack.forward.<locals>.<genexpr>[  s   è ø€ ò 
àð �=ô ñ
ùs   ‚)Úlast_hidden_stater&  r�   Ú
attentionsÚcross_attentions)!r²   r}  r'  Úoutput_hidden_statesr¨  rÝ   rœ  Úsizer  rî   r  ræ   rç   rÜ  Úis_encoder_decoderr   r
   r  r   r/   r}   r;   r   r›  rd   r   r   r¾   Ú	enumeraterÞ  rß  rG   r   )r�   r³  r]   r¥  r¦  rè  r&  r}  r'  rð  r¨  r(  Úerr_msg_prefixr)  r   r\  Úpast_key_values_lengthÚmask_seq_lengthr3  Úall_hidden_statesÚall_attentionsÚall_cross_attentionsr%  r§  r�   rK   Úlayer_moduleÚlayer_outputss                               r6   rª   zLongT5Stack.forwardÑ  s  € ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø1BÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐ  ]Ð%>Ø+/¯?ª?™ZÀˆNÜØ*¨>Ð*:¸.ÈÐHXÐXvÐwóð ð Ð"Ø#Ÿ.™.Ó*ˆKØ!Ÿ™ r¨;°r©?Ó;‰IØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰Kà+/¯?ª?™ZÀˆNÜÐ:¸>Ð:JÈ-ÐXfÐWgÐgtÐuÓvÐvà×&Ò&¨4¯=ª=ÙÜ×#Ñ#Øpôð "�	àÐ Ø×$Ñ$Ð0ÐpÐ2pÓpÐ0Ø ×-Ñ-¨iÓ8ˆMà!,Ñˆ
�Jà�?Š?Ù˜_Ð4Ø—;‘;×1Ò1Ü&9Ü$¨D¯K©KÔ8¼,ÈdÏkÉkÔ:Zó'‘Oô '3¸$¿+¹+Ô&F‘OØ—’ð #ˆOàETÐE` ×!?Ñ!?Ô!AÐfgÐØÐ!Ô*BÔ*Dà4°zÑAˆOÜ"ŸZ™Z¨
°OÈM×L`ÑL`ÔaˆNà�?Š?Ü,Ø—{‘{Ø+Ø-Ø /ô	‰Kð �[‰[×/Ñ/°7Ò:Ü3°NÀDÇNÁNÐTa×ThÑThÓi‰Kà(ˆKà!Ð-Ü%>Ø—{‘{Ø+Ø5Ø&;ô	&Ð"ñ #7™B¸DÐÙ0™°dˆÙ&7¸D¿OºO™rÐRVÐØˆØ(,Ð%àŸ™ ]Ó3ˆä(¨¯©Ó4ò  	V‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!á(ØØØØ%Ø&Ø-Ø /Ø#Ø"3Ø'ôˆMð  *¨!Ñ,ˆMð
 *¨!Ñ,ˆMØ�ŠÐ#8Ð#DØ0=ÑCT¹aÐZ[Ñ0\Ð-â Ø!/°=ÀÑ3CÐ2EÑ!E�Ø—?“?Ø+?À=ÐQRÑCSÐBUÑ+UÑ(ðA 	VðD ×-Ñ-¨mÓ<ˆØŸ™ ]Ó3ˆñ  Ø 1°]Ð4DÑ DÐáÜñ 
ð "Ø#Ø%Ø"Ø(ðô
ó 
ð 
ô 9Ø+Ø+Ø+Ø%Ø1ô
ð 	
r8   )
NNNNNNNNNN)r«   r¬   r­   r™   rä  rª   r®   r¯   s   @r6   rÙ  rÙ  µ  s3   ø„ ôò0+ð
 ØØ"Ø#ØØØØØ!Ø÷[
r8   rÙ  c                   ó   ‡ — e Zd ZdgZdddœZdefˆ fd„Zd„ Zd„ Ze		 	 	 	 	 	 	 	 	 	 	 	 dd	e
j                  dz  d
e
j                  dz  de
j                  dz  de
j                  dz  deee
j                        dz  dedz  de
j                   dz  de
j                   dz  dedz  dedz  dedz  dedz  dee
j                     ez  fd„«       Zˆ xZS )r¿  úFdecoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weightúshared.weight)úencoder.embed_tokens.weightúdecoder.embed_tokens.weightr²   c                 ó‚  •— t         ‰| �  |«       t        j                  |j                  |j
                  «      | _        t        j                  |«      }d|_	        d|_
        t        |«      | _        t        j                  |«      }d|_	        |j                  |_        t        |«      | _        | j!                  «        y )NFT)r˜   r™   r   rì   rÛ  r¸   rÃ  ÚcopyÚdeepcopyrÝ   r}  rÙ  ÚencoderÚnum_decoder_layersrÝ  Údecoderrà  ©r�   r²   Úencoder_configÚdecoder_configr    s       €r6   r™   zLongT5Model.__init__y  s’   ø€ Ü‰Ñ˜Ô Ü—l‘l 6×#4Ñ#4°f·n±nÓEˆŒäŸ™ vÓ.ˆØ$)ˆÔ!Ø#(ˆÔ Ü" >Ó2ˆŒäŸ™ vÓ.ˆØ$(ˆÔ!Ø$*×$=Ñ$=ˆÔ!Ü" >Ó2ˆŒð 	�‰Õr8   c                 ó   — | j                   S rÃ   ©rÃ  ©r�   s    r6   Úget_input_embeddingsz LongT5Model.get_input_embeddingsŠ  ó   € Ø�{‰{Ðr8   c                 ó~   — || _         | j                  j                  |«       | j                  j                  |«       y rÃ   ©rÃ  r  rä  r  râ  s     r6   rä  z LongT5Model.set_input_embeddings�  ó-   € Ø$ˆŒØ�‰×)Ñ)¨.Ô9Ø�‰×)Ñ)¨.Õ9r8   Nr³  r]   r²  r´  Úencoder_outputsr&  rè  Údecoder_inputs_embedsr}  r'  rð  r¨  r    c                 óD  — |	�|	n| j                   j                  }	|�|n| j                   j                  }|€| j                  ||||
||¬«      }nI|rGt	        |t
        «      s7t        |d   t        |«      dkD  r|d   ndt        |«      dkD  r|d   nd¬«      }|d   }| j                  |||||||	|
||¬«
      }|s||z   S t        |j                  |j                  |j                  |j                  |j                  |j                  |j                  |j                  ¬«      S )	aµ	  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5
            Training](./longt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            LONGT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If
            `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
            `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [LONGT5
            Training](./longt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, LongT5Model

        >>> tokenizer = AutoTokenizer.from_pretrained("google/long-t5-local-base")
        >>> model = LongT5Model.from_pretrained("google/long-t5-local-base")

        >>> # Let's try a very long encoder input.
        >>> input_ids = tokenizer(
        ...     100 * "Studies have been shown that owning a dog is good for you", return_tensors="pt"
        ... ).input_ids  # Batch size 1

        >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids  # Batch size 1

        >>> # forward pass
        >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        ```N©r³  r]   rè  r'  rð  r¨  r   r   r_   ©rí  r�   rî  ©
r³  r]   rè  r&  r¥  r¦  r}  r'  rð  r¨  )rí  r&  Údecoder_hidden_statesÚdecoder_attentionsrï  Úencoder_last_hidden_stater¥  Úencoder_attentions)r²   r}  r¨  r  rÄ   r   Úlenr  r   rí  r&  r�   rî  rï  )r�   r³  r]   r²  r´  r  r&  rè  r  r}  r'  rð  r¨  r(  r�   Údecoder_outputss                   r6   rª   zLongT5Model.forward’  sO  € ðD "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ð%<‘kÀ$Ç+Á+×BYÑBYˆð Ð"Ø"Ÿl™lØ#Ø-Ø+Ø"3Ø%9Ø'ð +ó ‰Oñ ¤¨O¼_Ô!MÜ-Ø"1°!Ñ"4Ü47¸Ó4HÈ1Ò4L˜o¨aÒ0ÐRVÜ14°_Ó1EÈÒ1I˜?¨1Ò-ÈtôˆOð (¨Ñ*ˆð Ÿ,™,Ø'Ø1Ø/Ø+Ø"/Ø#1ØØ/Ø!5Ø#ð 'ó 
ˆñ Ø" _Ñ4Ð4ä!Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô	
ð 		
r8   )NNNNNNNNNNNN)r«   r¬   r­   Ú"_keys_to_ignore_on_load_unexpectedÚ_tied_weights_keysr   r™   r  rä  r   r/   Ú
LongTensorÚFloatTensorÚ
BoolTensorrG   r	   rt   r_  r   rª   r®   r¯   s   @r6   r¿  r¿  o  sx  ø„ ð 	Rð*Ð&ð (7Ø'6ñÐð
˜|õ ò"ò:ð
 ð .2Ø37Ø59Ø:>ØBFØ(,Ø-1Ø59Ø!%Ø)-Ø,0Ø#'ñq
à×#Ñ# dÑ*ðq
ð ×)Ñ)¨DÑ0ðq
ð !×+Ñ+¨dÑ2ð	q
ð
 !&× 0Ñ 0°4Ñ 7ðq
ð ˜u U×%6Ñ%6Ñ7Ñ8¸4Ñ?ðq
ð  ™ðq
ð —|‘| dÑ*ðq
ð  %Ÿ|™|¨dÑ2ðq
ð ˜$‘;ðq
ð   $™;ðq
ð # T™kðq
ð ˜D‘[ðq
ð 
ˆu× Ñ Ñ	!Ð$6Ñ	6òq
ó ôq
r8   r¿  z>
    LONGT5 Model with a `language modeling` head on top.
    )Úcustom_introc                   óâ  ‡ — e Zd ZdgZddddœZdefˆ fd„Zd„ Zd„ Ze		 	 	 	 	 	 	 	 	 	 	 	 	 dd	e
j                  dz  d
e
j                  dz  de
j                  dz  de
j                  dz  deee
j                        dz  dedz  de
j                  dz  de
j                  dz  de
j                  dz  dedz  dedz  dedz  dedz  dee
j                     ez  fd„«       Zde
j                  fd„Zˆ xZS )rÀ  rý  rþ  )rÿ  r   zlm_head.weightr²   c                 ó  •— t         ‰| �  |«       |j                  | _        t	        j
                  |j                  |j                  «      | _        t        j                  |«      }d|_
        d|_        t        |«      | _        t        j                  |«      }d|_
        |j                  |_        t        |«      | _        t	        j"                  |j                  |j                  d¬«      | _        | j'                  «        y )NFTrµ   )r˜   r™   r¸   Ú	model_dimr   rì   rÛ  rÃ  r  r  rÝ   r}  rÙ  r  r  rÝ  r  r·   r¹  rà  r  s       €r6   r™   z'LongT5ForConditionalGeneration.__init__  s»   ø€ Ü‰Ñ˜Ô ØŸ™ˆŒä—l‘l 6×#4Ñ#4°f·n±nÓEˆŒäŸ™ vÓ.ˆØ$)ˆÔ!Ø#(ˆÔ Ü" >Ó2ˆŒäŸ™ vÓ.ˆØ$(ˆÔ!Ø$*×$=Ñ$=ˆÔ!Ü" >Ó2ˆŒä—y‘y §¡°×1BÑ1BÈÔOˆŒð 	�‰Õr8   c                 ó   — | j                   S rÃ   r  r  s    r6   r  z3LongT5ForConditionalGeneration.get_input_embeddings+  r  r8   c                 ó~   — || _         | j                  j                  |«       | j                  j                  |«       y rÃ   r  râ  s     r6   rä  z3LongT5ForConditionalGeneration.set_input_embeddings.  r  r8   Nr³  r]   r²  r´  r  r&  rè  r  Úlabelsr}  r'  rð  r¨  r    c                 óÂ  — |
�|
n| j                   j                  }
|�|n| j                   j                  }|€| j                  ||||||¬«      }nI|rGt	        |t
        «      s7t        |d   t        |«      dkD  r|d   ndt        |«      dkD  r|d   nd¬«      }|d   }|	�|€|€| j                  |	«      }| j                  |||||||
|||¬«
      }|d   }| j                   j                  r|| j                  dz  z  }| j                  |«      }d}|	�^t        d	¬
«      }|	j                  |j                  «      }	 ||j                  d|j!                  d«      «      |	j                  d«      «      }|s|f|dd z   |z   }|�|f|z   S |S t#        |||j$                  |j&                  |j(                  |j*                  |j,                  |j&                  |j(                  ¬«	      S )a7  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            [What are input IDs?](../glossary#input-ids)

            To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5
            Training](./longt5#training).
        decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Indices of decoder input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are decoder input IDs?](../glossary#decoder-input-ids)

            LONGT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If
            `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
            `past_key_values`).

            To know more on how to prepare `decoder_input_ids` for pretraining take a look at [LONGT5
            Training](./longt5#training).
        decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
            Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
            be used by default.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
            config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
            labels in `[0, ..., config.vocab_size]`

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, LongT5ForConditionalGeneration

        >>> tokenizer = AutoTokenizer.from_pretrained("Stancld/longt5-tglobal-large-16384-pubmed-3k_steps")
        >>> model = LongT5ForConditionalGeneration.from_pretrained(
        ...     "Stancld/longt5-tglobal-large-16384-pubmed-3k_steps"
        ... )

        >>> # Let's try a very long input.
        >>> inputs = tokenizer(100 * "studies have shown that owning a dog is good for you ", return_tensors="pt")
        >>> input_ids = inputs.input_ids

        >>> outputs = model.generate(input_ids)
        >>> print(tokenizer.decode(outputs[0], skip_special_tokens=True))
        abstractthe aim of this article is to provide an overview of the literature on the role of dog
        ```Nr  r   r   r_   r  r  rº  rÊ  )Úignore_indexr%   )	ÚlossÚlogitsr&  r  r  rï  r  r¥  r  )r²   r}  r¨  r  rÄ   r   r  rÐ  r  rÅ  r&  r¹  r   rY   r;   r  rñ  r   r&  r�   rî  rï  rí  )r�   r³  r]   r²  r´  r  r&  rè  r  r)  r}  r'  rð  r¨  r(  r�   r  Úsequence_outputÚ	lm_logitsr,  Úloss_fctÚoutputs                         r6   rª   z&LongT5ForConditionalGeneration.forward3  s  € ðL "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ð%<‘kÀ$Ç+Á+×BYÑBYˆð Ð"à"Ÿl™lØ#Ø-Ø+Ø"3Ø%9Ø'ð +ó ‰Oñ ¤¨O¼_Ô!MÜ-Ø"1°!Ñ"4Ü47¸Ó4HÈ1Ò4L˜o¨aÒ0ÐRVÜ14°_Ó1EÈÒ1I˜?¨1Ò-ÈtôˆOð (¨Ñ*ˆàÐÐ"3Ð";Ð@UÐ@]à $× 1Ñ 1°&Ó 9Ðð Ÿ,™,Ø'Ø1Ø/Ø+Ø"/Ø#1ØØ/Ø!5Ø#ð 'ó 
ˆð *¨!Ñ,ˆà�;‰;×*Ò*Ø-°·±ÀÑ1EÑFˆOà—L‘L Ó1ˆ	àˆØÐÜ'°TÔ:ˆHà—Y‘Y˜y×/Ñ/Ó0ˆFÙ˜IŸN™N¨2¨y¯~©~¸bÓ/AÓBÀFÇKÁKÐPRÃOÓTˆDáØ�\ O°A°BÐ$7Ñ7¸/ÑIˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEäØØØ+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ô

ð 
	
r8   c                 ó$   — | j                  |«      S rÃ   )rÐ  )r�   r)  s     r6   Ú%prepare_decoder_input_ids_from_labelszDLongT5ForConditionalGeneration.prepare_decoder_input_ids_from_labelsÀ  s   € Ø× Ñ  Ó(Ð(r8   )NNNNNNNNNNNNN)r«   r¬   r­   r  r  r   r™   r  rä  r   r/   r   r!  r"  rG   rt   r	   r_  r   rª   r3  r®   r¯   s   @r6   rÀ  rÀ    s¦  ø„ ð 	Rð*Ð&ð (7Ø'6Ø)ñÐð˜|õ ò*ò:ð
 ð .2Ø37Ø59Ø:>Ø=AØ(,Ø26Ø:>Ø*.Ø!%Ø)-Ø,0Ø#'ñJ
à×#Ñ# dÑ*ðJ
ð ×)Ñ)¨DÑ0ðJ
ð !×+Ñ+¨dÑ2ð	J
ð
 !&× 0Ñ 0°4Ñ 7ðJ
ð ˜u U§\¡\Ñ2Ñ3°dÑ:ðJ
ð  ™ðJ
ð ×(Ñ(¨4Ñ/ðJ
ð  %×0Ñ0°4Ñ7ðJ
ð × Ñ  4Ñ'ðJ
ð ˜$‘;ðJ
ð   $™;ðJ
ð # T™kðJ
ð ˜D‘[ðJ
ð  
ˆu× Ñ Ñ	! OÑ	3ò!J
ó ðJ
ðX)¸E¿L¹L÷ )r8   rÀ  c                   óú   ‡ — e Zd ZddiZdgZdefˆ fd„Zd„ Zd„ Ze		 	 	 	 	 	 dd	e
j                  dz  d
e
j                  dz  de
j                  dz  dedz  dedz  dedz  dee
j                     ez  fd„«       Zˆ xZS )rÁ  rÿ  rþ  r  r²   c                 óú   •— t         ‰| �  |«       t        j                  |j                  |j
                  «      | _        t        j                  |«      }d|_	        t        |«      | _        | j                  «        y )NF)r˜   r™   r   rì   rÛ  r¸   rÃ  r  r  r}  rÙ  r  rà  )r�   r²   r  r    s      €r6   r™   zLongT5EncoderModel.__init__Ë  sZ   ø€ Ü‰Ñ˜Ô Ü—l‘l 6×#4Ñ#4°f·n±nÓEˆŒäŸ™ vÓ.ˆØ#(ˆÔ Ü" >Ó2ˆŒð 	�‰Õr8   c                 ó   — | j                   S rÃ   r  r  s    r6   r  z'LongT5EncoderModel.get_input_embeddingsÖ  r  r8   c                 óH   — || _         | j                  j                  |«       y rÃ   )rÃ  r  rä  râ  s     r6   rä  z'LongT5EncoderModel.set_input_embeddingsÙ  s   € Ø$ˆŒØ�‰×)Ñ)¨.Õ9r8   Nr³  r]   rè  r'  rð  r¨  r    c                 óh   — |�|n| j                   j                  }| j                  ||||||¬«      }|S )aŠ  
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. LongT5 is a model with relative position embeddings so
            you should be able to pad the inputs on both the right and the left.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for detail.

            To know more on how to prepare `input_ids` for pretraining take a look a [LONGT5
            Training](./longt5#training).

        Example:

        ```python
        >>> from transformers import AutoTokenizer, LongT5ForConditionalGeneration

        >>> tokenizer = AutoTokenizer.from_pretrained("google/long-t5-local-base")
        >>> model = LongT5EncoderModel.from_pretrained("google/long-t5-local-base")
        >>> input_ids = tokenizer(
        ...     100 * "Studies have been shown that owning a dog is good for you ", return_tensors="pt"
        ... ).input_ids  # Batch size 1
        >>> outputs = model(input_ids=input_ids)
        >>> last_hidden_states = outputs.last_hidden_state
        ```r  )r²   r¨  r  )	r�   r³  r]   rè  r'  rð  r¨  r(  r  s	            r6   rª   zLongT5EncoderModel.forwardÝ  sH   € ðF &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàŸ,™,ØØ)Ø'Ø/Ø!5Ø#ð 'ó 
ˆð Ðr8   )NNNNNN)r«   r¬   r­   r  r  r   r™   r  rä  r   r/   r   r!  r_  rG   r   rª   r®   r¯   s   @r6   rÁ  rÁ  Ä  sÛ   ø„ ð 	& ðÐð +5¨Ð&ð	˜|õ 	òò:ð ð .2Ø37Ø26Ø)-Ø,0Ø#'ñ-à×#Ñ# dÑ*ð-ð ×)Ñ)¨DÑ0ð-ð ×(Ñ(¨4Ñ/ð	-ð
   $™;ð-ð # T™kð-ð ˜D‘[ð-ð 
ˆu× Ñ Ñ	! OÑ	3ò-ó ô-r8   rÁ  )rÁ  rÀ  r¿  r¯  )r   )KrŠ  r  rø   Útypingr   r/   r   Útorch.nnr   rç  r   r½  Úactivationsr   Úcache_utilsr	   r
   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   Úutilsr   r   r   r   r   Úconfiguration_longt5r   Ú
get_loggerr«   ræ   rt   r~   r7   r@   rM   rU   r\   r;   rd   rG   r‡   rŽ   r”   ÚModuler–   r±   rÈ   rÐ   rÚ   r>  ra  rx  r‚  rŒ  r‘  r—  r¯  rÙ  r¿  rÀ  rÁ  Ú__all__r&   r8   r6   ú<module>rG     sh  ðñ ã Û Ý ã Ý Ý %å &Ý !ß CÑ CÝ )ß JÝ 9÷ó õ .÷õ õ /ð 
ˆ×	Ñ	˜HÓ	%€ñ˜Ÿ™ð °ð ¸3ð È3ð ÐW\×WcÑWcó ð #˜%Ÿ,™,ð #°3ð #¸Sð #ÀUÇ\Á\ó #ñ4˜UŸ\™\ð 4°cð 4Èð 4ÐY\ð 4Ðej×eqÑeqó 4ð2!°#ð !¸%¿,¹,ó !ðB°U·\±\ð BÈcð BÐV[×VbÑVbó Bð8¨e¯l©lð 8Àsð 8ÐTY×T`ÑT`ð 8Ðej×eqÑeqó 8ð .PØ—L‘Lð.PØ58ð.Pà
ˆ5�<‰<˜Ÿ™Ð%Ñ&ó.Pðb4°U·\±\ð 4ÐVYð 4Ð^c×^jÑ^jó 4ð	jØ—<‘<ð	jØ,1¯L©Lð	jØJMð	jà
‡\�\ó	jô+�b—i‘iô +ô4˜"Ÿ)™)ô ô,˜rŸy™yô ô&�B—I‘Iô ô&�b—i‘iô ôDi˜2Ÿ9™9ô iôXl R§Y¡Yô lô`˜rŸy™yô ôB B§I¡Iô ô:¨b¯i©iô ô@ §	¡	ô ô>T
Ð,ô T
ðn ôS!˜Oó S!ó ðS!ôlw
Ð'ô w
ðt ôT
Ð'ó T
ó ðT
ñn ðôô
u)Ð%:¸Oó u)óð
u)ðp ôFÐ.ó Fó ðFòR k�r8   