Ë
    ÜÓ^jùp ã                   óÒ  — d Z ddlZddlmZ ddlZddlmZ ddlmZmZm	Z	 ddl
mZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZ ddlmZmZmZ ddlm Z   ejB                  e"«      Z# ed¬«      e G d„ de«      «       «       Z$ ed¬«      e G d„ de«      «       «       Z% ed¬«      e G d„ de«      «       «       Z& ed¬«      e G d„ de«      «       «       Z' ed¬«      e G d„ de«      «       «       Z( ed ¬«      e G d!„ d"e«      «       «       Z) ed#¬«      e G d$„ d%e«      «       «       Z* ed&¬«      e G d'„ d(e«      «       «       Z+ ed)¬«      e G d*„ d+e«      «       «       Z, ed,¬«      e G d-„ d.e«      «       «       Z- G d/„ d0ej\                  «      Z/ G d1„ d2ej\                  «      Z0 G d3„ d4ej\                  «      Z1 G d5„ d6ej\                  «      Z2 G d7„ d8ej\                  «      Z3 G d9„ d:ej\                  «      Z4 G d;„ d<ej\                  «      Z5 G d=„ d>e«      Z6 G d?„ d@ej\                  «      Z7 G dA„ dBej\                  «      Z8 G dC„ dDej\                  «      Z9 G dE„ dFej\                  «      Z:e G dG„ dHe«      «       Z; edI¬«       G dJ„ dKe;«      «       Z<dL„ Z= G dM„ dNej\                  «      Z> edO¬«       G dP„ dQe;«      «       Z? edR¬«       G dS„ dTe;«      «       Z@ edU¬«       G dV„ dWe;«      «       ZA edX¬«       G dY„ dZe;«      «       ZB ed[¬«       G d\„ d]e;«      «       ZC ed^¬«       G d_„ d`e;«      «       ZDe G da„ dbe;«      «       ZEe G dc„ dde;«      «       ZFg de¢ZGy)fzPyTorch LUKE model.é    N)Ú	dataclass)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FNÚgelu)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPooling)ÚPreTrainedModel)Úapply_chunking_to_forward)ÚModelOutputÚauto_docstringÚloggingé   )Ú
LukeConfigz3
    Base class for outputs of the LUKE model.
    )Úcustom_introc                   ól   — e Zd ZU dZdZej                  dz  ed<   dZe	ej                  df   dz  ed<   y)ÚBaseLukeModelOutputWithPoolingax  
    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`):
        Last layer hidden-state of the first token of the sequence (classification token) further processed by a
        Linear layer and a Tanh activation function.
    entity_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, entity_length, hidden_size)`):
        Sequence of entity hidden-states at the output of the last layer of the model.
    entity_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output of each
        layer plus the initial entity embedding outputs.
    NÚentity_last_hidden_state.Úentity_hidden_states©
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__r   Útuple© ó    úq/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/luke/modeling_luke.pyr   r   %   s?   … ñ
ð :>Ð˜e×/Ñ/°$Ñ6Ó=ØAEÐ˜% × 1Ñ 1°3Ð 6Ñ7¸$Ñ>ÔEr&   r   zV
    Base class for model's outputs, with potential hidden states and attentions.
    c                   ól   — e Zd ZU dZdZej                  dz  ed<   dZe	ej                  df   dz  ed<   y)ÚBaseLukeModelOutputa„  
    entity_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, entity_length, hidden_size)`):
        Sequence of entity hidden-states at the output of the last layer of the model.
    entity_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output of each
        layer plus the initial entity embedding outputs.
    Nr   .r   r   r%   r&   r'   r)   r)   <   s?   … ñð :>Ð˜e×/Ñ/°$Ñ6Ó=ØAEÐ˜% × 1Ñ 1°3Ð 6Ñ7¸$Ñ>ÔEr&   r)   c                   ól  — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
ej                  dz  ed<   dZej                  dz  ed<   dZeej                     dz  ed<   dZeej                  d	f   dz  ed
<   dZeej                  d	f   dz  ed<   y)ÚLukeMaskedLMOutputa:  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        The sum of masked language modeling (MLM) loss and entity prediction loss.
    mlm_loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Masked language modeling (MLM) loss.
    mep_loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Masked entity prediction (MEP) loss.
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
    entity_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction scores of the entity prediction head (scores for each entity vocabulary token before SoftMax).
    entity_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output of each
        layer plus the initial entity embedding outputs.
    NÚlossÚmlm_lossÚmep_lossÚlogitsÚentity_logitsÚhidden_states.r   Ú
attentions)r   r   r   r    r,   r!   r"   r#   r-   r.   r/   r0   r1   r$   r   r2   r%   r&   r'   r+   r+   P   sË   … ñð" &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø)-€Hˆe×Ñ $Ñ&Ó-Ø)-€Hˆe×Ñ $Ñ&Ó-Ø'+€FˆE×Ñ Ñ$Ó+Ø.2€M�5×$Ñ$ tÑ+Ó2Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9ØAEÐ˜% × 1Ñ 1°3Ð 6Ñ7¸$Ñ>ÓEØ7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ô;r&   r+   z2
    Outputs of entity classification models.
    c                   óø   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)	ÚEntityClassificationOutputá¿  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Classification loss.
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Classification scores (before SoftMax).
    entity_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output of each
        layer plus the initial entity embedding outputs.
    Nr,   r/   .r1   r   r2   ©r   r   r   r    r,   r!   r"   r#   r/   r1   r$   r   r2   r%   r&   r'   r4   r4   r   ó‘   … ñ	ð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø'+€FˆE×Ñ Ñ$Ó+Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>ØAEÐ˜% × 1Ñ 1°3Ð 6Ñ7¸$Ñ>ÓEØ7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ô;r&   r4   z7
    Outputs of entity pair classification models.
    c                   óø   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)	ÚEntityPairClassificationOutputr5   Nr,   r/   .r1   r   r2   r6   r%   r&   r'   r9   r9   ‹   r7   r&   r9   z7
    Outputs of entity span classification models.
    c                   óø   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)	ÚEntitySpanClassificationOutputaÎ  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Classification loss.
    logits (`torch.FloatTensor` of shape `(batch_size, entity_length, config.num_labels)`):
        Classification scores (before SoftMax).
    entity_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output of each
        layer plus the initial entity embedding outputs.
    Nr,   r/   .r1   r   r2   r6   r%   r&   r'   r;   r;   ¤   r7   r&   r;   z4
    Outputs of sentence classification models.
    c                   óø   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)	ÚLukeSequenceClassifierOutputa  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Classification (or regression if config.num_labels==1) loss.
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Classification (or regression if config.num_labels==1) scores (before SoftMax).
    entity_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output of each
        layer plus the initial entity embedding outputs.
    Nr,   r/   .r1   r   r2   r6   r%   r&   r'   r=   r=   ½   r7   r&   r=   z@
    Base class for outputs of token classification models.
    c                   óø   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)	ÚLukeTokenClassifierOutputaÐ  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Classification loss.
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`):
        Classification scores (before SoftMax).
    entity_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output of each
        layer plus the initial entity embedding outputs.
    Nr,   r/   .r1   r   r2   r6   r%   r&   r'   r?   r?   Ö   r7   r&   r?   z/
    Outputs of question answering models.
    c                   ó   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
eej                  df   dz  ed<   dZeej                  df   dz  ed<   dZeej                  df   dz  ed	<   y)
Ú LukeQuestionAnsweringModelOutputa‡  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Total span extraction loss is the sum of a Cross-Entropy for the start and end positions.
    entity_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output of each
        layer plus the initial entity embedding outputs.
    Nr,   Ústart_logitsÚ
end_logits.r1   r   r2   )r   r   r   r    r,   r!   r"   r#   rB   rC   r1   r$   r   r2   r%   r&   r'   rA   rA   ï   s¦   … ñð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø-1€L�%×#Ñ# dÑ*Ó1Ø+/€J�×!Ñ! DÑ(Ó/Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>ØAEÐ˜% × 1Ñ 1°3Ð 6Ñ7¸$Ñ>ÓEØ7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ô;r&   rA   z,
    Outputs of multiple choice models.
    c                   óø   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)	ÚLukeMultipleChoiceModelOutputa  
    loss (`torch.FloatTensor` of shape *(1,)*, *optional*, returned when `labels` is provided):
        Classification loss.
    logits (`torch.FloatTensor` of shape `(batch_size, num_choices)`):
        *num_choices* is the second dimension of the input tensors. (see *input_ids* above).

        Classification scores (before SoftMax).
    entity_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
        shape `(batch_size, entity_length, hidden_size)`. Entity hidden-states of the model at the output of each
        layer plus the initial entity embedding outputs.
    Nr,   r/   .r1   r   r2   r6   r%   r&   r'   rE   rE     s‘   … ñð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø'+€FˆE×Ñ Ñ$Ó+Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>ØAEÐ˜% × 1Ñ 1°3Ð 6Ñ7¸$Ñ>ÓEØ7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ô;r&   rE   c                   ó8   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 dd„Zd„ Zˆ xZS )ÚLukeEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    c                 ó”  •— t         ‰| �  «        t        j                  |j                  |j
                  |j                  ¬«      | _        t        j                  |j                  |j
                  «      | _	        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        |j                  | _        t        j                  |j                  |j
                  | j"                  ¬«      | _	        y )N©Úpadding_idx©Úeps)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutrJ   ©ÚselfÚconfigÚ	__class__s     €r'   rN   zLukeEmbeddings.__init__'  sã   ø€ Ü‰ÑÔÜ!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ#%§<¡<°×0NÑ0NÐPV×PbÑPbÓ#cˆÔ Ü%'§\¡\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"äŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒð "×.Ñ.ˆÔÜ#%§<¡<Ø×*Ñ*¨F×,>Ñ,>ÈD×L\ÑL\ô$
ˆÕ r&   c                 ó  — |€C|�0t        || j                  «      j                  |j                  «      }n| j	                  |«      }|�|j                  «       }n|j                  «       d d }|€:t        j                  |t        j                  | j                  j                  ¬«      }|€| j                  |«      }| j                  |«      }| j                  |«      }||z   |z   }| j                  |«      }| j                  |«      }|S )Néÿÿÿÿ©ÚdtypeÚdevice)Ú"create_position_ids_from_input_idsrJ   Útore   Ú&create_position_ids_from_inputs_embedsÚsizer!   ÚzerosÚlongÚposition_idsrS   rU   rW   rX   r\   )	r^   Ú	input_idsÚtoken_type_idsrl   Úinputs_embedsÚinput_shaperU   rW   Ú
embeddingss	            r'   ÚforwardzLukeEmbeddings.forward6  sþ   € ð ÐØÐ$äAÀ)ÈT×M]ÑM]Ó^×aÑaÐbk×brÑbrÓs‘à#×JÑJÈ=ÓY�àÐ Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKàÐ!Ü"Ÿ[™[¨¼E¿J¹JÈt×O`ÑO`×OgÑOgÔhˆNàÐ Ø ×0Ñ0°Ó;ˆMà"×6Ñ6°|ÓDÐØ $× :Ñ :¸>Ó JÐà"Ð%8Ñ8Ð;PÑPˆ
Ø—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr&   c                 ó  — |j                  «       dd }|d   }t        j                  | j                  dz   || j                  z   dz   t        j                  |j
                  ¬«      }|j                  d«      j                  |«      S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nrb   r   rc   r   )ri   r!   ÚarangerJ   rk   re   Ú	unsqueezeÚexpand)r^   ro   rp   Úsequence_lengthrl   s        r'   rh   z5LukeEmbeddings.create_position_ids_from_inputs_embedsW  s€   € ð $×(Ñ(Ó*¨3¨BÐ/ˆØ% a™.ˆä—|‘|Ø×Ñ˜qÑ  /°D×4DÑ4DÑ"DÀqÑ"HÔPU×PZÑPZÐcp×cwÑcwô
ˆð ×%Ñ% aÓ(×/Ñ/°Ó<Ð<r&   )NNNN)r   r   r   r    rN   rr   rh   Ú__classcell__©r`   s   @r'   rG   rG   "  s&   ø„ ñô
ð" ØØØóöB=r&   rG   c                   ó~   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dej                  dej                  dz  fd„Zˆ xZS )	ÚLukeEntityEmbeddingsr_   c                 óŠ  •— t         ‰| �  «        || _        t        j                  |j
                  |j                  d¬«      | _        |j                  |j                  k7  r1t        j                  |j                  |j                  d¬«      | _
        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                   ¬«      | _        t        j"                  |j$                  «      | _        y )Nr   rI   F©ÚbiasrK   )rM   rN   r_   r   rO   Úentity_vocab_sizeÚentity_emb_sizeÚentity_embeddingsrQ   ÚLinearÚentity_embedding_denserT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   s     €r'   rN   zLukeEntityEmbeddings.__init__j  sà   ø€ Ü‰ÑÔØˆŒä!#§¡¨f×.FÑ.FÈ×H^ÑH^ÐlmÔ!nˆÔØ×!Ñ! V×%7Ñ%7Ò7Ü*,¯)©)°F×4JÑ4JÈF×L^ÑL^ÐejÔ*kˆDÔ'ä#%§<¡<°×0NÑ0NÐPV×PbÑPbÓ#cˆÔ Ü%'§\¡\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"äŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r&   NÚ
entity_idsrl   rn   c                 óR  — |€t        j                  |«      }| j                  |«      }| j                  j                  | j                  j
                  k7  r| j                  |«      }| j                  |j                  d¬«      «      }|dk7  j                  |«      j                  d«      }||z  }t        j                  |d¬«      }||j                  d¬«      j                  d¬«      z  }| j                  |«      }||z   |z   }| j                  |«      }| j                  |«      }|S )Nr   )Úminrb   éþÿÿÿ©ÚdimgH¯¼šò×z>)r!   Ú
zeros_liker�   r_   r€   rQ   rƒ   rU   ÚclampÚtype_asru   ÚsumrW   rX   r\   )	r^   r„   rl   rn   r�   rU   Úposition_embedding_maskrW   rq   s	            r'   rr   zLukeEntityEmbeddings.forwardx  s'  € ð Ð!Ü"×-Ñ-¨jÓ9ˆNà ×2Ñ2°:Ó>ÐØ�;‰;×&Ñ&¨$¯+©+×*AÑ*AÒAØ $× ;Ñ ;Ð<MÓ NÐà"×6Ñ6°|×7IÑ7IÈaÐ7IÓ7PÓQÐØ#/°2Ñ#5×">Ñ">Ð?RÓ"S×"]Ñ"]Ð^`Ó"aÐØ1Ð4KÑKÐÜ#Ÿi™iÐ(;ÀÔDÐØ1Ð4K×4OÑ4OÐTVÐ4OÓ4W×4]Ñ4]ÐbfÐ4]Ó4gÑgÐà $× :Ñ :¸>Ó JÐà&Ð)<Ñ<Ð?TÑTˆ
Ø—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
àÐr&   ©N)	r   r   r   r   rN   r!   Ú
LongTensorrr   rx   ry   s   @r'   r{   r{   i  sL   ø„ ð>˜zõ >ð$ 37ñ	à×$Ñ$ðð ×&Ñ&ðð ×(Ñ(¨4Ñ/÷	r&   r{   c                   ó0   ‡ — e Zd Zˆ fd„Zd„ Z	 	 dd„Zˆ xZS )ÚLukeSelfAttentionc                 ó   •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _        |j                  | _	        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        | j                  r�t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j$                  |j&                  «      | _        y )Nr   Úembedding_sizezThe hidden size z4 is not a multiple of the number of attention heads ú.)rM   rN   rQ   Únum_attention_headsÚhasattrÚ
ValueErrorÚintÚattention_head_sizeÚall_head_sizeÚuse_entity_aware_attentionr   r‚   ÚqueryÚkeyÚvalueÚ	w2e_queryÚ	e2w_queryÚ	e2e_queryrZ   Úattention_probs_dropout_probr\   r]   s     €r'   rN   zLukeSelfAttention.__init__•  s�  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ" 6×#5Ñ#5Ð"6ð 7Ø×3Ñ3Ð4°Að7óð ð
 $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔØ*0×*KÑ*KˆÔ'ä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
à×*Ò*ÜŸY™Y v×'9Ñ'9¸4×;MÑ;MÓNˆDŒNÜŸY™Y v×'9Ñ'9¸4×;MÑ;MÓNˆDŒNÜŸY™Y v×'9Ñ'9¸4×;MÑ;MÓNˆDŒNä—z‘z &×"EÑ"EÓFˆ�r&   c                 ó    — |j                  «       d d | j                  | j                  fz   } |j                  |Ž }|j	                  dddd«      S )Nrb   r   é   r   r   )ri   r–   rš   ÚviewÚpermute)r^   ÚxÚnew_x_shapes      r'   Útranspose_for_scoresz&LukeSelfAttention.transpose_for_scores­  sN   € Ø—f‘f“h˜s �m t×'?Ñ'?À×AYÑAYÐ&ZÑZˆØˆA�F‰F�KÐ ˆØ�y‰y˜˜A˜q !Ó$Ð$r&   c                 óÂ  — |j                  d«      }|€|}nt        j                  ||gd¬«      }| j                  | j	                  |«      «      }| j                  | j                  |«      «      }| j                  �r«|��¨| j                  | j                  |«      «      }	| j                  | j                  |«      «      }
| j                  | j                  |«      «      }| j                  | j                  |«      «      }|d d …d d …d |…d d …f   }|d d …d d …d |…d d …f   }|d d …d d …|d …d d …f   }|d d …d d …|d …d d …f   }t        j                  |	|j                  dd«      «      }t        j                  |
|j                  dd«      «      }t        j                  ||j                  dd«      «      }t        j                  ||j                  dd«      «      }t        j                  ||gd¬«      }t        j                  ||gd¬«      }t        j                  ||gd¬«      }nF| j                  | j                  |«      «      }t        j                  ||j                  dd«      «      }|t        j                  | j                  «      z  }|�||z   }t         j"                  j%                  |d¬«      }| j'                  |«      }t        j                  ||«      }|j)                  dddd«      j+                  «       }|j                  «       d d | j,                  fz   } |j.                  |Ž }|d d …d |…d d …f   }|€d }n|d d …|d …d d …f   }|r|||f}|S ||f}|S )Nr   rˆ   rb   r‡   r   r¥   r   )ri   r!   Úcatrª   rž   rŸ   rœ   r�   r    r¡   r¢   ÚmatmulÚ	transposeÚmathÚsqrtrš   r   Ú
functionalÚsoftmaxr\   r§   Ú
contiguousr›   r¦   )r^   Úword_hidden_statesr   Úattention_maskÚoutput_attentionsÚ	word_sizeÚconcat_hidden_statesÚ	key_layerÚvalue_layerÚw2w_query_layerÚw2e_query_layerÚe2w_query_layerÚe2e_query_layerÚw2w_key_layerÚe2w_key_layerÚw2e_key_layerÚe2e_key_layerÚw2w_attention_scoresÚw2e_attention_scoresÚe2w_attention_scoresÚe2e_attention_scoresÚword_attention_scoresÚentity_attention_scoresÚattention_scoresÚquery_layerÚattention_probsÚcontext_layerÚnew_context_layer_shapeÚoutput_word_hidden_statesÚoutput_entity_hidden_statesÚoutputss                                  r'   rr   zLukeSelfAttention.forward²  sf  € ð '×+Ñ+¨AÓ.ˆ	àÐ'Ø#5Ñ ä#(§9¡9Ð.@ÐBVÐ-WÐ]^Ô#_Ð à×-Ñ-¨d¯h©hÐ7KÓ.LÓMˆ	Ø×/Ñ/°·
±
Ð;OÓ0PÓQˆà×*Ó*Ð/CÑ/Oð #×7Ñ7¸¿
¹
ÐCUÓ8VÓWˆOØ"×7Ñ7¸¿¹ÐGYÓ8ZÓ[ˆOØ"×7Ñ7¸¿¹ÐG[Ó8\Ó]ˆOØ"×7Ñ7¸¿¹ÐG[Ó8\Ó]ˆOð &¢aª¨J¨Y¨JºÐ&9Ñ:ˆMØ%¢aª¨J¨Y¨JºÐ&9Ñ:ˆMØ%¢aª¨I©JºÐ&9Ñ:ˆMØ%¢aª¨I©JºÐ&9Ñ:ˆMô $)§<¡<°À×AXÑAXÐY[Ð]_ÓA`Ó#aÐ Ü#(§<¡<°À×AXÑAXÐY[Ð]_ÓA`Ó#aÐ Ü#(§<¡<°À×AXÑAXÐY[Ð]_ÓA`Ó#aÐ Ü#(§<¡<°À×AXÑAXÐY[Ð]_ÓA`Ó#aÐ ô %*§I¡IÐ/CÐEYÐ.ZÐ`aÔ$bÐ!Ü&+§i¡iÐ1EÐG[Ð0\ÐbcÔ&dÐ#Ü$Ÿy™yÐ*?ÐAXÐ)YÐ_`ÔaÑð ×3Ñ3°D·J±JÐ?SÓ4TÓUˆKÜ$Ÿ|™|¨K¸×9LÑ9LÈRÐQSÓ9TÓUÐà+¬d¯i©i¸×8PÑ8PÓ.QÑQÐØÐ%à/°.Ñ@Ðô Ÿ-™-×/Ñ/Ð0@ÀbÐ/ÓIˆð Ÿ,™, Ó7ˆäŸ™ _°kÓBˆà%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ*˜×*Ñ*Ð,CÐDˆà$1²!°Z°i°ZÂÐ2BÑ$CÐ!ØÐ'Ø*.Ñ'à*7º¸9¹:ÂqÐ8HÑ*IÐ'áØ0Ð2MÈÐ_ˆGð ˆð 1Ð2MÐNˆGàˆr&   ©NF)r   r   r   rN   rª   rr   rx   ry   s   @r'   r’   r’   ”  s   ø„ ôGò0%ð Ø÷Kr&   r’   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚLukeSelfOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _
        y ©NrK   )rM   rN   r   r‚   rQ   ÚdenserX   rY   rZ   r[   r\   r]   s     €r'   rN   zLukeSelfOutput.__init__  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r&   r1   Úinput_tensorÚreturnc                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S r�   ©rÖ   r\   rX   ©r^   r1   r×   s      r'   rr   zLukeSelfOutput.forward  ó7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°|Ñ'CÓDˆØÐr&   ©r   r   r   rN   r!   ÚTensorrr   rx   ry   s   @r'   rÓ   rÓ     ó1   ø„ ô>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ r&   rÓ   c                   ó*   ‡ — e Zd Zˆ fd„Z	 	 dd„Zˆ xZS )ÚLukeAttentionc                 ób   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        y r�   )rM   rN   r’   r^   rÓ   Úoutputr]   s     €r'   rN   zLukeAttention.__init__  s&   ø€ Ü‰ÑÔÜ% fÓ-ˆŒ	Ü$ VÓ,ˆ�r&   c                 óD  — |j                  d«      }| j                  ||||«      }|€|d   }|}n3t        j                  |d d d¬«      }t        j                  ||gd¬«      }| j	                  ||«      }	|	d d …d |…d d …f   }
|€d }n|	d d …|d …d d …f   }|
|f|dd  z   }|S )Nr   r   r¥   rˆ   )ri   r^   r!   r¬   rã   )r^   r´   r   rµ   r¶   r·   Úself_outputsÚconcat_self_outputsr¸   Úattention_outputÚword_attention_outputÚentity_attention_outputrÐ   s                r'   rr   zLukeAttention.forward  sà   € ð '×+Ñ+¨AÓ.ˆ	Ø—y‘yØØ ØØó	
ˆð  Ð'Ø".¨q¡/ÐØ#5Ñ ä"'§)¡)¨L¸¸!Ð,<À!Ô"DÐÜ#(§9¡9Ð.@ÐBVÐ-WÐ]^Ô#_Ð àŸ;™;Ð':Ð<PÓQÐà 0²°J°Y°JÂÐ1AÑ BÐØÐ'Ø&*Ñ#à&6²q¸)¹*ÂaÐ7GÑ&HÐ#ð )Ð*AÐBÀ\ÐRSÐRTÐEUÑUˆàˆr&   rÑ   ©r   r   r   rN   rr   rx   ry   s   @r'   rá   rá     s   ø„ ô-ð Ø÷ r&   rá   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚLukeIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y r�   )rM   rN   r   r‚   rQ   Úintermediate_sizerÖ   Ú
isinstanceÚ
hidden_actÚstrr
   Úintermediate_act_fnr]   s     €r'   rN   zLukeIntermediate.__init__:  s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$r&   r1   rØ   c                 óJ   — | j                  |«      }| j                  |«      }|S r�   )rÖ   rò   ©r^   r1   s     r'   rr   zLukeIntermediate.forwardB  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr&   rÝ   ry   s   @r'   rì   rì   9  s#   ø„ ô9ð U§\¡\ð °e·l±l÷ r&   rì   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )Ú
LukeOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y rÕ   )rM   rN   r   r‚   rî   rQ   rÖ   rX   rY   rZ   r[   r\   r]   s     €r'   rN   zLukeOutput.__init__J  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r&   r1   r×   rØ   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S r�   rÚ   rÛ   s      r'   rr   zLukeOutput.forwardP  rÜ   r&   rÝ   ry   s   @r'   rö   rö   I  rß   r&   rö   c                   ó0   ‡ — e Zd Zˆ fd„Z	 	 dd„Zd„ Zˆ xZS )Ú	LukeLayerc                 ó²   •— t         ‰| �  «        |j                  | _        d| _        t	        |«      | _        t        |«      | _        t        |«      | _	        y ©Nr   )
rM   rN   Úchunk_size_feed_forwardÚseq_len_dimrá   Ú	attentionrì   Úintermediaterö   rã   r]   s     €r'   rN   zLukeLayer.__init__X  sI   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ& vÓ.ˆŒÜ,¨VÓ4ˆÔÜ  Ó(ˆ�r&   c                 óH  — |j                  d«      }| j                  ||||¬«      }|€|d   }nt        j                  |d d d¬«      }|dd  }t	        | j
                  | j                  | j                  |«      }	|	d d …d |…d d …f   }
|€d }n|	d d …|d …d d …f   }|
|f|z   }|S )Nr   )r¶   r   r¥   rˆ   )ri   rÿ   r!   r¬   r   Úfeed_forward_chunkrý   rþ   )r^   r´   r   rµ   r¶   r·   Úself_attention_outputsÚconcat_attention_outputrÐ   Úlayer_outputÚword_layer_outputÚentity_layer_outputs               r'   rr   zLukeLayer.forward`  sà   € ð '×+Ñ+¨AÓ.ˆ	à!%§¡ØØ ØØ/ð	 "0ó "
Ðð  Ð'Ø&<¸QÑ&?Ñ#ä&+§i¡iÐ0FÀrÈÐ0JÐPQÔ&RÐ#à(¨¨Ð,ˆä0Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUló
ˆð )ª¨J¨Y¨JºÐ)9Ñ:ÐØÐ'Ø"&Ñà".ªq°)±*ºaÐ/?Ñ"@Ðà$Ð&9Ð:¸WÑDˆàˆr&   c                 óL   — | j                  |«      }| j                  ||«      }|S r�   )r   rã   )r^   rç   Úintermediate_outputr  s       r'   r  zLukeLayer.feed_forward_chunkƒ  s,   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6Ð8HÓIˆØÐr&   rÑ   )r   r   r   rN   rr   r  rx   ry   s   @r'   rú   rú   W  s   ø„ ô)ð Øó!öFr&   rú   c                   ó.   ‡ — e Zd Zˆ fd„Z	 	 	 	 dd„Zˆ xZS )ÚLukeEncoderc                 óÐ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _	        y c c}w rÑ   )
rM   rN   r_   r   Ú
ModuleListÚrangeÚnum_hidden_layersrú   ÚlayerÚgradient_checkpointing)r^   r_   Ú_r`   s      €r'   rN   zLukeEncoder.__init__Š  sN   ø€ Ü‰ÑÔØˆŒÜ—]‘]¼uÀV×E]ÑE]Ó?^Ö#_¸!¤I¨fÕ$5Ò#_Ó`ˆŒ
Ø&+ˆÕ#ùò $`s   ½A#c                 ó0  — |rdnd }|rdnd }|rdnd }	t        | j                  «      D ]6  \  }
}|r||fz   }||fz   } |||||«      }|d   }|�|d   }|sŒ.|	|d   fz   }	Œ8 |r||fz   }||fz   }|st        d„ |||	||fD «       «      S t        |||	||¬«      S )Nr%   r   r   r¥   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wr�   r%   ©Ú.0Úvs     r'   ú	<genexpr>z&LukeEncoder.forward.<locals>.<genexpr>¶  ó   è ø€ ò 
àð �=ô ñ
ùó   ‚)Úlast_hidden_stater1   r2   r   r   )Ú	enumerater  r$   r)   )r^   r´   r   rµ   r¶   Úoutput_hidden_statesÚreturn_dictÚall_word_hidden_statesÚall_entity_hidden_statesÚall_self_attentionsÚiÚlayer_moduleÚlayer_outputss                r'   rr   zLukeEncoder.forward�  s  € ñ (<¡ÀÐÙ)=¡2À4Ð Ù$5™b¸4Ðä(¨¯©Ó4ò 	P‰OˆAˆ|Ù#Ø)?ÐCUÐBWÑ)WÐ&Ø+CÐG[ÐF]Ñ+]Ð(á(Ø"Ø$ØØ!ó	ˆMð "/¨qÑ!1Ðà#Ð/Ø'4°QÑ'7Ð$â Ø&9¸]È1Ñ=MÐ<OÑ&OÑ#ð%	Pñ(  Ø%;Ð?QÐ>SÑ%SÐ"Ø'?ÐCWÐBYÑ'YÐ$áÜñ 
ð 'Ø*Ø'Ø(Ø,ðô
ó 
ð 
ô #Ø0Ø0Ø*Ø%9Ø!9ô
ð 	
r&   )NFFTrê   ry   s   @r'   r  r  ‰  s   ø„ ô,ð ØØ"Ø÷7
r&   r  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )Ú
LukePoolerc                 ó²   •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  «       | _        y r�   )rM   rN   r   r‚   rQ   rÖ   ÚTanhÚ
activationr]   s     €r'   rN   zLukePooler.__init__Ì  s9   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'™'›)ˆ�r&   r1   rØ   c                 ó\   — |d d …df   }| j                  |«      }| j                  |«      }|S )Nr   )rÖ   r)  )r^   r1   Úfirst_token_tensorÚpooled_outputs       r'   rr   zLukePooler.forwardÑ  s6   € ð +ª1¨a¨4Ñ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐr&   rÝ   ry   s   @r'   r&  r&  Ë  s#   ø„ ô$ð
 U§\¡\ð °e·l±l÷ r&   r&  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEntityPredictionHeadTransformc                 óh  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        n|j                  | _        t        j                  |j
                  |j                  ¬«      | _        y rÕ   )rM   rN   r   r‚   rQ   r€   rÖ   rï   rð   rñ   r
   Útransform_act_fnrX   rY   r]   s     €r'   rN   z&EntityPredictionHeadTransform.__init__Û  s{   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3IÑ3IÓJˆŒ
Ü�f×'Ñ'¬Ô-Ü$*¨6×+<Ñ+<Ñ$=ˆDÕ!à$*×$5Ñ$5ˆDÔ!ÜŸ™ f×&<Ñ&<À&×BWÑBWÔXˆ�r&   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r�   )rÖ   r0  rX   rô   s     r'   rr   z%EntityPredictionHeadTransform.forwardä  s4   € ØŸ
™
 =Ó1ˆØ×-Ñ-¨mÓ<ˆØŸ™ }Ó5ˆØÐr&   rê   ry   s   @r'   r.  r.  Ú  s   ø„ ôYör&   r.  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEntityPredictionHeadc                 ó   •— t         ‰| �  «        || _        t        |«      | _        t        j                  |j                  |j                  d¬«      | _	        t        j                  t        j                  |j                  «      «      | _        y )NFr}   )rM   rN   r_   r.  Ú	transformr   r‚   r€   r   ÚdecoderÚ	Parameterr!   rj   r~   r]   s     €r'   rN   zEntityPredictionHead.__init__ì  sa   ø€ Ü‰ÑÔØˆŒÜ6°vÓ>ˆŒÜ—y‘y ×!7Ñ!7¸×9QÑ9QÐX]Ô^ˆŒÜ—L‘L¤§¡¨V×-EÑ-EÓ!FÓGˆ�	r&   c                 ód   — | j                  |«      }| j                  |«      | j                  z   }|S r�   )r5  r6  r~   rô   s     r'   rr   zEntityPredictionHead.forwardó  s-   € ØŸ™ }Ó5ˆØŸ™ ]Ó3°d·i±iÑ?ˆàÐr&   rê   ry   s   @r'   r3  r3  ë  s   ø„ ôHör&   r3  c                   ó|   ‡ — e Zd ZU eed<   dZdZddgZ ej                  «       de
j                  fˆ fd„«       Zˆ xZS )ÚLukePreTrainedModelr_   ÚlukeTrá   r{   Úmodulec                 óÆ  •— t         ‰| �  |«       t        |t        j                  «      r·|j
                  dk(  r t        j                  |j                  «       n6t        j                  |j                  d| j                  j                  ¬«       |j                  �Et        |j                  dd«      s-t        j                  |j                  |j                     «       yyyy)zInitialize the weightsr   g        )ÚmeanÚstdNÚ_is_hf_initializedF)rM   Ú_init_weightsrï   r   rO   Úembedding_dimÚinitÚzeros_ÚweightÚnormal_r_   Úinitializer_rangerJ   Úgetattr)r^   r<  r`   s     €r'   rA  z!LukePreTrainedModel._init_weights  s¢   ø€ ô 	‰Ñ˜fÔ%Ü�fœbŸl™lÔ+Ø×#Ñ# qÒ(Ü—‘˜FŸM™MÕ*ä—‘˜VŸ]™]°¸$¿+¹+×:WÑ:WÕXà×!Ñ!Ð-´g¸f¿m¹mÐMaÐchÔ6iÜ—‘˜FŸM™M¨&×*<Ñ*<Ñ=Õ>ð 7jÐ-ð ,r&   )r   r   r   r   r#   Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesr!   Úno_gradr   ÚModulerA  rx   ry   s   @r'   r:  r:  ú  sF   ø… àÓØÐØ&*Ð#Ø(Ð*@ÐAÐà€U‡]�]ƒ_ð
? B§I¡Iô 
?ó ô
?r&   r:  zt
    The bare LUKE model transformer outputting raw hidden-states for both word tokens and entities without any
    c                   ó¤  ‡ — e Zd Zddedefˆ fd„Zd„ Zd„ Zd„ Zd„ Z	e
	 	 	 	 	 	 	 	 	 	 	 	 dd	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  deez  fd„«       Zˆ xZS )Ú	LukeModelr_   Úadd_pooling_layerc                 óÚ   •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        t        |«      | _        |rt        |«      nd| _
        | j                  «        y)zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        N)rM   rN   r_   rG   rq   r{   r�   r  Úencoderr&  ÚpoolerÚ	post_init)r^   r_   rP  r`   s      €r'   rN   zLukeModel.__init__  sZ   ø€ ô
 	‰Ñ˜Ô ØˆŒä(¨Ó0ˆŒÜ!5°fÓ!=ˆÔÜ" 6Ó*ˆŒá,=”j Ô(À4ˆŒð 	�‰Õr&   c                 ó.   — | j                   j                  S r�   ©rq   rS   ©r^   s    r'   Úget_input_embeddingszLukeModel.get_input_embeddings&  s   € Ø�‰×.Ñ.Ð.r&   c                 ó&   — || j                   _        y r�   rV  ©r^   rŸ   s     r'   Úset_input_embeddingszLukeModel.set_input_embeddings)  s   € Ø*/ˆ�‰Õ'r&   c                 ó.   — | j                   j                   S r�   ©r�   rW  s    r'   Úget_entity_embeddingszLukeModel.get_entity_embeddings,  s   € Ø×%Ñ%×7Ñ7Ð7r&   c                 ó&   — || j                   _         y r�   r]  rZ  s     r'   Úset_entity_embeddingszLukeModel.set_entity_embeddings/  s   € Ø38ˆ×ÑÕ0r&   Nrm   rµ   rn   rl   r„   Úentity_attention_maskÚentity_token_type_idsÚentity_position_idsro   r¶   r  r  rØ   c                 ó´  — |
�|
n| j                   j                  }
|�|n| j                   j                  }|�|n| j                   j                  }|�|	�t	        d«      ‚|�#| j                  ||«       |j                  «       }n!|	�|	j                  «       dd }nt	        d«      ‚|\  }}|�|j                  n|	j                  }|€t        j                  ||f|¬«      }|€&t        j                  |t        j                  |¬«      }|�V|j                  d«      }|€t        j                  ||f|¬«      }|€(t        j                  ||ft        j                  |¬«      }| j                  ||||	¬«      }|�t        j                  ||gd¬	«      }t        | j                   |d
   j                  |j                   «      |¬«      }|€d}n| j#                  |||«      }| j%                  ||||
||¬«      }|d   }| j&                  �| j'                  |«      nd}|s
||f|dd z   S t)        |||j*                  |j,                  |j.                  |j0                  ¬«      S )uz  
        entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`):
            Indices of entity tokens in the entity vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.
        entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*):
            Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`:

            - 1 for entity tokens that are **not masked**,
            - 0 for entity tokens that are **masked**.
        entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*):
            Segment token indices to indicate first and second portions of the entity token inputs. Indices are
            selected in `[0, 1]`:

            - 0 corresponds to a *portion A* entity token,
            - 1 corresponds to a *portion B* entity token.
        entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*):
            Indices of positions of each input entity in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, LukeModel

        >>> tokenizer = AutoTokenizer.from_pretrained("studio-ousia/luke-base")
        >>> model = LukeModel.from_pretrained("studio-ousia/luke-base")
        # Compute the contextualized entity representation corresponding to the entity mention "BeyoncÃ©"

        >>> text = "BeyoncÃ© lives in Los Angeles."
        >>> entity_spans = [(0, 7)]  # character-based entity span corresponding to "BeyoncÃ©"

        >>> encoding = tokenizer(text, entity_spans=entity_spans, add_prefix_space=True, return_tensors="pt")
        >>> outputs = model(**encoding)
        >>> word_last_hidden_state = outputs.last_hidden_state
        >>> entity_last_hidden_state = outputs.entity_last_hidden_state
        # Input Wikipedia entities to obtain enriched contextualized representations of word tokens

        >>> text = "BeyoncÃ© lives in Los Angeles."
        >>> entities = [
        ...     "BeyoncÃ©",
        ...     "Los Angeles",
        ... ]  # Wikipedia entity titles corresponding to the entity mentions "BeyoncÃ©" and "Los Angeles"
        >>> entity_spans = [
        ...     (0, 7),
        ...     (17, 28),
        ... ]  # character-based entity spans corresponding to "BeyoncÃ©" and "Los Angeles"

        >>> encoding = tokenizer(
        ...     text, entities=entities, entity_spans=entity_spans, add_prefix_space=True, return_tensors="pt"
        ... )
        >>> outputs = model(**encoding)
        >>> word_last_hidden_state = outputs.last_hidden_state
        >>> entity_last_hidden_state = outputs.entity_last_hidden_state
        ```NzDYou cannot specify both input_ids and inputs_embeds at the same timerb   z5You have to specify either input_ids or inputs_embeds)re   rc   r   )rm   rl   rn   ro   rˆ   ).N)r_   ro   rµ   )rµ   r¶   r  r  r   )r  Úpooler_outputr1   r2   r   r   )r_   r¶   r  r  r˜   Ú%warn_if_padding_and_no_attention_maskri   re   r!   Úonesrj   rk   rq   r¬   r   rg   rd   r�   rR  rS  r   r1   r2   r   r   )r^   rm   rµ   rn   rl   r„   ra  rb  rc  ro   r¶   r  r  Úkwargsrp   Ú
batch_sizeÚ
seq_lengthre   Úentity_seq_lengthÚword_embedding_outputÚentity_embedding_outputÚencoder_outputsÚsequence_outputr,  s                           r'   rr   zLukeModel.forward2  sš  € ðR 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐ  ]Ð%>ÜÐcÓdÐdØÐ"Ø×6Ñ6°yÀ.ÔQØ#Ÿ.™.Ó*‰KØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà!,Ñˆ
�JØ%.Ð%:�×!Ò!À×@TÑ@TˆàÐ!Ü"ŸZ™Z¨°ZÐ(@ÈÔPˆNØÐ!Ü"Ÿ[™[¨¼E¿J¹JÈvÔVˆNØÐ!Ø *§¡°Ó 2ÐØ$Ð,Ü(-¯
©
°JÐ@QÐ3RÐ[aÔ(bÐ%Ø$Ð,Ü(-¯©°ZÐARÐ4SÔ[`×[eÑ[eÐntÔ(uÐ%ð !%§¡ØØ%Ø)Ø'ð	 !0ó !
Ðð !Ð,Ü"ŸY™Y¨Ð8MÐ'NÐTVÔWˆNä2Ø—;‘;à(¨Ñ3×6Ñ6Ð7L×7RÑ7RÓSØ)ô	
ˆð ÐØ&*Ñ#à&*×&<Ñ&<¸ZÐI\Ð^sÓ&tÐ#ð Ÿ,™,Ø!Ø#Ø)Ø/Ø!5Ø#ð 'ó 
ˆð *¨!Ñ,ˆð 9=¿¹Ð8O˜Ÿ™ OÔ4ÐUYˆáØ# ]Ð3°oÀaÀbÐ6IÑIÐIä-Ø-Ø'Ø)×7Ñ7Ø&×1Ñ1Ø%4×%MÑ%MØ!0×!EÑ!Eô
ð 	
r&   )T)NNNNNNNNNNNN)r   r   r   r   ÚboolrN   rX  r[  r^  r`  r   r!   r�   r"   r$   r   rr   rx   ry   s   @r'   rO  rO    sh  ø„ ñ˜zð ¸dõ ò"/ò0ò8ò9ð ð .2Ø37Ø26Ø04Ø.2Ø:>Ø9=Ø7;Ø26Ø)-Ø,0Ø#'ñX
à×#Ñ# dÑ*ðX
ð ×)Ñ)¨DÑ0ðX
ð ×(Ñ(¨4Ñ/ð	X
ð
 ×&Ñ&¨Ñ-ðX
ð ×$Ñ$ tÑ+ðX
ð  %×0Ñ0°4Ñ7ðX
ð  %×/Ñ/°$Ñ6ðX
ð #×-Ñ-°Ñ4ðX
ð ×(Ñ(¨4Ñ/ðX
ð   $™;ðX
ð # T™kðX
ð ˜D‘[ðX
ð 
Ð/Ñ	/òX
ó ôX
r&   rO  c                 ó¸   — | j                  |«      j                  «       }t        j                  |d¬«      j	                  |«      |z  }|j                  «       |z   S )a  
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's `utils.make_positions`.

    Args:
        x: torch.Tensor x:

    Returns: torch.Tensor
    r   rˆ   )Úner™   r!   ÚcumsumrŒ   rk   )rm   rJ   ÚmaskÚincremental_indicess       r'   rf   rf   Î  sP   € ð �<‰<˜Ó$×(Ñ(Ó*€DÜ Ÿ<™<¨°!Ô4×<Ñ<¸TÓBÀdÑJÐØ×#Ñ#Ó%¨Ñ3Ð3r&   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )Ú
LukeLMHeadz*Roberta Head for masked language modeling.c                 ó¬  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  «      | _
        t        j                  t        j                  |j                  «      «      | _        y rÕ   )rM   rN   r   r‚   rQ   rÖ   rX   rY   Ú
layer_normrP   r6  r7  r!   rj   r~   r]   s     €r'   rN   zLukeLMHead.__init__â  s‡   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒä—y‘y ×!3Ñ!3°V×5FÑ5FÓGˆŒÜ—L‘L¤§¡¨V×->Ñ->Ó!?Ó@ˆ�	r&   c                 ó‚   — | j                  |«      }t        |«      }| j                  |«      }| j                  |«      }|S r�   )rÖ   r   ry  r6  )r^   Úfeaturesrh  r¨   s       r'   rr   zLukeLMHead.forwardê  s;   € Ø�J‰J�xÓ ˆÜ�‹GˆØ�O‰O˜AÓˆð �L‰L˜‹Oˆàˆr&   )r   r   r   r    rN   rr   rx   ry   s   @r'   rw  rw  ß  s   ø„ Ù4ôAör&   rw  z—
    The LUKE model with a language modeling head and entity prediction head on top for masked language modeling and
    masked entity prediction.
    c            !       óÖ  ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  deez  fd„«       Zˆ xZS )ÚLukeForMaskedLMz/luke.entity_embeddings.entity_embeddings.weightzlm_head.decoder.bias)z!entity_predictions.decoder.weightzlm_head.biasc                 óÖ   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        t        |«      | _        t        j                  «       | _
        | j                  «        y r�   )rM   rN   rO  r;  rw  Úlm_headr3  Úentity_predictionsr   r   Úloss_fnrT  r]   s     €r'   rN   zLukeForMaskedLM.__init__  sQ   ø€ Ü‰Ñ˜Ô ä˜fÓ%ˆŒ	ä! &Ó)ˆŒÜ"6°vÓ">ˆÔä×*Ñ*Ó,ˆŒð 	�‰Õr&   c                 ó.   — | j                   j                  S r�   ©r  r6  rW  s    r'   Úget_output_embeddingsz%LukeForMaskedLM.get_output_embeddings  s   € Ø�|‰|×#Ñ#Ð#r&   c                 ó&   — || j                   _        y r�   rƒ  )r^   Únew_embeddingss     r'   Úset_output_embeddingsz%LukeForMaskedLM.set_output_embeddings  s   € Ø-ˆ�‰Õr&   Nrm   rµ   rn   rl   r„   ra  rb  rc  ÚlabelsÚentity_labelsro   r¶   r  r  rØ   c                 óH  — |�|n| j                   j                  }| j                  |||||||||||d¬«      }d}d}| j                  |j                  «      }|	�d|	j                  |j                  «      }	| j                  |j                  d| j                   j                  «      |	j                  d«      «      }|€|}d}d}|j                  �l| j                  |j                  «      }|
�O| j                  |j                  d| j                   j                  «      |
j                  d«      «      }|€|}n||z   }|s8t        d„ ||||||j                  |j                  |j                   fD «       «      S t#        ||||||j                  |j                  |j                   ¬«      S )aC  
        entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`):
            Indices of entity tokens in the entity vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.
        entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*):
            Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`:

            - 1 for entity tokens that are **not masked**,
            - 0 for entity tokens that are **masked**.
        entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*):
            Segment token indices to indicate first and second portions of the entity token inputs. Indices are
            selected in `[0, 1]`:

            - 0 corresponds to a *portion A* entity token,
            - 1 corresponds to a *portion B* entity token.
        entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*):
            Indices of positions of each input entity in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        entity_labels (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        NT©rm   rµ   rn   rl   r„   ra  rb  rc  ro   r¶   r  r  rb   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wr�   r%   r  s     r'   r  z*LukeForMaskedLM.forward.<locals>.<genexpr>m  s   è ø€ ò àð �=ô ñùr  )r,   r-   r.   r/   r0   r1   r   r2   )r_   r  r;  r  r  rg   re   r�  r¦   rP   r   r€  r   r$   r1   r   r2   r+   )r^   rm   rµ   rn   rl   r„   ra  rb  rc  rˆ  r‰  ro   r¶   r  r  rh  rÐ   r,   r-   r/   r.   r0   s                         r'   rr   zLukeForMaskedLM.forward  s¿  € ðb &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—)‘)ØØ)Ø)Ø%Ø!Ø"7Ø"7Ø 3Ø'Ø/Ø!5Øð ó 
ˆð ˆàˆØ—‘˜g×7Ñ7Ó8ˆØÐà—Y‘Y˜vŸ}™}Ó-ˆFØ—|‘| F§K¡K°°D·K±K×4JÑ4JÓ$KÈVÏ[É[ÐY[Ë_Ó]ˆHØˆ|Ø�àˆØˆØ×+Ñ+Ð7Ø ×3Ñ3°G×4TÑ4TÓUˆMØÐ(ØŸ<™<¨×(:Ñ(:¸2¸t¿{¹{×?\Ñ?\Ó(]Ð_l×_qÑ_qÐrtÓ_uÓv�Ø�<Ø#‘Dà (™?�DáÜñ ð ØØØØ!Ø×)Ñ)Ø×0Ñ0Ø×&Ñ&ð	ôó ð ô "ØØØØØ'Ø!×/Ñ/Ø!(×!=Ñ!=Ø×)Ñ)ô	
ð 		
r&   ©NNNNNNNNNNNNNN)r   r   r   Ú_tied_weights_keysrN   r„  r‡  r   r!   r�   r"   rp  r$   r+   rr   rx   ry   s   @r'   r}  r}  õ  s“  ø„ ð ._Ø.ñÐô
ò$ò.ð ð .2Ø37Ø26Ø04Ø.2Ø9=Ø9=Ø7;Ø*.Ø15Ø26Ø)-Ø,0Ø#'ñp
à×#Ñ# dÑ*ðp
ð ×)Ñ)¨DÑ0ðp
ð ×(Ñ(¨4Ñ/ð	p
ð
 ×&Ñ&¨Ñ-ðp
ð ×$Ñ$ tÑ+ðp
ð  %×/Ñ/°$Ñ6ðp
ð  %×/Ñ/°$Ñ6ðp
ð #×-Ñ-°Ñ4ðp
ð × Ñ  4Ñ'ðp
ð ×'Ñ'¨$Ñ.ðp
ð ×(Ñ(¨4Ñ/ðp
ð   $™;ðp
ð # T™kðp
ð ˜D‘[ðp
ð" 
Ð#Ñ	#ò#p
ó ôp
r&   r}  zº
    The LUKE model with a classification head on top (a linear layer on top of the hidden state of the first entity
    token) for entity classification tasks, such as Open Entity.
    c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  de	e
z  fd„«       Zˆ xZS )ÚLukeForEntityClassificationc                 ó,  •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r�   ©rM   rN   rO  r;  Ú
num_labelsr   rZ   r[   r\   r‚   rQ   Ú
classifierrT  r]   s     €r'   rN   z$LukeForEntityClassification.__init__�  si   ø€ Ü‰Ñ˜Ô ä˜fÓ%ˆŒ	à ×+Ñ+ˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr&   Nrm   rµ   rn   rl   r„   ra  rb  rc  ro   rˆ  r¶   r  r  rØ   c                 óâ  — |�|n| j                   j                  }| j                  |||||||||	||d¬«      }|j                  dd…ddd…f   }| j	                  |«      }| j                  |«      }d}|
�˜|
j                  |j                  «      }
|
j                  dk(  r!t        j                  j                  ||
«      }nMt        j                  j                  |j                  d«      |
j                  d«      j                  |«      «      }|s5t        d„ |||j                   |j"                  |j$                  fD «       «      S t'        |||j                   |j"                  |j$                  ¬«      S )	u¸
  
        entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`):
            Indices of entity tokens in the entity vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.
        entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*):
            Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`:

            - 1 for entity tokens that are **not masked**,
            - 0 for entity tokens that are **masked**.
        entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*):
            Segment token indices to indicate first and second portions of the entity token inputs. Indices are
            selected in `[0, 1]`:

            - 0 corresponds to a *portion A* entity token,
            - 1 corresponds to a *portion B* entity token.
        entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*):
            Indices of positions of each input entity in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.
        labels (`torch.LongTensor` of shape `(batch_size,)` or `(batch_size, num_labels)`, *optional*):
            Labels for computing the classification loss. If the shape is `(batch_size,)`, the cross entropy loss is
            used for the single-label classification. In this case, labels should contain the indices that should be in
            `[0, ..., config.num_labels - 1]`. If the shape is `(batch_size, num_labels)`, the binary cross entropy
            loss is used for the multi-label classification. In this case, labels should only contain `[0, 1]`, where 0
            and 1 indicate false and true, respectively.

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, LukeForEntityClassification

        >>> tokenizer = AutoTokenizer.from_pretrained("studio-ousia/luke-large-finetuned-open-entity")
        >>> model = LukeForEntityClassification.from_pretrained("studio-ousia/luke-large-finetuned-open-entity")

        >>> text = "BeyoncÃ© lives in Los Angeles."
        >>> entity_spans = [(0, 7)]  # character-based entity span corresponding to "BeyoncÃ©"
        >>> inputs = tokenizer(text, entity_spans=entity_spans, return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> logits = outputs.logits
        >>> predicted_class_idx = logits.argmax(-1).item()
        >>> print("Predicted class:", model.config.id2label[predicted_class_idx])
        Predicted class: person
        ```NTr‹  r   r   rb   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wr�   r%   r  s     r'   r  z6LukeForEntityClassification.forward.<locals>.<genexpr>ú  ó   è ø€ ò àØ�=ô ñùr  ©r,   r/   r1   r   r2   )r_   r  r;  r   r\   r”  rg   re   Úndimr   r±   Úcross_entropyÚ binary_cross_entropy_with_logitsr¦   rŒ   r$   r1   r   r2   r4   ©r^   rm   rµ   rn   rl   r„   ra  rb  rc  ro   rˆ  r¶   r  r  rh  rÐ   Úfeature_vectorr/   r,   s                      r'   rr   z#LukeForEntityClassification.forward›  sj  € ð| &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—)‘)ØØ)Ø)Ø%Ø!Ø"7Ø"7Ø 3Ø'Ø/Ø!5Øð ó 
ˆð !×9Ñ9º!¸QÂ¸'ÑBˆØŸ™ nÓ5ˆØ—‘ Ó0ˆàˆØÐð —Y‘Y˜vŸ}™}Ó-ˆFØ�{‰{˜aÒÜ—}‘}×2Ñ2°6¸6ÓB‘ä—}‘}×EÑEÀfÇkÁkÐRTÃoÐW]×WbÑWbÐceÓWf×WnÑWnÐouÓWvÓw�áÜñ à ¨×(=Ñ(=¸w×?[Ñ?[Ð]d×]oÑ]oÐpôó ð ô *ØØØ!×/Ñ/Ø!(×!=Ñ!=Ø×)Ñ)ô
ð 	
r&   ©NNNNNNNNNNNNN)r   r   r   rN   r   r!   r�   r"   rp  r$   r4   rr   rx   ry   s   @r'   r�  r�  ˆ  s_  ø„ ô
ð ð .2Ø37Ø26Ø04Ø.2Ø:>Ø9=Ø7;Ø26Ø+/Ø)-Ø,0Ø#'ñj
à×#Ñ# dÑ*ðj
ð ×)Ñ)¨DÑ0ðj
ð ×(Ñ(¨4Ñ/ð	j
ð
 ×&Ñ&¨Ñ-ðj
ð ×$Ñ$ tÑ+ðj
ð  %×0Ñ0°4Ñ7ðj
ð  %×/Ñ/°$Ñ6ðj
ð #×-Ñ-°Ñ4ðj
ð ×(Ñ(¨4Ñ/ðj
ð ×!Ñ! DÑ(ðj
ð   $™;ðj
ð # T™kðj
ð ˜D‘[ðj
ð  
Ð+Ñ	+ò!j
ó ôj
r&   r�  zº
    The LUKE model with a classification head on top (a linear layer on top of the hidden states of the two entity
    tokens) for entity pair classification tasks, such as TACRED.
    c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  de	e
z  fd„«       Zˆ xZS )ÚLukeForEntityPairClassificationc                 ó4  •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  «      | _        t        j                  |j                  dz  |j                  d«      | _        | j                  «        y )Nr¥   Fr’  r]   s     €r'   rN   z(LukeForEntityPairClassification.__init__  sp   ø€ Ü‰Ñ˜Ô ä˜fÓ%ˆŒ	à ×+Ñ+ˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒÜŸ)™) F×$6Ñ$6¸Ñ$:¸F×<MÑ<MÈuÓUˆŒð 	�‰Õr&   Nrm   rµ   rn   rl   r„   ra  rb  rc  ro   rˆ  r¶   r  r  rØ   c                 ó8  — |�|n| j                   j                  }| j                  |||||||||	||d¬«      }t        j                  |j
                  dd…ddd…f   |j
                  dd…ddd…f   gd¬«      }| j                  |«      }| j                  |«      }d}|
�˜|
j                  |j                  «      }
|
j                  dk(  r!t        j                  j                  ||
«      }nMt        j                  j                  |j                  d«      |
j                  d«      j!                  |«      «      }|s5t#        d„ |||j$                  |j&                  |j(                  fD «       «      S t+        |||j$                  |j&                  |j(                  ¬	«      S )
u  
        entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`):
            Indices of entity tokens in the entity vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.
        entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*):
            Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`:

            - 1 for entity tokens that are **not masked**,
            - 0 for entity tokens that are **masked**.
        entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*):
            Segment token indices to indicate first and second portions of the entity token inputs. Indices are
            selected in `[0, 1]`:

            - 0 corresponds to a *portion A* entity token,
            - 1 corresponds to a *portion B* entity token.
        entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*):
            Indices of positions of each input entity in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.
        labels (`torch.LongTensor` of shape `(batch_size,)` or `(batch_size, num_labels)`, *optional*):
            Labels for computing the classification loss. If the shape is `(batch_size,)`, the cross entropy loss is
            used for the single-label classification. In this case, labels should contain the indices that should be in
            `[0, ..., config.num_labels - 1]`. If the shape is `(batch_size, num_labels)`, the binary cross entropy
            loss is used for the multi-label classification. In this case, labels should only contain `[0, 1]`, where 0
            and 1 indicate false and true, respectively.

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, LukeForEntityPairClassification

        >>> tokenizer = AutoTokenizer.from_pretrained("studio-ousia/luke-large-finetuned-tacred")
        >>> model = LukeForEntityPairClassification.from_pretrained("studio-ousia/luke-large-finetuned-tacred")

        >>> text = "BeyoncÃ© lives in Los Angeles."
        >>> entity_spans = [
        ...     (0, 7),
        ...     (17, 28),
        ... ]  # character-based entity spans corresponding to "BeyoncÃ©" and "Los Angeles"
        >>> inputs = tokenizer(text, entity_spans=entity_spans, return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> logits = outputs.logits
        >>> predicted_class_idx = logits.argmax(-1).item()
        >>> print("Predicted class:", model.config.id2label[predicted_class_idx])
        Predicted class: per:cities_of_residence
        ```NTr‹  r   r   rˆ   rb   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wr�   r%   r  s     r'   r  z:LukeForEntityPairClassification.forward.<locals>.<genexpr>€  r—  r  r˜  )r_   r  r;  r!   r¬   r   r\   r”  rg   re   r™  r   r±   rš  r›  r¦   rŒ   r$   r1   r   r2   r9   rœ  s                      r'   rr   z'LukeForEntityPairClassification.forward  s•  € ðB &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—)‘)ØØ)Ø)Ø%Ø!Ø"7Ø"7Ø 3Ø'Ø/Ø!5Øð ó 
ˆô Ÿ™Ø×-Ñ-ªa°²A¨gÑ6¸×8XÑ8XÒYZÐ\]Ò_`ÐY`Ñ8aÐbÐhiô
ˆð Ÿ™ nÓ5ˆØ—‘ Ó0ˆàˆØÐð —Y‘Y˜vŸ}™}Ó-ˆFØ�{‰{˜aÒÜ—}‘}×2Ñ2°6¸6ÓB‘ä—}‘}×EÑEÀfÇkÁkÐRTÃoÐW]×WbÑWbÐceÓWf×WnÑWnÐouÓWvÓw�áÜñ à ¨×(=Ñ(=¸w×?[Ñ?[Ð]d×]oÑ]oÐpôó ð ô .ØØØ!×/Ñ/Ø!(×!=Ñ!=Ø×)Ñ)ô
ð 	
r&   rž  )r   r   r   rN   r   r!   r�   r"   rp  r$   r9   rr   rx   ry   s   @r'   r   r   	  s_  ø„ ô
ð ð .2Ø37Ø26Ø04Ø.2Ø:>Ø9=Ø7;Ø26Ø*.Ø)-Ø,0Ø#'ño
à×#Ñ# dÑ*ðo
ð ×)Ñ)¨DÑ0ðo
ð ×(Ñ(¨4Ñ/ð	o
ð
 ×&Ñ&¨Ñ-ðo
ð ×$Ñ$ tÑ+ðo
ð  %×0Ñ0°4Ñ7ðo
ð  %×/Ñ/°$Ñ6ðo
ð #×-Ñ-°Ñ4ðo
ð ×(Ñ(¨4Ñ/ðo
ð × Ñ  4Ñ'ðo
ð   $™;ðo
ð # T™kðo
ð ˜D‘[ðo
ð  
Ð/Ñ	/ò!o
ó ôo
r&   r   z£
    The LUKE model with a span classification head on top (a linear layer on top of the hidden states output) for tasks
    such as named entity recognition.
    c            #       óà  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  de	e
z  f d„«       Zˆ xZS )ÚLukeForEntitySpanClassificationc                 ó2  •— t         ‰| �  |«       t        |«      | _        |j                  | _        t        j                  |j                  «      | _        t        j                  |j                  dz  |j                  «      | _        | j                  «        y )Nr   r’  r]   s     €r'   rN   z(LukeForEntitySpanClassification.__init__–  sn   ø€ Ü‰Ñ˜Ô ä˜fÓ%ˆŒ	à ×+Ñ+ˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒÜŸ)™) F×$6Ñ$6¸Ñ$:¸F×<MÑ<MÓNˆŒð 	�‰Õr&   Nrm   rµ   rn   rl   r„   ra  rb  rc  Úentity_start_positionsÚentity_end_positionsro   rˆ  r¶   r  r  rØ   c                 ó²  — |�|n| j                   j                  }| j                  |||||||||||d¬«      }|j                  j	                  d«      }|	j                  d«      j                  dd|«      }	|	j                  |j                  j                  k7  r%|	j                  |j                  j                  «      }	t        j                  |j                  d|	«      }|
j                  d«      j                  dd|«      }
|
j                  |j                  j                  k7  r%|
j                  |j                  j                  «      }
t        j                  |j                  d|
«      }t        j                  |||j                  gd¬«      }| j                  |«      }| j                  |«      }d}|�Á|j                  |j                  «      }|j                  dk(  rJt         j"                  j%                  |j'                  d| j(                  «      |j'                  d«      «      }nMt         j"                  j+                  |j'                  d«      |j'                  d«      j-                  |«      «      }|s5t/        d„ |||j0                  |j2                  |j4                  fD «       «      S t7        |||j0                  |j2                  |j4                  ¬	«      S )
u  
        entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`):
            Indices of entity tokens in the entity vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.
        entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*):
            Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`:

            - 1 for entity tokens that are **not masked**,
            - 0 for entity tokens that are **masked**.
        entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*):
            Segment token indices to indicate first and second portions of the entity token inputs. Indices are
            selected in `[0, 1]`:

            - 0 corresponds to a *portion A* entity token,
            - 1 corresponds to a *portion B* entity token.
        entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*):
            Indices of positions of each input entity in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.
        entity_start_positions (`torch.LongTensor`):
            The start positions of entities in the word token sequence.
        entity_end_positions (`torch.LongTensor`):
            The end positions of entities in the word token sequence.
        labels (`torch.LongTensor` of shape `(batch_size, entity_length)` or `(batch_size, entity_length, num_labels)`, *optional*):
            Labels for computing the classification loss. If the shape is `(batch_size, entity_length)`, the cross
            entropy loss is used for the single-label classification. In this case, labels should contain the indices
            that should be in `[0, ..., config.num_labels - 1]`. If the shape is `(batch_size, entity_length,
            num_labels)`, the binary cross entropy loss is used for the multi-label classification. In this case,
            labels should only contain `[0, 1]`, where 0 and 1 indicate false and true, respectively.

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, LukeForEntitySpanClassification

        >>> tokenizer = AutoTokenizer.from_pretrained("studio-ousia/luke-large-finetuned-conll-2003")
        >>> model = LukeForEntitySpanClassification.from_pretrained("studio-ousia/luke-large-finetuned-conll-2003")

        >>> text = "BeyoncÃ© lives in Los Angeles"
        # List all possible entity spans in the text

        >>> word_start_positions = [0, 8, 14, 17, 21]  # character-based start positions of word tokens
        >>> word_end_positions = [7, 13, 16, 20, 28]  # character-based end positions of word tokens
        >>> entity_spans = []
        >>> for i, start_pos in enumerate(word_start_positions):
        ...     for end_pos in word_end_positions[i:]:
        ...         entity_spans.append((start_pos, end_pos))

        >>> inputs = tokenizer(text, entity_spans=entity_spans, return_tensors="pt")
        >>> outputs = model(**inputs)
        >>> logits = outputs.logits
        >>> predicted_class_indices = logits.argmax(-1).squeeze().tolist()
        >>> for span, predicted_class_idx in zip(entity_spans, predicted_class_indices):
        ...     if predicted_class_idx != 0:
        ...         print(text[span[0] : span[1]], model.config.id2label[predicted_class_idx])
        BeyoncÃ© PER
        Los Angeles LOC
        ```NTr‹  rb   r‡   r¥   rˆ   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wr�   r%   r  s     r'   r  z:LukeForEntitySpanClassification.forward.<locals>.<genexpr>  r—  r  r˜  )r_   r  r;  r  ri   ru   rv   re   rg   r!   Úgatherr¬   r   r\   r”  r™  r   r±   rš  r¦   r“  r›  rŒ   r$   r1   r   r2   r;   )r^   rm   rµ   rn   rl   r„   ra  rb  rc  r§  r¨  ro   rˆ  r¶   r  r  rh  rÐ   rQ   Ústart_statesÚ
end_statesr�  r/   r,   s                           r'   rr   z'LukeForEntitySpanClassification.forward¢  s‡  € ð^ &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—)‘)ØØ)Ø)Ø%Ø!Ø"7Ø"7Ø 3Ø'Ø/Ø!5Øð ó 
ˆð ×/Ñ/×4Ñ4°RÓ8ˆà!7×!AÑ!AÀ"Ó!E×!LÑ!LÈRÐQSÐU`Ó!aÐØ!×(Ñ(¨G×,EÑ,E×,LÑ,LÒLØ%;×%>Ñ%>¸w×?XÑ?X×?_Ñ?_Ó%`Ð"Ü—|‘| G×$=Ñ$=¸rÐCYÓZˆà3×=Ñ=¸bÓA×HÑHÈÈRÐQ\Ó]ÐØ×&Ñ&¨'×*CÑ*C×*JÑ*JÒJØ#7×#:Ñ#:¸7×;TÑ;T×;[Ñ;[Ó#\Ð Ü—\‘\ '×";Ñ";¸RÐAUÓVˆ
äŸ™ L°*¸g×>^Ñ>^Ð#_ÐefÔgˆàŸ™ nÓ5ˆØ—‘ Ó0ˆàˆØÐà—Y‘Y˜vŸ}™}Ó-ˆFð �{‰{˜aÒÜ—}‘}×2Ñ2°6·;±;¸rÀ4Ç?Á?Ó3SÐU[×U`ÑU`ÐacÓUdÓe‘ä—}‘}×EÑEÀfÇkÁkÐRTÃoÐW]×WbÑWbÐceÓWf×WnÑWnÐouÓWvÓw�áÜñ à ¨×(=Ñ(=¸w×?[Ñ?[Ð]d×]oÑ]oÐpôó ð ô .ØØØ!×/Ñ/Ø!(×!=Ñ!=Ø×)Ñ)ô
ð 	
r&   )NNNNNNNNNNNNNNN)r   r   r   rN   r   r!   r�   r"   rp  r$   r;   rr   rx   ry   s   @r'   r¥  r¥  �  s‘  ø„ ô
ð ð .2Ø37Ø26Ø04Ø.2Ø9=Ø9=Ø7;Ø:>Ø8<Ø26Ø*.Ø)-Ø,0Ø#'ñ!G
à×#Ñ# dÑ*ðG
ð ×)Ñ)¨DÑ0ðG
ð ×(Ñ(¨4Ñ/ð	G
ð
 ×&Ñ&¨Ñ-ðG
ð ×$Ñ$ tÑ+ðG
ð  %×/Ñ/°$Ñ6ðG
ð  %×/Ñ/°$Ñ6ðG
ð #×-Ñ-°Ñ4ðG
ð !&× 0Ñ 0°4Ñ 7ðG
ð $×.Ñ.°Ñ5ðG
ð ×(Ñ(¨4Ñ/ðG
ð × Ñ  4Ñ'ðG
ð   $™;ðG
ð # T™kðG
ð  ˜D‘[ð!G
ð$ 
Ð/Ñ	/ò%G
ó ôG
r&   r¥  z 
    The LUKE Model transformer with a sequence classification/regression head on top (a linear layer on top of the
    pooled output) e.g. for GLUE tasks.
    c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  de	e
z  fd„«       Zˆ xZS )ÚLukeForSequenceClassificationc                 ó\  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  �|j                  n|j                  «      | _	        t        j                  |j                  |j                  «      | _        | j                  «        y r�   ©rM   rN   r“  rO  r;  r   rZ   Úclassifier_dropoutr[   r\   r‚   rQ   r”  rT  r]   s     €r'   rN   z&LukeForSequenceClassification.__init__4  s‚   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ˜fÓ%ˆŒ	Ü—z‘zØ)/×)BÑ)BÐ)NˆF×%Ò%ÐTZ×TnÑTnó
ˆŒô Ÿ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr&   Nrm   rµ   rn   rl   r„   ra  rb  rc  ro   rˆ  r¶   r  r  rØ   c                 óä  — |�|n| j                   j                  }| j                  |||||||||	||d¬«      }|j                  }| j	                  |«      }| j                  |«      }d}|
��¢|
j                  |j                  «      }
| j                   j                  €�| j                  dk(  rd| j                   _        nl| j                  dkD  rL|
j                  t        j                  k(  s|
j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j                  dk(  rIt        «       }| j                  dk(  r& ||j                  «       |
j                  «       «      }nŒ |||
«      }n‚| j                   j                  dk(  r=t!        «       } ||j#                  d| j                  «      |
j#                  d«      «      }n,| j                   j                  dk(  rt%        «       } |||
«      }|s5t'        d	„ |||j(                  |j*                  |j,                  fD «       «      S t/        |||j(                  |j*                  |j,                  ¬
«      S )a�  
        entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`):
            Indices of entity tokens in the entity vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.
        entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*):
            Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`:

            - 1 for entity tokens that are **not masked**,
            - 0 for entity tokens that are **masked**.
        entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*):
            Segment token indices to indicate first and second portions of the entity token inputs. Indices are
            selected in `[0, 1]`:

            - 0 corresponds to a *portion A* entity token,
            - 1 corresponds to a *portion B* entity token.
        entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*):
            Indices of positions of each input entity in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NTr‹  r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationrb   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wr�   r%   r  s     r'   r  z8LukeForSequenceClassification.forward.<locals>.<genexpr>›  r—  r  r˜  )r_   r  r;  re  r\   r”  rg   re   Úproblem_typer“  rd   r!   rk   r™   r   Úsqueezer   r¦   r   r$   r1   r   r2   r=   )r^   rm   rµ   rn   rl   r„   ra  rb  rc  ro   rˆ  r¶   r  r  rh  rÐ   r,  r/   r,   Úloss_fcts                       r'   rr   z%LukeForSequenceClassification.forward@  s   € ðV &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—)‘)ØØ)Ø)Ø%Ø!Ø"7Ø"7Ø 3Ø'Ø/Ø!5Øð ó 
ˆð  ×-Ñ-ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆàˆØÑà—Y‘Y˜vŸ}™}Ó-ˆFØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�áÜñ à ¨×(=Ñ(=¸w×?[Ñ?[Ð]d×]oÑ]oÐpôó ð ô ,ØØØ!×/Ñ/Ø!(×!=Ñ!=Ø×)Ñ)ô
ð 	
r&   rž  )r   r   r   rN   r   r!   r�   r"   rp  r$   r=   rr   rx   ry   s   @r'   r¯  r¯  -  s_  ø„ ô
ð ð .2Ø37Ø26Ø04Ø.2Ø:>Ø9=Ø7;Ø26Ø+/Ø)-Ø,0Ø#'ñf
à×#Ñ# dÑ*ðf
ð ×)Ñ)¨DÑ0ðf
ð ×(Ñ(¨4Ñ/ð	f
ð
 ×&Ñ&¨Ñ-ðf
ð ×$Ñ$ tÑ+ðf
ð  %×0Ñ0°4Ñ7ðf
ð  %×/Ñ/°$Ñ6ðf
ð #×-Ñ-°Ñ4ðf
ð ×(Ñ(¨4Ñ/ðf
ð ×!Ñ! DÑ(ðf
ð   $™;ðf
ð # T™kðf
ð ˜D‘[ðf
ð  
Ð-Ñ	-ò!f
ó ôf
r&   r¯  zú
    The LUKE Model with a token classification head on top (a linear layer on top of the hidden-states output). To
    solve Named-Entity Recognition (NER) task using LUKE, `LukeForEntitySpanClassification` is more suitable than this
    class.
    c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  de	e
z  fd„«       Zˆ xZS )ÚLukeForTokenClassificationc                 ó`  •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        t        j                  |j                  �|j                  n|j                  «      | _	        t        j                  |j                  |j                  «      | _        | j                  «        y ©NF)rP  r±  r]   s     €r'   rN   z#LukeForTokenClassification.__init__²  s„   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä˜f¸Ô>ˆŒ	Ü—z‘zØ)/×)BÑ)BÐ)NˆF×%Ò%ÐTZ×TnÑTnó
ˆŒô Ÿ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr&   Nrm   rµ   rn   rl   r„   ra  rb  rc  ro   rˆ  r¶   r  r  rØ   c                 óL  — |�|n| j                   j                  }| j                  |||||||||	||d¬«      }|j                  }| j	                  |«      }| j                  |«      }d}|
�W|
j                  |j                  «      }
t        «       } ||j                  d| j                  «      |
j                  d«      «      }|s5t        d„ |||j                  |j                  |j                  fD «       «      S t        |||j                  |j                  |j                  ¬«      S )aM  
        entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`):
            Indices of entity tokens in the entity vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.
        entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*):
            Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`:

            - 1 for entity tokens that are **not masked**,
            - 0 for entity tokens that are **masked**.
        entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*):
            Segment token indices to indicate first and second portions of the entity token inputs. Indices are
            selected in `[0, 1]`:

            - 0 corresponds to a *portion A* entity token,
            - 1 corresponds to a *portion B* entity token.
        entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*):
            Indices of positions of each input entity in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        NTr‹  rb   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wr�   r%   r  s     r'   r  z5LukeForTokenClassification.forward.<locals>.<genexpr>  r—  r  r˜  )r_   r  r;  r  r\   r”  rg   re   r   r¦   r“  r$   r1   r   r2   r?   )r^   rm   rµ   rn   rl   r„   ra  rb  rc  ro   rˆ  r¶   r  r  rh  rÐ   ro  r/   r,   rº  s                       r'   rr   z"LukeForTokenClassification.forward¿  s.  € ðV &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—)‘)ØØ)Ø)Ø%Ø!Ø"7Ø"7Ø 3Ø'Ø/Ø!5Øð ó 
ˆð "×3Ñ3ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÐà—Y‘Y˜vŸ}™}Ó-ˆFÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDáÜñ à ¨×(=Ñ(=¸w×?[Ñ?[Ð]d×]oÑ]oÐpôó ð ô )ØØØ!×/Ñ/Ø!(×!=Ñ!=Ø×)Ñ)ô
ð 	
r&   rž  )r   r   r   rN   r   r!   r�   r"   rp  r$   r?   rr   rx   ry   s   @r'   r¼  r¼  ª  s_  ø„ ôð ð .2Ø37Ø26Ø04Ø.2Ø:>Ø9=Ø7;Ø26Ø+/Ø)-Ø,0Ø#'ñT
à×#Ñ# dÑ*ðT
ð ×)Ñ)¨DÑ0ðT
ð ×(Ñ(¨4Ñ/ð	T
ð
 ×&Ñ&¨Ñ-ðT
ð ×$Ñ$ tÑ+ðT
ð  %×0Ñ0°4Ñ7ðT
ð  %×/Ñ/°$Ñ6ðT
ð #×-Ñ-°Ñ4ðT
ð ×(Ñ(¨4Ñ/ðT
ð ×!Ñ! DÑ(ðT
ð   $™;ðT
ð # T™kðT
ð ˜D‘[ðT
ð  
Ð*Ñ	*ò!T
ó ôT
r&   r¼  c            !       óÀ  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  de	e
z  fd„«       Zˆ xZS )ÚLukeForQuestionAnsweringc                 óè   •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r¾  )
rM   rN   r“  rO  r;  r   r‚   rQ   Ú
qa_outputsrT  r]   s     €r'   rN   z!LukeForQuestionAnswering.__init__  sU   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒä˜f¸Ô>ˆŒ	ÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr&   Nrm   rµ   rn   rl   r„   ra  rb  rc  ro   Ústart_positionsÚend_positionsr¶   r  r  rØ   c                 ó^  — |�|n| j                   j                  }| j                  |||||||||	||d¬«      }|j                  }| j	                  |«      }|j                  dd¬«      \  }}|j                  d«      }|j                  d«      }d}|
�·|�µt        |
j                  «       «      dkD  r|
j                  d«      }
t        |j                  «       «      dkD  r|j                  d«      }|j                  d«      }|
j                  d|«       |j                  d|«       t        |¬«      } |||
«      } |||«      }||z   d	z  }|s6t        d
„ ||||j                  |j                  |j                  fD «       «      S t        ||||j                  |j                  |j                  ¬«      S )a  
        entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`):
            Indices of entity tokens in the entity vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.
        entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*):
            Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`:

            - 1 for entity tokens that are **not masked**,
            - 0 for entity tokens that are **masked**.
        entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*):
            Segment token indices to indicate first and second portions of the entity token inputs. Indices are
            selected in `[0, 1]`:

            - 0 corresponds to a *portion A* entity token,
            - 1 corresponds to a *portion B* entity token.
        entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*):
            Indices of positions of each input entity in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.
        NTr‹  r   rb   rˆ   r   )Úignore_indexr¥   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wr�   r%   r  s     r'   r  z3LukeForQuestionAnswering.forward.<locals>.<genexpr>v  s   è ø€ ò àð �=ô ñùr  )r,   rB   rC   r1   r   r2   )r_   r  r;  r  rÄ  Úsplitr¹  Úlenri   Úclamp_r   r$   r1   r   r2   rA   )r^   rm   rµ   rn   rl   r„   ra  rb  rc  ro   rÅ  rÆ  r¶   r  r  rh  rÐ   ro  r/   rB   rC   Ú
total_lossÚignored_indexrº  Ú
start_lossÚend_losss                             r'   rr   z LukeForQuestionAnswering.forward$  sÙ  € ðP &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—)‘)ØØ)Ø)Ø%Ø!Ø"7Ø"7Ø 3Ø'Ø/Ø!5Øð ó 
ˆð "×3Ñ3ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/ˆØ×'Ñ'¨Ó+ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ×"Ñ" 1 mÔ4Ø× Ñ   MÔ2ä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJáÜñ ð Ø ØØ×)Ñ)Ø×0Ñ0Ø×&Ñ&ðôó ð ô 0ØØ%Ø!Ø!×/Ñ/Ø!(×!=Ñ!=Ø×)Ñ)ô
ð 	
r&   r�  )r   r   r   rN   r   r!   r�   r"   rp  r$   rA   rr   rx   ry   s   @r'   rÂ  rÂ    sx  ø„ ô	ð ð .2Ø37Ø26Ø15Ø.2Ø:>Ø9=Ø7;Ø26Ø37Ø15Ø)-Ø,0Ø#'ñe
à×#Ñ# dÑ*ðe
ð ×)Ñ)¨DÑ0ðe
ð ×(Ñ(¨4Ñ/ð	e
ð
 ×'Ñ'¨$Ñ.ðe
ð ×$Ñ$ tÑ+ðe
ð  %×0Ñ0°4Ñ7ðe
ð  %×/Ñ/°$Ñ6ðe
ð #×-Ñ-°Ñ4ðe
ð ×(Ñ(¨4Ñ/ðe
ð ×)Ñ)¨DÑ0ðe
ð ×'Ñ'¨$Ñ.ðe
ð   $™;ðe
ð # T™kðe
ð ˜D‘[ðe
ð" 
Ð1Ñ	1ò#e
ó ôe
r&   rÂ  c                   ó   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  de	e
z  fd„«       Zˆ xZS )ÚLukeForMultipleChoicec                 ó&  •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  �|j                  n|j                  «      | _        t	        j                  |j                  d«      | _        | j                  «        y rü   )rM   rN   rO  r;  r   rZ   r²  r[   r\   r‚   rQ   r”  rT  r]   s     €r'   rN   zLukeForMultipleChoice.__init__�  so   ø€ Ü‰Ñ˜Ô ä˜fÓ%ˆŒ	Ü—z‘zØ)/×)BÑ)BÐ)NˆF×%Ò%ÐTZ×TnÑTnó
ˆŒô Ÿ)™) F×$6Ñ$6¸Ó:ˆŒð 	�‰Õr&   Nrm   rµ   rn   rl   r„   ra  rb  rc  ro   rˆ  r¶   r  r  rØ   c                 ó8  — |�|n| j                   j                  }|�|j                  d   n|	j                  d   }|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|	�1|	j                  d|	j	                  d«      |	j	                  d«      «      nd}	|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�1|j                  d|j	                  d«      |j	                  d«      «      nd}| j                  |||||||||	||d¬«      }|j                  }| j                  |«      }| j                  |«      }|j                  d|«      }d}|
�.|
j                  |j                  «      }
t        «       } |||
«      }|s5t        d„ |||j                  |j                  |j                  fD «       «      S t!        |||j                  |j                  |j                  ¬«      S )	a^  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        entity_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`):
            Indices of entity tokens in the entity vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.
        entity_attention_mask (`torch.FloatTensor` of shape `(batch_size, entity_length)`, *optional*):
            Mask to avoid performing attention on padding entity token indices. Mask values selected in `[0, 1]`:

            - 1 for entity tokens that are **not masked**,
            - 0 for entity tokens that are **masked**.
        entity_token_type_ids (`torch.LongTensor` of shape `(batch_size, entity_length)`, *optional*):
            Segment token indices to indicate first and second portions of the entity token inputs. Indices are
            selected in `[0, 1]`:

            - 0 corresponds to a *portion A* entity token,
            - 1 corresponds to a *portion B* entity token.
        entity_position_ids (`torch.LongTensor` of shape `(batch_size, entity_length, max_mention_length)`, *optional*):
            Indices of positions of each input entity in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   rb   r‡   Tr‹  c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wr�   r%   r  s     r'   r  z0LukeForMultipleChoice.forward.<locals>.<genexpr>  r  r  r˜  )r_   r  Úshaper¦   ri   r;  re  r\   r”  rg   re   r   r$   r1   r   r2   rE   )r^   rm   rµ   rn   rl   r„   ra  rb  rc  ro   rˆ  r¶   r  r  rh  Únum_choicesrÐ   r,  r/   Úreshaped_logitsr,   rº  s                         r'   rr   zLukeForMultipleChoice.forward›  sÛ  € ðF &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆØ,5Ð,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆà>GÐ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÐG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ð(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð BLÐAW�Z—_‘_ R¨¯©¸Ó)<Ô=Ð]aˆ
ð %Ð0ð "×&Ñ& rÐ+@×+EÑ+EÀbÓ+IÔJàð 	ð %Ð0ð "×&Ñ& rÐ+@×+EÑ+EÀbÓ+IÔJàð 	ð #Ð.ð  ×$Ñ$ RÐ)<×)AÑ)AÀ"Ó)EÐGZ×G_ÑG_Ð`bÓGcÔdàð 	ð —)‘)ØØ)Ø)Ø%Ø!Ø"7Ø"7Ø 3Ø'Ø/Ø!5Øð ó 
ˆð  ×-Ñ-ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÐà—Y‘Y˜×5Ñ5Ó6ˆFÜ'Ó)ˆHÙ˜O¨VÓ4ˆDáÜñ 
ð Ø#Ø×)Ñ)Ø×0Ñ0Ø×&Ñ&ðô
ó 
ð 
ô -ØØ"Ø!×/Ñ/Ø!(×!=Ñ!=Ø×)Ñ)ô
ð 	
r&   rž  )r   r   r   rN   r   r!   r�   r"   rp  r$   rE   rr   rx   ry   s   @r'   rÒ  rÒ  �  s_  ø„ ô
ð ð .2Ø37Ø26Ø04Ø.2Ø:>Ø9=Ø7;Ø26Ø+/Ø)-Ø,0Ø#'ñO
à×#Ñ# dÑ*ðO
ð ×)Ñ)¨DÑ0ðO
ð ×(Ñ(¨4Ñ/ð	O
ð
 ×&Ñ&¨Ñ-ðO
ð ×$Ñ$ tÑ+ðO
ð  %×0Ñ0°4Ñ7ðO
ð  %×/Ñ/°$Ñ6ðO
ð #×-Ñ-°Ñ4ðO
ð ×(Ñ(¨4Ñ/ðO
ð ×!Ñ! DÑ(ðO
ð   $™;ðO
ð # T™kðO
ð ˜D‘[ðO
ð  
Ð.Ñ	.ò!O
ó ôO
r&   rÒ  )
r�  r   r¥  rÒ  rÂ  r¯  r¼  r}  rO  r:  )Hr    r¯   Údataclassesr   r!   r   Útorch.nnr   r   r   Ú r	   rC  Úactivationsr
   r   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_utilsr   Úpytorch_utilsr   Úutilsr   r   r   Úconfiguration_luker   Ú
get_loggerr   Úloggerr   r)   r+   r4   r9   r;   r=   r?   rA   rE   rM  rG   r{   r’   rÓ   rá   rì   rö   rú   r  r&  r.  r3  r:  rO  rf   rw  r}  r�  r   r¥  r¯  r¼  rÂ  rÒ  Ú__all__r%   r&   r'   ú<module>rç     s�  ðñ ã Ý !ã Ý ß AÑ Aå &ß 'Ý 6Ý 9ß KÝ -Ý 6ß 9Ñ 9Ý *ð 
ˆ×	Ñ	˜HÓ	%€ñ ðôð
 ôFÐ%?ó Fó óðFñ" ðôð
 ôF˜/ó Fó óðFñ ðôð
 ô<˜ó <ó óð<ñ8 ðôð
 ô< ó <ó óð<ñ& ðôð
 ô< [ó <ó óð<ñ& ðôð
 ô< [ó <ó óð<ñ& ðôð
 ô< ;ó <ó óð<ñ& ðôð
 ô< ó <ó óð<ñ& ðôð
 ô< {ó <ó óð<ñ$ ðôð
 ô< Kó <ó óð<ô*D=�R—Y‘Yô D=ôN(˜2Ÿ9™9ô (ôVi˜Ÿ	™	ô iôZ�R—Y‘Yô ô&�B—I‘Iô &ôT�r—y‘yô ô �—‘ô ô/Ð*ô /ôd>
�"—)‘)ô >
ôD�—‘ô ô B§I¡Iô ô"˜2Ÿ9™9ô ð ô?˜/ó ?ó ð?ñ( ðôô
w
Ð#ó w
óð
w
òt4ô"�—‘ô ñ, ðôôJ
Ð)ó J
óðJ
ñZ ðôôx
Ð"5ó x
óðx
ñv ðôô}
Ð&9ó }
óð}
ñ@ ðôôU
Ð&9ó U
óðU
ñp ðôôt
Ð$7ó t
óðt
ñn ðôôc
Ð!4ó c
óðc
ðL ôr
Ð2ó r
ó ðr
ðj ô]
Ð/ó ]
ó ð]
ò@�r&   