Ë
    ÜÓ^j8} ã                   ó®  — d Z ddlmZ ddlmZ ddlZddlmZ ddlmZm	Z	m
Z
 ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZ ddlmZ  ej:                  e«      Z G d„ dej@                  «      Z! G d„ dej@                  «      Z" G d„ dej@                  «      Z# G d„ dej@                  «      Z$ G d„ dej@                  «      Z% G d„ dej@                  «      Z& G d„ dej@                  «      Z'e G d„ de«      «       Z( ed ¬!«      e G d"„ d#e«      «       «       Z) ed$¬!«      e G d%„ d&e«      «       «       Z* ed'¬!«      e G d(„ d)e«      «       «       Z+ ed*¬!«      e G d+„ d,e«      «       «       Z, ed-¬!«      e G d.„ d/e«      «       «       Z- ed0¬!«      e G d1„ d2e«      «       «       Z. ed3¬!«      e G d4„ d5e«      «       «       Z/e G d6„ d7e(«      «       Z0 ed8¬!«       G d9„ d:e(e«      «       Z1 ed;¬!«       G d<„ d=e(«      «       Z2e G d>„ d?e(«      «       Z3e G d@„ dAe(«      «       Z4 edB¬!«       G dC„ dDe(«      «       Z5e G dE„ dFe(«      «       Z6g dG¢Z7y)Hz
PyTorch XLNet model.
é    )ÚCallable)Ú	dataclassN)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FNÚget_activation)ÚGenerationMixin)ÚPreTrainedModel)Úapply_chunking_to_forward)ÚModelOutputÚauto_docstringÚloggingé   )ÚXLNetConfigc                   óf   ‡ — e Zd Zˆ fd„Zedd„«       Zedd„«       Z	 	 	 dd„Zd	d„Z	 	 	 dd„Z	ˆ xZ
S )
ÚXLNetRelativeAttentionc                 óô  •— t         ‰| �  «        |j                  |j                  z  dk7  r%t	        d|j                  › d|j                  › �«      ‚|j                  | _        |j
                  | _        |j                  | _        d|j
                  dz  z  | _        t        j                  t        j                  |j                  | j                  | j
                  «      «      | _        t        j                  t        j                  |j                  | j                  | j
                  «      «      | _        t        j                  t        j                  |j                  | j                  | j
                  «      «      | _        t        j                  t        j                  |j                  | j                  | j
                  «      «      | _        t        j                  t        j                  |j                  | j                  | j
                  «      «      | _        t        j                  t        j                  | j                  | j
                  «      «      | _        t        j                  t        j                  | j                  | j
                  «      «      | _        t        j                  t        j                  | j                  | j
                  «      «      | _        t        j                  t        j                  d| j                  | j
                  «      «      | _        t        j(                  |j                  |j*                  ¬«      | _        t        j.                  |j0                  «      | _        y )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (r   ç      à?é   ©Úeps)ÚsuperÚ__init__Úd_modelÚn_headÚ
ValueErrorÚd_headÚscaler   Ú	ParameterÚtorchÚFloatTensorÚqÚkÚvÚoÚrÚr_r_biasÚr_s_biasÚr_w_biasÚ	seg_embedÚ	LayerNormÚlayer_norm_epsÚ
layer_normÚDropoutÚdropout©ÚselfÚconfigÚ	__class__s     €ús/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/xlnet/modeling_xlnet.pyr   zXLNetRelativeAttention.__init__'   s$  ø€ Ü‰ÑÔà�>‰>˜FŸM™MÑ)¨QÒ.ÜØ# F§N¡NÐ#3ð 4Ø Ÿ-™-˜ð*óð ð
 —m‘mˆŒØ—m‘mˆŒØ—~‘~ˆŒØ˜&Ÿ-™-¨Ñ,Ñ-ˆŒ
ä—‘œe×/Ñ/°·±ÀÇÁÈTÏ[É[ÓYÓZˆŒÜ—‘œe×/Ñ/°·±ÀÇÁÈTÏ[É[ÓYÓZˆŒÜ—‘œe×/Ñ/°·±ÀÇÁÈTÏ[É[ÓYÓZˆŒÜ—‘œe×/Ñ/°·±ÀÇÁÈTÏ[É[ÓYÓZˆŒÜ—‘œe×/Ñ/°·±ÀÇÁÈTÏ[É[ÓYÓZˆŒäŸ™¤U×%6Ñ%6°t·{±{ÀDÇKÁKÓ%PÓQˆŒÜŸ™¤U×%6Ñ%6°t·{±{ÀDÇKÁKÓ%PÓQˆŒÜŸ™¤U×%6Ñ%6°t·{±{ÀDÇKÁKÓ%PÓQˆŒÜŸ™¤e×&7Ñ&7¸¸4¿;¹;ÈÏÉÓ&TÓUˆŒäŸ,™, v§~¡~¸6×;PÑ;PÔQˆŒÜ—z‘z &§.¡.Ó1ˆ�ó    c           	      ó@  — | j                   }| j                  |d   |d   |d   |d   «      } | dd…df   } | j                  |d   |d   dz
  |d   |d   «      } t        j                  | dt        j                  || j
                  t        j                  ¬«      «      } | S )z<perform relative shift to form the relative attention score.r   r   r   r	   N.©ÚdeviceÚdtype©ÚshapeÚreshaper$   Úindex_selectÚaranger<   Úlong©ÚxÚklenÚx_sizes      r8   Ú	rel_shiftz XLNetRelativeAttention.rel_shiftC   s–   € ð —‘ˆà�I‰I�f˜Q‘i ¨¡¨F°1©I°v¸a±yÓAˆØˆa‰b�#ˆg‰JˆØ�I‰I�f˜Q‘i ¨¡¨Q¡°°q±	¸6À!¹9ÓEˆä×Ñ˜q !¤U§\¡\°$¸q¿x¹xÌuÏzÉzÔ%ZÓ[ˆàˆr9   c           	      óP  — | j                   }| j                  |d   |d   |d   |d   «      } | d d …d d …dd …d d …f   } | j                  |d   |d   |d   |d   dz
  «      } t        j                  | dt        j                  || j
                  t        j                  ¬«      «      } | S )Nr   r   r	   r   r;   r>   rD   s      r8   Úrel_shift_bnijz%XLNetRelativeAttention.rel_shift_bnijP   sœ   € à—‘ˆà�I‰I�f˜Q‘i ¨¡¨F°1©I°v¸a±yÓAˆØŠa’�A‘Bšˆk‰NˆØ�I‰I�f˜Q‘i ¨¡¨F°1©I°v¸a±yÀ1±}ÓEˆô ×Ñ˜q !¤U§\¡\°$¸q¿x¹xÌuÏzÉzÔ%ZÓ[ˆð ˆr9   c                 ó
  — t        j                  d|| j                  z   |«      }t        j                  d|| j                  z   |«      }	| j	                  |	|j
                  d   ¬«      }	|€d}
nEt        j                  d|| j                  z   | j                  «      }
t        j                  d||
«      }
||	z   |
z   | j                  z  }|�V|j                  t         j                  k(  r|dt        j                  d|«      z  z
  }n|d	t        j                  d|«      z  z
  }t        j                  j                  |d¬
«      }| j                  |«      }t        j                  d||«      }|r|t        j                  d|«      fS |S )z.Core relative positional attention operations.zibnd,jbnd->bnijr	   )rF   r   zibnd,snd->ibnszijbs,ibns->bnijéÜÿ  z
ijbn->bnijçêŒ 9Y>)F©Údimzbnij,jbnd->ibndz
bnij->ijbn)r$   Úeinsumr-   r+   rJ   r?   r,   r.   r"   r=   Úfloat16r   Ú
functionalÚsoftmaxr3   )r5   Úq_headÚk_head_hÚv_head_hÚk_head_rÚseg_matÚ	attn_maskÚoutput_attentionsÚacÚbdÚefÚ
attn_scoreÚ	attn_probÚattn_vecs                 r8   Úrel_attn_corez$XLNetRelativeAttention.rel_attn_core_   sX  € ô �\‰\Ð+¨V°d·m±mÑ-CÀXÓNˆô �\‰\Ð+¨V°d·m±mÑ-CÀXÓNˆØ× Ñ  ¨"¯(©(°1©+Ð Ó6ˆð ˆ?Ø‰Bä—‘Ð.°¸¿¹Ñ0FÈÏÉÓWˆBÜ—‘Ð/°¸"Ó=ˆBð ˜2‘g ‘l d§j¡jÑ0ˆ
ØÐ à�‰¤%§-¡-Ò/Ø'¨%´%·,±,¸|ÈYÓ2WÑ*WÑW‘
à'¨$´·±¸lÈIÓ1VÑ*VÑV�
ô —M‘M×)Ñ)¨*¸!Ð)Ó<ˆ	Ø—L‘L Ó+ˆ	ô —<‘<Ð 1°9¸hÓGˆáØœUŸ\™\¨,¸	ÓBÐBÐBàˆr9   c                 óš   — t        j                  d|| j                  «      }| j                  |«      }|r||z   }| j	                  |«      }|S )zPost-attention processing.zibnd,hnd->ibh)r$   rP   r)   r3   r1   )r5   Úhr`   ÚresidualÚattn_outÚoutputs         r8   Úpost_attentionz%XLNetRelativeAttention.post_attentionŽ   sG   € ô —<‘< °¸4¿6¹6ÓBˆà—<‘< Ó)ˆÙØ !‘|ˆHØ—‘ Ó*ˆàˆr9   c
           	      ó\  — |��‘|�-|j                  «       dkD  rt        j                  ||gd¬«      }
n|}
t        j                  d|
| j                  «      }t        j                  d|
| j
                  «      }t        j                  d|| j                  «      }t        j                  d|| j                  «      }| j                  |||||||	¬«      }|	r|\  }}| j                  ||«      }t        j                  d|| j                  «      }|�Nt        j                  d||«      }| j                  |||||||	¬«      }|	r|\  }}t        j                  d||«      }n| j                  |||||||	¬«      }|	r|\  }}| j                  ||«      }|	�rf}�n|�-|j                  «       dkD  rt        j                  ||gd¬«      }
n|}
t        j                  d|| j                  «      }t        j                  d|
| j                  «      }t        j                  d|
| j
                  «      }t        j                  d|j                  | j                  j                  «      | j                  «      }| j                  |||||||	¬«      }|	r|\  }}| j                  ||«      }d }||f}|	r|fz   }|S )Nr   r   rN   zibh,hnd->ibnd)rX   rY   rZ   zmbnd,mlb->lbndzlbnd,mlb->mbnd)rO   r$   ÚcatrP   r'   r(   r*   r&   ra   rg   Útyper=   )r5   rc   ÚgÚattn_mask_hÚattn_mask_gr*   rX   ÚmemsÚtarget_mappingrZ   ri   rU   rV   rW   Úq_head_hÚ
attn_vec_hÚattn_prob_hÚoutput_hÚq_head_gÚ
attn_vec_gÚattn_prob_gÚoutput_gr_   r`   Úoutputss                            r8   ÚforwardzXLNetRelativeAttention.forwardš   sÊ  € ð ‰=ð Ð D§H¡H£J°¢NÜ—i‘i  q 	¨qÔ1‘à�ô —|‘| O°S¸$¿&¹&ÓAˆHô —|‘| O°S¸$¿&¹&ÓAˆHô —|‘| O°Q¸¿¹Ó?ˆHô —|‘| O°Q¸¿¹Ó?ˆHð ×+Ñ+ØØØØØØ%Ø"3ð ,ó ˆJñ !Ø*4Ñ'�
˜Kð ×*Ñ*¨1¨jÓ9ˆHô —|‘| O°Q¸¿¹Ó?ˆHð Ð)Ü Ÿ<™<Ð(8¸(ÀNÓS�Ø!×/Ñ/ØØØØØ#Ø)Ø&7ð 0ó �
ñ %Ø.8Ñ+�J ä"Ÿ\™\Ð*:¸JÈÓW‘
à!×/Ñ/ØØØØØ#Ø)Ø&7ð 0ó �
ñ %Ø.8Ñ+�J ð ×*Ñ*¨1¨jÓ9ˆHâ Ø'¨Ð4’	ð Ð D§H¡H£J°¢NÜ—i‘i  q 	¨qÔ1‘à�ô —|‘| O°Q¸¿¹Ó?ˆHÜ—|‘| O°S¸$¿&¹&ÓAˆHÜ—|‘| O°S¸$¿&¹&ÓAˆHô —|‘| O°Q·V±V¸D¿F¹F¿L¹LÓ5IÈ4Ï6É6ÓRˆHð ×)Ñ)ØØØØØØ%Ø"3ð *ó ˆHñ !Ø&.Ñ#�˜)ð ×*Ñ*¨1¨hÓ7ˆHØˆHà˜XÐ&ˆÙØ  Ñ,ˆGØˆr9   ©éÿÿÿÿ©NNF)T)Ú__name__Ú
__module__Ú__qualname__r   ÚstaticmethodrH   rJ   ra   rg   ry   Ú__classcell__©r7   s   @r8   r   r   &   sW   ø„ ô2ð8 ò
ó ð
ð òó ðð( ØØó-ó^
ð( ØØ÷@r9   r   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚXLNetFeedForwardc                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  ¬«      | _        t        j                  |j                  |j                  «      | _	        t        j                  |j                  |j                  «      | _
        t        j                  |j                  «      | _        t        |j                  t        «      rt         |j                     | _        y |j                  | _        y )Nr   )r   r   r   r/   r   r0   r1   ÚLinearÚd_innerÚlayer_1Úlayer_2r2   r3   Ú
isinstanceÚff_activationÚstrr   Úactivation_functionr4   s     €r8   r   zXLNetFeedForward.__init__  s£   ø€ Ü‰ÑÔÜŸ,™, v§~¡~¸6×;PÑ;PÔQˆŒÜ—y‘y §¡°·±Ó@ˆŒÜ—y‘y §¡°·±Ó@ˆŒÜ—z‘z &§.¡.Ó1ˆŒÜ�f×*Ñ*¬CÔ0Ü'-¨f×.BÑ.BÑ'CˆDÕ$à'-×';Ñ';ˆDÕ$r9   c                 óÜ   — |}| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j	                  ||z   «      }|S ©N)rˆ   r�   r3   r‰   r1   )r5   Úinprf   s      r8   ry   zXLNetFeedForward.forward)  sf   € ØˆØ—‘˜fÓ%ˆØ×)Ñ)¨&Ó1ˆØ—‘˜fÓ%ˆØ—‘˜fÓ%ˆØ—‘˜fÓ%ˆØ—‘ ¨#¡Ó.ˆØˆr9   )r}   r~   r   r   ry   r�   r‚   s   @r8   r„   r„     s   ø„ ô	<ör9   r„   c                   ó2   ‡ — e Zd Zˆ fd„Z	 	 	 dd„Zd„ Zˆ xZS )Ú
XLNetLayerc                 óÚ   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        t        j                  |j                  «      | _        |j                  | _	        d| _
        y ©Nr   )r   r   r   Úrel_attnr„   Úffr   r2   r3   Úchunk_size_feed_forwardÚseq_len_dimr4   s     €r8   r   zXLNetLayer.__init__5  sP   ø€ Ü‰ÑÔÜ.¨vÓ6ˆŒÜ" 6Ó*ˆŒÜ—z‘z &§.¡.Ó1ˆŒØ'-×'EÑ'EˆÔ$ØˆÕr9   c
                 ó  — | j                  |||||||||	¬«	      }
|
d d \  }}|�,t        | j                  | j                  | j                  |«      }t        | j                  | j                  | j                  |«      }||f|
dd  z   }
|
S )N)rn   ro   rZ   r   )r•   r   Úff_chunkr—   r˜   )r5   rs   rw   rl   rm   r*   rX   rn   ro   rZ   rx   s              r8   ry   zXLNetLayer.forward=  s«   € ð —-‘-ØØØØØØØØ)Ø/ð  ó 

ˆð % R a˜[Ñˆ�(àÐÜ0Ø—‘˜t×;Ñ;¸T×=MÑ=MÈxóˆHô -¨T¯]©]¸D×<XÑ<XÐZ^×ZjÑZjÐltÓuˆà˜XÐ&¨°°¨Ñ4ˆØˆr9   c                 ó(   — | j                  |«      }|S r�   )r–   )r5   Úoutput_xs     r8   rš   zXLNetLayer.ff_chunk_  s   € Ø—7‘7˜8Ó$ˆØˆr9   r|   )r}   r~   r   r   ry   rš   r�   r‚   s   @r8   r’   r’   4  s   ø„ ôð  ØØó öDr9   r’   c                   ó€   ‡ — e Zd ZdZdefˆ fd„Zd	dej                  dej                  dz  dej                  fd„Zˆ xZ	S )
ÚXLNetPoolerStartLogitszÍ
    Compute SQuAD start logits from sequence hidden states.

    Args:
        config ([`XLNetConfig`]):
            The config used by the model, will be used to grab the `hidden_size` of the model.
    r6   c                 ól   •— t         ‰| �  «        t        j                  |j                  d«      | _        y r”   )r   r   r   r†   Úhidden_sizeÚdenser4   s     €r8   r   zXLNetPoolerStartLogits.__init__n  s&   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°1Ó5ˆ�
r9   NÚhidden_statesÚp_maskÚreturnc                 óÀ   — | j                  |«      j                  d«      }|�;|j                  t        j                  k(  r|d|z
  z  d|z  z
  }|S |d|z
  z  d|z  z
  }|S )aì  
        Args:
            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
                The final hidden states of the model.
            p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*):
                Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token
                should be masked.

        Returns:
            `torch.FloatTensor`: The start logits for SQuAD.
        r{   r   rL   rM   )r¡   Úsqueezer=   r$   rQ   )r5   r¢   r£   rE   s       r8   ry   zXLNetPoolerStartLogits.forwardr  so   € ð �J‰J�}Ó%×-Ñ-¨bÓ1ˆàÐØ�|‰|œuŸ}™}Ò,Ø˜˜V™Ñ$ u¨v¡~Ñ5�ð ˆð ˜˜V™Ñ$ t¨f¡}Ñ4�àˆr9   r�   )
r}   r~   r   Ú__doc__r   r   r$   r%   ry   r�   r‚   s   @r8   rž   rž   e  sG   ø„ ñð6˜{õ 6ñ U×%6Ñ%6ð À×@QÑ@QÐTXÑ@Xð Ðdi×duÑdu÷ r9   rž   c                   óÂ   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 ddej                  dej                  dz  dej                  dz  dej                  dz  d	ej                  f
d
„Z	ˆ xZ
S )ÚXLNetPoolerEndLogitsz÷
    Compute SQuAD end logits from sequence hidden states.

    Args:
        config ([`XLNetConfig`]):
            The config used by the model, will be used to grab the `hidden_size` of the model and the `layer_norm_eps`
            to use.
    r6   c                 ób  •— t         ‰| �  «        t        j                  |j                  dz  |j                  «      | _        t        j                  «       | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  d«      | _
        y )Nr   r   r   )r   r   r   r†   r    Údense_0ÚTanhÚ
activationr/   r0   Údense_1r4   s     €r8   r   zXLNetPoolerEndLogits.__init__”  st   ø€ Ü‰ÑÔÜ—y‘y ×!3Ñ!3°aÑ!7¸×9KÑ9KÓLˆŒÜŸ'™'›)ˆŒÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—y‘y ×!3Ñ!3°QÓ7ˆ�r9   Nr¢   Ústart_statesÚstart_positionsr£   r¤   c                 ó  — |€	|€J d«       ‚|�R|j                   dd \  }}|dd…ddf   j                  dd|«      }|j                  d|«      }|j                  d|d«      }| j                  t	        j
                  ||gd¬«      «      }| j                  |«      }| j                  |«      }| j                  |«      j                  d«      }|�;|j                  t        j                  k(  r|d|z
  z  d|z  z
  }|S |d|z
  z  d|z  z
  }|S )	aë  
        Args:
            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
                The final hidden states of the model.
            start_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`, *optional*):
                The hidden states of the first tokens for the labeled span.
            start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
                The position of the first token for the labeled span.
            p_mask (`torch.FloatTensor` of shape `(batch_size, seq_len)`, *optional*):
                Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token
                should be masked.

        <Tip>

        One of `start_states` or `start_positions` should be not `None`. If both are set, `start_positions` overrides
        `start_states`.

        </Tip>

        Returns:
            `torch.FloatTensor`: The end logits for SQuAD.
        Nú7One of start_states, start_positions should be not Noneéþÿÿÿr{   rN   r   rL   rM   )r?   ÚexpandÚgatherr«   r$   ri   r­   r/   r®   r¦   r=   rQ   )r5   r¢   r¯   r°   r£   ÚslenÚhszrE   s           r8   ry   zXLNetPoolerEndLogits.forward›  s&  € ð: Ð'¨?Ð+Fð 	
ØEó	
ÐFð Ð&Ø%×+Ñ+¨B¨CÐ0‰IˆD�#Ø-ªa°°t¨mÑ<×CÑCÀBÈÈCÓPˆOØ(×/Ñ/°°OÓDˆLØ'×.Ñ.¨r°4¸Ó<ˆLà�L‰LœŸ™ M°<Ð#@ÀbÔIÓJˆØ�O‰O˜AÓˆØ�N‰N˜1ÓˆØ�L‰L˜‹O×#Ñ# BÓ'ˆàÐØ�|‰|œuŸ}™}Ò,Ø˜˜V™Ñ$ u¨v¡~Ñ5�ð ˆð ˜˜V™Ñ$ t¨f¡}Ñ4�àˆr9   ©NNN©r}   r~   r   r§   r   r   r$   r%   Ú
LongTensorry   r�   r‚   s   @r8   r©   r©   Š  s�   ø„ ñð8˜{õ 8ð 26Ø37Ø+/ñ1à×(Ñ(ð1ð ×'Ñ'¨$Ñ.ð1ð ×)Ñ)¨DÑ0ð	1ð
 ×!Ñ! DÑ(ð1ð 
×	Ñ	÷1r9   r©   c                   óÂ   ‡ — e Zd ZdZdefˆ fd„Z	 	 	 ddej                  dej                  dz  dej                  dz  dej                  dz  d	ej                  f
d
„Z	ˆ xZ
S )ÚXLNetPoolerAnswerClasszè
    Compute SQuAD 2.0 answer class from classification and start tokens hidden states.

    Args:
        config ([`XLNetConfig`]):
            The config used by the model, will be used to grab the `hidden_size` of the model.
    r6   c                 ó  •— t         ‰| �  «        t        j                  |j                  dz  |j                  «      | _        t        j                  «       | _        t        j                  |j                  dd¬«      | _        y )Nr   r   F©Úbias)	r   r   r   r†   r    r«   r¬   r­   r®   r4   s     €r8   r   zXLNetPoolerAnswerClass.__init__Ù  sX   ø€ Ü‰ÑÔÜ—y‘y ×!3Ñ!3°aÑ!7¸×9KÑ9KÓLˆŒÜŸ'™'›)ˆŒÜ—y‘y ×!3Ñ!3°Q¸UÔCˆ�r9   Nr¢   r¯   r°   Ú	cls_indexr¤   c                 óþ  — |j                   d   }|€	|€J d«       ‚|�<|dd…ddf   j                  dd|«      }|j                  d|«      j                  d«      }|�=|dd…ddf   j                  dd|«      }|j                  d|«      j                  d«      }n|dd…ddd…f   }| j	                  t        j                  ||gd¬«      «      }| j                  |«      }| j                  |«      j                  d«      }|S )a¸  
        Args:
            hidden_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`):
                The final hidden states of the model.
            start_states (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`, *optional*):
                The hidden states of the first tokens for the labeled span.
            start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
                The position of the first token for the labeled span.
            cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
                Position of the CLS token for each sentence in the batch. If `None`, takes the last token.

        <Tip>

        One of `start_states` or `start_positions` should be not `None`. If both are set, `start_positions` overrides
        `start_states`.

        </Tip>

        Returns:
            `torch.FloatTensor`: The SQuAD 2.0 answer class.
        r{   Nr²   r³   rN   )	r?   r´   rµ   r¦   r«   r$   ri   r­   r®   )r5   r¢   r¯   r°   rÀ   r·   Úcls_token_staterE   s           r8   ry   zXLNetPoolerAnswerClass.forwardß  s  € ð: ×!Ñ! "Ñ%ˆØÐ'¨?Ð+Fð 	
ØEó	
ÐFð Ð&Ø-ªa°°t¨mÑ<×CÑCÀBÈÈCÓPˆOØ(×/Ñ/°°OÓD×LÑLÈRÓPˆLàÐ Ø!¢! T¨4 -Ñ0×7Ñ7¸¸BÀÓDˆIØ+×2Ñ2°2°yÓA×IÑIÈ"ÓM‰Oà+ªA¨r²1¨HÑ5ˆOà�L‰LœŸ™ L°/Ð#BÈÔKÓLˆØ�O‰O˜AÓˆØ�L‰L˜‹O×#Ñ# BÓ'ˆàˆr9   r¸   r¹   r‚   s   @r8   r¼   r¼   Ð  sƒ   ø„ ñðD˜{õ Dð 26Ø37Ø-1ñ/à×(Ñ(ð/ð ×'Ñ'¨$Ñ.ð/ð ×)Ñ)¨DÑ0ð	/ð
 ×#Ñ# dÑ*ð/ð 
×	Ñ	÷/r9   r¼   c                   ó‚   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej                  dej                  dz  dej                  fd„Z	ˆ xZ
S )
ÚXLNetSequenceSummaryaÍ  
    Compute a single vector summary of a sequence hidden states.

    Args:
        config ([`XLNetConfig`]):
            The config used by the model. Relevant arguments in the config class of the model are (refer to the actual
            config class of your model for the default values it uses):

            - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are:

                - `"last"` -- Take the last token hidden state (like XLNet)
                - `"first"` -- Take the first token hidden state (like Bert)
                - `"mean"` -- Take the mean of all tokens hidden states
                - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2)
                - `"attn"` -- Not implemented now, use multi-head attention

            - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
            - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
              (otherwise to `config.hidden_size`).
            - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
              another string or `None` will add no activation.
            - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
            - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.
    r6   c                 óf  •— t         ‰| �  «        t        |dd«      | _        | j                  dk(  rt        ‚t        j                  «       | _        t        |d«      rq|j                  ret        |d«      r(|j                  r|j                  dkD  r|j                  }n|j                  }t        j                  |j                  |«      | _        t        |dd «      }|rt        |«      nt        j                  «       | _        t        j                  «       | _        t        |d«      r3|j"                  dkD  r$t        j$                  |j"                  «      | _        t        j                  «       | _        t        |d	«      r5|j(                  dkD  r%t        j$                  |j(                  «      | _        y y y )
NÚsummary_typeÚlastÚattnÚsummary_use_projÚsummary_proj_to_labelsr   Úsummary_activationÚsummary_first_dropoutÚsummary_last_dropout)r   r   ÚgetattrrÆ   ÚNotImplementedErrorr   ÚIdentityÚsummaryÚhasattrrÉ   rÊ   Ú
num_labelsr    r†   r   r­   Úfirst_dropoutrÌ   r2   Úlast_dropoutrÍ   )r5   r6   Únum_classesÚactivation_stringr7   s       €r8   r   zXLNetSequenceSummary.__init__,  sU  ø€ Ü‰ÑÔä# F¨N¸FÓCˆÔØ×Ñ Ò&ô &Ð%ä—{‘{“}ˆŒÜ�6Ð-Ô.°6×3JÒ3JÜ�vÐ7Ô8¸V×=ZÒ=ZÐ_e×_pÑ_pÐstÒ_tØ$×/Ñ/‘à$×0Ñ0�ÜŸ9™9 V×%7Ñ%7¸ÓEˆDŒLä# FÐ,@À$ÓGÐÙIZ¤NÐ3DÔ$EÔ`b×`kÑ`kÓ`mˆŒäŸ[™[›]ˆÔÜ�6Ð2Ô3¸×8TÑ8TÐWXÒ8XÜ!#§¡¨F×,HÑ,HÓ!IˆDÔäŸK™K›MˆÔÜ�6Ð1Ô2°v×7RÑ7RÐUVÒ7VÜ "§
¡
¨6×+FÑ+FÓ GˆDÕð 8WÐ2r9   Nr¢   rÀ   r¤   c                 óü  — | j                   dk(  r|dd…df   }�n| j                   dk(  r|dd…df   }�n| j                   dk(  r|j                  d¬«      }ná| j                   d	k(  r½|€At        j                  |d
dd…dd…f   |j                  d   dz
  t        j
                  ¬«      }nX|j                  d«      j                  d«      }|j                  d|j                  «       dz
  z  |j                  d«      fz   «      }|j                  d|«      j                  d«      }n| j                   dk(  rt        ‚| j                  «      }| j                  |«      }| j                  |«      }| j!                  |«      }|S )ak  
        Compute a single vector summary of a sequence hidden states.

        Args:
            hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`):
                The hidden states of the last layer.
            cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*):
                Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token.

        Returns:
            `torch.FloatTensor`: The summary of the sequence hidden states.
        rÇ   Nr{   Úfirstr   Úmeanr   rN   rÀ   .r³   )r=   rz   rÈ   )rÆ   rÚ   r$   Ú	full_liker?   rC   Ú	unsqueezer´   rO   Úsizerµ   r¦   rÏ   rÔ   rÑ   r­   rÕ   )r5   r¢   rÀ   rf   s       r8   ry   zXLNetSequenceSummary.forwardI  sn  € ð ×Ñ Ò&Ø"¢1 b 5Ñ)ŠFØ×Ñ 'Ò)Ø"¢1 a 4Ñ(ŠFØ×Ñ &Ò(Ø"×'Ñ'¨AÐ'Ó.‰FØ×Ñ +Ò-ØÐ Ü!ŸO™OØ! # r¨ rª1 *Ñ-Ø!×'Ñ'¨Ñ+¨aÑ/ÜŸ*™*ô‘	ð &×/Ñ/°Ó3×=Ñ=¸bÓA�	Ø%×,Ñ,¨U°i·m±m³oÈÑ6IÑ-JÈm×N`ÑN`ÐacÓNdÐMfÑ-fÓg�	à"×)Ñ)¨"¨iÓ8×@Ñ@ÀÓD‰FØ×Ñ &Ò(Ü%Ð%à×#Ñ# FÓ+ˆØ—‘˜fÓ%ˆØ—‘ Ó(ˆØ×"Ñ" 6Ó*ˆàˆr9   r�   r¹   r‚   s   @r8   rÄ   rÄ     sQ   ø„ ñð2H˜{õ Hð< VZñ)Ø"×.Ñ.ð)Ø;@×;KÑ;KÈdÑ;Rð)à	×	Ñ	÷)r9   rÄ   c                   óV   ‡ — e Zd ZU eed<   dZ ej                  «       ˆ fd„«       Zˆ xZ	S )ÚXLNetPreTrainedModelr6   Útransformerc           	      ó   •— t         ‰| �  |«       t        |t        «      r—|j                  |j
                  |j                  |j                  |j                  |j                  |j                  |j                  |j                  f	D ].  }t        j                  |d| j                  j                   ¬«       Œ0 yt        |t"        «      r7t        j                  |j$                  d| j                  j                   ¬«       yy)zInitialize the weights.g        )rÚ   ÚstdN)r   Ú_init_weightsrŠ   r   r&   r'   r(   r)   r*   r+   r,   r-   r.   ÚinitÚnormal_r6   Úinitializer_rangeÚ
XLNetModelÚmask_emb)r5   ÚmoduleÚparamr7   s      €r8   rã   z"XLNetPreTrainedModel._init_weightsz  s¼   ø€ ô 	‰Ñ˜fÔ%Ü�fÔ4Ô5à—‘Ø—‘Ø—‘Ø—‘Ø—‘Ø—‘Ø—‘Ø—‘Ø× Ñ ð
ò Q�ô —‘˜U¨°$·+±+×2OÑ2OÖPñQô ˜¤
Ô+Ü�L‰L˜Ÿ™¨s¸¿¹×8UÑ8UÖVð ,r9   )
r}   r~   r   r   Ú__annotations__Úbase_model_prefixr$   Úno_gradrã   r�   r‚   s   @r8   rß   rß   u  s+   ø… àÓØ%Ðà€U‡]�]ƒ_óWó ôWr9   rß   z(
    Output type of [`XLNetModel`].
    )Úcustom_introc                   óÂ   — e Zd ZU dZej
                  ed<   dZeej
                     dz  ed<   dZ	e
ej
                  df   dz  ed<   dZe
ej
                  df   dz  ed<   y)ÚXLNetModelOutputa‚  
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_predict, hidden_size)`):
        Sequence of hidden-states at the last layer of the model.

        `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict`
        corresponds to `sequence_length`.
    mems (`list[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    Úlast_hidden_stateNrn   .r¢   Ú
attentions)r}   r~   r   r§   r$   r%   rë   rn   Úlistr¢   Útuplerò   © r9   r8   rð   rð   �  sm   … ñ
ð ×(Ñ(Ó(Ø+/€Dˆ$ˆu× Ñ Ñ
! DÑ
(Ó/Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>Ø7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ô;r9   rð   z.
    Output type of [`XLNetLMHeadModel`].
    c                   óô   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                     dz  ed<   dZeej                  df   dz  ed<   dZeej                  df   dz  ed<   y)	ÚXLNetLMHeadModelOutputaB  
    loss (`torch.FloatTensor` of shape *(1,)*, *optional*, returned when `labels` is provided):
        Language modeling loss (for next-token prediction).
    logits (`torch.FloatTensor` of shape `(batch_size, num_predict, config.vocab_size)`):
        Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).

        `num_predict` corresponds to `target_mapping.shape[1]`. If `target_mapping` is `None`, then `num_predict`
        corresponds to `sequence_length`.
    mems (`list[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    NÚlossÚlogitsrn   .r¢   rò   ©r}   r~   r   r§   rø   r$   r%   rë   rù   rn   ró   r¢   rô   rò   rõ   r9   r8   r÷   r÷   ¨  s‹   … ñð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø'+€FˆE×Ñ Ñ$Ó+Ø+/€Dˆ$ˆu× Ñ Ñ
! DÑ
(Ó/Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>Ø7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ô;r9   r÷   z<
    Output type of [`XLNetForSequenceClassification`].
    c                   óô   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                     dz  ed<   dZeej                  df   dz  ed<   dZeej                  df   dz  ed<   y)	Ú$XLNetForSequenceClassificationOutputa�  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `label` is provided):
        Classification (or regression if config.num_labels==1) loss.
    logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
        Classification (or regression if config.num_labels==1) scores (before SoftMax).
    mems (`list[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    Nrø   rù   rn   .r¢   rò   rú   rõ   r9   r8   rü   rü   Ä  ó‹   … ñ	ð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø'+€FˆE×Ñ Ñ$Ó+Ø+/€Dˆ$ˆu× Ñ Ñ
! DÑ
(Ó/Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>Ø7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ô;r9   rü   z?
    Output type of [`XLNetForTokenClassificationOutput`].
    c                   óô   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                     dz  ed<   dZeej                  df   dz  ed<   dZeej                  df   dz  ed<   y)	Ú!XLNetForTokenClassificationOutputaO  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Classification loss.
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.num_labels)`):
        Classification scores (before SoftMax).
    mems (`list[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    Nrø   rù   rn   .r¢   rò   rú   rõ   r9   r8   rÿ   rÿ   Ý  rý   r9   rÿ   z4
    Output type of [`XLNetForMultipleChoice`].
    c                   óô   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                     dz  ed<   dZeej                  df   dz  ed<   dZeej                  df   dz  ed<   y)	ÚXLNetForMultipleChoiceOutputa–  
    loss (`torch.FloatTensor` of shape *(1,)*, *optional*, returned when `labels` is provided):
        Classification loss.
    logits (`torch.FloatTensor` of shape `(batch_size, num_choices)`):
        *num_choices* is the second dimension of the input tensors. (see *input_ids* above).

        Classification scores (before SoftMax).
    mems (`list[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    Nrø   rù   rn   .r¢   rò   rú   rõ   r9   r8   r  r  ö  s‹   … ñð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø'+€FˆE×Ñ Ñ$Ó+Ø+/€Dˆ$ˆu× Ñ Ñ
! DÑ
(Ó/Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>Ø7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ô;r9   r  z=
    Output type of [`XLNetForQuestionAnsweringSimple`].
    c                   ó  — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
eej                     dz  ed<   dZeej                  df   dz  ed<   dZeej                  df   dz  ed	<   y)
Ú%XLNetForQuestionAnsweringSimpleOutputaþ  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
        Total span extraction loss is the sum of a Cross-Entropy for the start and end positions.
    start_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length,)`):
        Span-start scores (before SoftMax).
    end_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length,)`):
        Span-end scores (before SoftMax).
    mems (`list[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    Nrø   Ústart_logitsÚ
end_logitsrn   .r¢   rò   )r}   r~   r   r§   rø   r$   r%   rë   r  r  rn   ró   r¢   rô   rò   rõ   r9   r8   r  r    s    … ñð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø-1€L�%×#Ñ# dÑ*Ó1Ø+/€J�×!Ñ! DÑ(Ó/Ø+/€Dˆ$ˆu× Ñ Ñ
! DÑ
(Ó/Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>Ø7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ô;r9   r  z7
    Output type of [`XLNetForQuestionAnswering`].
    c                   ó”  — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZej                  dz  ed<   dZej                  dz  ed<   dZej                  dz  ed<   dZeej                     dz  ed	<   dZeej                  d
f   dz  ed<   dZeej                  d
f   dz  ed<   y)ÚXLNetForQuestionAnsweringOutputax  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned if both `start_positions` and `end_positions` are provided):
        Classification loss as the sum of start token, end token (and is_impossible if provided) classification
        losses.
    start_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
        Log probabilities for the top config.start_n_top start token possibilities (beam-search).
    start_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
        Indices for the top config.start_n_top start token possibilities (beam-search).
    end_top_log_probs (`torch.FloatTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
        Log probabilities for the top `config.start_n_top * config.end_n_top` end token possibilities
        (beam-search).
    end_top_index (`torch.LongTensor` of shape `(batch_size, config.start_n_top * config.end_n_top)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
        Indices for the top `config.start_n_top * config.end_n_top` end token possibilities (beam-search).
    cls_logits (`torch.FloatTensor` of shape `(batch_size,)`, *optional*, returned if `start_positions` or `end_positions` is not provided):
        Log probabilities for the `is_impossible` label of the answers.
    mems (`list[torch.FloatTensor]` of length `config.n_layers`):
        Contains pre-computed hidden-states. Can be used (see `mems` input) to speed up sequential decoding. The
        token ids which have their past given to this model should not be passed as `input_ids` as they have
        already been computed.
    Nrø   Ústart_top_log_probsÚstart_top_indexÚend_top_log_probsÚend_top_indexÚ
cls_logitsrn   .r¢   rò   )r}   r~   r   r§   rø   r$   r%   rë   r  r	  rº   r
  r  r  rn   ró   r¢   rô   rò   rõ   r9   r8   r  r  -  sá   … ñð* &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø48Ð˜×*Ñ*¨TÑ1Ó8Ø/3€O�U×%Ñ%¨Ñ,Ó3Ø26Ð�u×(Ñ(¨4Ñ/Ó6Ø-1€M�5×#Ñ# dÑ*Ó1Ø+/€J�×!Ñ! DÑ(Ó/Ø+/€Dˆ$ˆu× Ñ Ñ
! DÑ
(Ó/Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>Ø7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ô;r9   r  c                   óž  ‡ — e Zd Zˆ fd„Zd„ Zd„ Zd„ Zd„ Zedd„«       Z	dd„Z
e	 	 	 	 	 	 	 	 	 	 	 	 dd	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  dedz  deez  fd„«       Zˆ xZS )rç   c                 óþ  •— t         ‰| �  |«       |j                  | _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        |j                  | _        |j                  | _	        t        j                  |j                  |j                  «      | _        t        j                  t        j                   dd|j                  «      «      | _        t        j$                  t'        |j                  «      D �cg c]  }t)        |«      ‘Œ c}«      | _        t        j,                  |j.                  «      | _        | j1                  «        y c c}w r”   )r   r   Úmem_lenÚ	reuse_lenr   Úsame_lengthÚ	attn_typeÚbi_dataÚ	clamp_lenÚn_layerr   Ú	EmbeddingÚ
vocab_sizeÚword_embeddingr#   r$   r%   rè   Ú
ModuleListÚranger’   Úlayerr2   r3   Ú	post_init)r5   r6   Ú_r7   s      €r8   r   zXLNetModel.__init__V  s   ø€ Ü‰Ñ˜Ô à—~‘~ˆŒØ×)Ñ)ˆŒØ—~‘~ˆŒØ!×-Ñ-ˆÔØ×)Ñ)ˆŒØ—~‘~ˆŒØ×)Ñ)ˆŒØ—~‘~ˆŒä Ÿl™l¨6×+<Ñ+<¸f¿n¹nÓMˆÔÜŸ™¤U×%6Ñ%6°q¸!¸V¿^¹^Ó%LÓMˆŒÜ—]‘]ÄÀfÇnÁnÓ@UÖ#V¸1¤J¨vÕ$6Ò#VÓWˆŒ
Ü—z‘z &§.¡.Ó1ˆŒð 	�‰Õùò	 $Ws   Ä'E:c                 ó   — | j                   S r�   ©r  ©r5   s    r8   Úget_input_embeddingszXLNetModel.get_input_embeddingsj  s   € Ø×"Ñ"Ð"r9   c                 ó   — || _         y r�   r  ©r5   Únew_embeddingss     r8   Úset_input_embeddingszXLNetModel.set_input_embeddingsm  s
   € Ø,ˆÕr9   c                 ó  — t        j                  |||z   f| j                  ¬«      }| j                  rC|dd…d|…f   j	                  d«      }|j                  |dz   «       |dd…d|…fxx   |z  cc<   |S |j                  |dz   «       |S )aD  
        Creates causal attention mask. Float mask where 1.0 indicates masked, 0.0 indicates not-masked.

        Args:
            qlen: Sequence length
            mlen: Mask length

        ::

                  same_length=False: same_length=True: <mlen > < qlen > <mlen > < qlen >
               ^ [0 0 0 0 0 1 1 1 1] [0 0 0 0 0 1 1 1 1]
                 [0 0 0 0 0 0 1 1 1] [1 0 0 0 0 0 1 1 1]
            qlen [0 0 0 0 0 0 0 1 1] [1 1 0 0 0 0 0 1 1]
                 [0 0 0 0 0 0 0 0 1] [1 1 1 0 0 0 0 0 1]
               v [0 0 0 0 0 0 0 0 0] [1 1 1 1 0 0 0 0 0]

        )r<   Nr{   r   )r$   Úonesr<   r  ÚtrilÚtriu_)r5   ÚqlenÚmlenÚmaskÚmask_los        r8   Úcreate_maskzXLNetModel.create_maskp  sŠ   € ô$ �z‰z˜4 ¨¡Ð-°d·k±kÔBˆØ×ÒØš1˜e˜t˜e˜8‘n×)Ñ)¨"Ó-ˆGØ�J‰J�t˜a‘xÔ Ø’�E�T�E�‹N˜gÑ%‹Nð ˆð �J‰J�t˜a‘xÔ àˆr9   c                 ó2  — | j                   �| j                   dkD  r|d | j                    }| j                  �| j                  dk(  rd}n| j                   }|€||d  }|j	                  «       S t        j                  ||gd¬«      |d  }|j	                  «       S )Nr   rN   )r  r  r$   ri   Údetach)r5   Úcurr_outÚprev_memÚcutoffÚnew_mems        r8   Ú	cache_memzXLNetModel.cache_memŒ  s�   € à�>‰>Ð%¨$¯.©.¸1Ò*<ØÐ 0 $§.¡.Ð1ˆHà�<‰<Ð 4§<¡<°1Ò#4ð ‰Fð —l‘l�]ˆFØÐà˜v˜wÐ'ˆGð �~‰~ÓÐô —i‘i ¨8Ð 4¸!Ô<¸V¸WÐEˆGà�~‰~ÓÐr9   Nc                 óô   — t        j                  d| |«      }t        j                  t        j                  |«      t        j                  |«      gd¬«      }|d d …d d d …f   }|�|j                  d|d«      }|S )Nzi,d->idr{   rN   )r$   rP   ri   ÚsinÚcosr´   )Úpos_seqÚinv_freqÚbszÚsinusoid_inpÚpos_embs        r8   Úpositional_embeddingzXLNetModel.positional_embedding¡  sh   € ä—|‘| I¨w¸ÓAˆÜ—)‘)œUŸY™Y |Ó4´e·i±iÀÓ6MÐNÐTVÔWˆØš!˜T¢1˜*Ñ%ˆàˆ?Ø—n‘n R¨¨bÓ1ˆGàˆr9   c                 óà  — t        j                  d| j                  dt         j                  |¬«      j	                  «       }dt        j
                  d|| j                  z  «      z  }| j                  dk(  r|| }}n-| j                  dk(  r|d}}nt        d	| j                  › d
�«      ‚| j                  �r9t        j                  ||dt         j                  |¬«      j	                  «       }	t        j                  | | dt         j                  |¬«      j	                  «       }
| j                  dkD  rN|	j                  | j                   | j                  «      }	|
j                  | j                   | j                  «      }
|�-| j                  |	||dz  «      }| j                  |
||dz  «      }n$| j                  |	|«      }| j                  |
|«      }t        j                  ||gd¬«      }|S t        j                  ||dt         j                  |¬«      j	                  «       }	| j                  dkD  r'|	j                  | j                   | j                  «      }	| j                  |	||«      }|S )Nr   g       @©r=   r<   r   i'  ÚbiÚunir{   zUnknown `attn_type` ú.g      ð¿ç      ð?r   rN   )r$   rB   r   Úint64ÚfloatÚpowr  r    r  r  Úclampr>  ri   )r5   r*  rF   r;  r<   Úfreq_seqr:  ÚbegÚendÚfwd_pos_seqÚbwd_pos_seqÚfwd_pos_embÚbwd_pos_embr=  s                 r8   Úrelative_positional_encodingz'XLNetModel.relative_positional_encoding¬  s  € ä—<‘<  4§<¡<°¼E¿K¹KÐPVÔW×]Ñ]Ó_ˆØ”u—y‘y ¨°D·L±LÑ)@ÓBÑBˆà�>‰>˜TÒ!à˜d˜U�‰CØ�^‰^˜uÒ$à˜R�‰CäÐ3°D·N±NÐ3CÀ1ÐEÓFÐFà�<‹<ÜŸ,™, s¨C°¼U¿[¹[ÐQWÔX×^Ñ^Ó`ˆKÜŸ,™,¨ t¨c¨T°3¼e¿k¹kÐRXÔY×_Ñ_ÓaˆKà�~‰~ Ò!Ø)×/Ñ/°·±°ÀÇÁÓP�Ø)×/Ñ/°·±°ÀÇÁÓP�àˆØ"×7Ñ7¸ÀXÈsÐVWÉxÓX�Ø"×7Ñ7¸ÀXÈsÐVWÉxÓX‘à"×7Ñ7¸ÀXÓN�Ø"×7Ñ7¸ÀXÓN�ä—i‘i ¨kÐ :ÀÔBˆGð ˆô  Ÿ,™, s¨C°¼U¿[¹[ÐQWÔX×^Ñ^Ó`ˆKØ�~‰~ Ò!Ø)×/Ñ/°·±°ÀÇÁÓP�Ø×/Ñ/°¸XÀsÓKˆGàˆr9   Ú	input_idsÚattention_maskrn   Ú	perm_maskro   Útoken_type_idsÚ
input_maskÚinputs_embedsÚuse_memsrZ   Úoutput_hidden_statesÚreturn_dictr¤   c                 óæ  — |
�|
n| j                   j                  }
|�|n| j                   j                  }|�|n| j                   j                  }| j                  r|	�|	n| j                   j
                  }	n|	�|	n| j                   j                  }	|�|�t        d«      ‚|�?|j                  dd«      j                  «       }|j                  d   |j                  d   }}nL|�?|j                  dd«      j                  «       }|j                  d   |j                  d   }}nt        d«      ‚|� |j                  dd«      j                  «       nd}|� |j                  dd«      j                  «       nd}|� |j                  dd«      j                  «       nd}|�!|j                  ddd«      j                  «       nd}|�!|j                  ddd«      j                  «       nd}|�|d   �|d   j                  d   nd}||z   }| j                  }| j                  }| j                  dk(  r | j                  ||«      }|dd…dd…ddf   }n*| j                  dk(  rd}nt        d	| j                  › �«      ‚|�	|�J d
«       ‚	 |€|�d|z
  }|�|�	|d   |z   }n|�|€|d   }n	|€|�|}nd}|�v|dkD  rMt!        j"                  |j                  d   ||g«      j%                  |«      }t!        j&                  ||gd¬«      }|€|dd…dd…dd…df   }n||dd…dd…dd…df   z  }|�|dkD  j%                  |«      }|�Št!        j(                  |«      j%                  |«       }|dkD  r=t!        j&                  t!        j"                  ||g«      j%                  |«      |gd¬«      }||dd…dd…ddf   z   dkD  j%                  |«      }nd}|�|}n| j+                  |«      }| j-                  |«      }|�<| j.                  j1                  |j                  d   |d«      }| j-                  |«      }nd}|�›|dkD  rBt!        j"                  ||gt         j2                  |¬«      }t!        j&                  ||gd¬«      }n|}|dd…df   |ddd…f   k7  j3                  «       }t4        j6                  j9                  |d¬«      j%                  |«      }nd}| j;                  ||||j                  ¬«      }| j-                  |«      }d} |€dgt=        | j>                  «      z  }|
rg nd}!|rg nd}"tA        | j>                  «      D ]l  \  }#}$|	r| | jC                  |||#   «      fz   } |r|"jE                  |�||fn|«        |$||||||||#   ||
¬«	      }%|%dd \  }}|
sŒY|!jE                  |%d   «       Œn |r|"jE                  |�||fn|«       | j-                  |�|n|«      }&|&j                  ddd«      j                  «       }&|	sd} |r'|�tG        d„ |"D «       «      }"ntG        d„ |"D «       «      }"|
r'|�tG        d„ |!D «       «      }!ntG        d„ |!D «       «      }!|stG        d„ |&| |"|!fD «       «      S tI        |&| |"|!¬«      S )áª  
        mems (`list[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
            decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
            they have already been computed.

            `use_mems` has to be set to `True` to make use of `mems`.
        perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
            Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

            - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
            - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

            If not set, each token attends to all the others (full bidirectional attention). Only used during
            pretraining (to define factorization order) or for sequential decoding (generation).
        target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
            Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
            on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
            (generation).
        input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*):
            Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
            real tokens and 1 for padding which is kept for compatibility with the original code base.

            Mask values selected in `[0, 1]`:

            - 1 for tokens that are **masked**,
            - 0 for tokens that are **not masked**.

            You can only uses one of `input_mask` and `attention_mask`.
        use_mems (`bool`, *optional*):
            Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden
            states from previous forward passes to compute attention, which can significantly improve performance for
            sequential decoding tasks.
        NzDYou cannot specify both input_ids and inputs_embeds at the same timer   r   z5You have to specify either input_ids or inputs_embedsr   rB  rA  zUnsupported attention type: z8You can only use one of input_mask (uses 1 for padding) rD  rN   r{   r@  )rÖ   )r;  r<   rõ   )rl   rm   r*   rX   rn   ro   rZ   c              3   ól   K  — | ],  }|D ]%  }|j                  d dd«      j                  «       –— Œ' Œ. y­w©r   r   r   N©ÚpermuteÚ
contiguous)Ú.0Úhsrc   s      r8   ú	<genexpr>z%XLNetModel.forward.<locals>.<genexpr>£  s5   è ø€ Ò%jÈ"ÐgiÒ%jÐbc a§i¡i°°1°aÓ&8×&CÑ&C×&EÐ%jÐ&EÑ%jùs   ‚24c              3   ó^   K  — | ]%  }|j                  d dd«      j                  «       –— Œ' y­wr]  r^  )ra  rb  s     r8   rc  z%XLNetModel.forward.<locals>.<genexpr>¥  s&   è ø€ Ò%_È2 b§j¡j°°A°qÓ&9×&DÑ&D×&FÑ%_ùs   ‚+-c              3   ó@   K  — | ]  }t        d „ |D «       «      –— Œ y­w)c              3   ó`   K  — | ]&  }|j                  d ddd«      j                  «       –— Œ( y­w©r   r	   r   r   Nr^  )ra  Ú
att_streams     r8   rc  z/XLNetModel.forward.<locals>.<genexpr>.<genexpr>«  s*   è ø€ ÒZÈ*˜*×,Ñ,¨Q°°1°aÓ8×CÑC×EÑZùó   ‚,.N)rô   ©ra  Úts     r8   rc  z%XLNetModel.forward.<locals>.<genexpr>ª  s!   è ø€ ò #Ø_`”EÑZÐXYÔZ×Zñ#ùs   ‚c              3   ó`   K  — | ]&  }|j                  d ddd«      j                  «       –— Œ( y­wrg  r^  rj  s     r8   rc  z%XLNetModel.forward.<locals>.<genexpr>®  s(   è ø€ Ò"ZÈ! 1§9¡9¨Q°°1°aÓ#8×#CÑ#C×#EÑ"Zùri  c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr�   rõ   )ra  r(   s     r8   rc  z%XLNetModel.forward.<locals>.<genexpr>±  s   è ø€ Òc˜qÐUVÑUbœÑcùs   ‚Š)rñ   rn   r¢   rò   )%r6   rZ   rX  rY  ÚtrainingÚuse_mems_trainÚuse_mems_evalr    Ú	transposer`  r?   r_  r=   r<   r  r.  r$   ÚzerosÚtori   Úeyer  r3   rè   r´   rC   r   rR   Úone_hotrP  Úlenr  Ú	enumerater5  Úappendrô   rð   )'r5   rQ  rR  rn   rS  ro   rT  rU  rV  rW  rZ   rX  rY  Úkwargsr*  r;  r+  rF   Údtype_floatr<   rY   Ú	data_maskÚ	mems_maskÚnon_tgt_maskÚ
word_emb_krs   Ú
word_emb_qrw   Úmem_padÚcat_idsrX   r=  Únew_memsrò   r¢   ÚiÚlayer_modulerx   rf   s'                                          r8   ry   zXLNetModel.forwardÒ  sò  € ðf 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà�=Š=Ø#+Ð#7‘x¸T¿[¹[×=WÑ=W‰Hà#+Ð#7‘x¸T¿[¹[×=VÑ=VˆHð
 Ð  ]Ð%>ÜÐcÓdÐdØÐ"Ø!×+Ñ+¨A¨qÓ1×<Ñ<Ó>ˆIØ!Ÿ™¨Ñ*¨I¯O©O¸AÑ,>�#‰DØÐ&Ø)×3Ñ3°A°qÓ9×DÑDÓFˆMØ%×+Ñ+¨AÑ.°×0CÑ0CÀAÑ0F�#‰DäÐTÓUÐUàHVÐHb˜×1Ñ1°!°QÓ7×BÑBÔDÐhlˆØ@JÐ@V�Z×)Ñ)¨!¨QÓ/×:Ñ:Ô<Ð\`ˆ
ØHVÐHb˜×1Ñ1°!°QÓ7×BÑBÔDÐhlˆØ?HÐ?T�I×%Ñ% a¨¨AÓ.×9Ñ9Ô;ÐZ^ˆ	ØIWÐIc˜×/Ñ/°°1°aÓ8×CÑCÔEÐimˆà#'Ð#3¸¸Q¹Ð8Kˆt�A‰w�}‰}˜QÒÐQRˆØ�d‰{ˆà—j‘jˆØ—‘ˆð �>‰>˜UÒ"Ø×(Ñ(¨¨tÓ4ˆIØ!¢!¢Q¨¨dÐ"2Ñ3‰IØ�^‰^˜tÒ#Ø‰IäÐ;¸D¿N¹NÐ;KÐLÓMÐMð Ð! ^Ð%;ÐwÐ=wÓwÐ;ØgØÐ .Ð"<Ø˜~Ñ-ˆJØÐ! iÐ&;Ø" 4Ñ(¨9Ñ4‰IØÐ#¨	Ð(9Ø" 4Ñ(‰IØÐ IÐ$9Ø!‰IàˆIàÐ à�aŠxÜ!ŸK™K¨¯©¸Ñ);¸TÀ3Ð(GÓH×KÑKÈIÓV�	Ü!ŸI™I y°)Ð&<À!ÔD�	ØÐ Ø%¢aªªA¨t mÑ4‘	à˜Y¢qª!ªQ° }Ñ5Ñ5�	àÐ Ø" Q™×*Ñ*¨;Ó7ˆIàÐ Ü!ŸI™I d›O×.Ñ.¨yÓ9Ð9ˆLØ�aŠxÜ$Ÿy™y¬%¯+©+°t¸T°lÓ*C×*FÑ*FÀyÓ*QÐS_Ð)`ÐfhÔi�Ø&¨²aº¸DÀ$Ð6FÑ)GÑGÈ1ÑL×PÑPÐQZÓ[‰LàˆLð Ð$Ø&‰Jà×,Ñ,¨YÓ7ˆJØ—<‘< 
Ó+ˆØÐ%ØŸ™×-Ñ-¨n×.BÑ.BÀ1Ñ.EÀsÈBÓOˆJð —|‘| JÓ/‰HàˆHð Ð%à�aŠxÜŸ+™+ t¨S k¼¿¹ÈFÔS�ÜŸ)™) W¨nÐ$=À1ÔE‘à(�ð &¢a¨ gÑ.°'¸$Â¸'Ñ2BÑB×HÑHÓJˆGÜ—m‘m×+Ñ+¨GÀÐ+ÓC×FÑFÀ{ÓS‰GàˆGð ×3Ñ3°D¸$ÀCÐPX×P_ÑP_Ð3Ó`ˆØ—,‘,˜wÓ'ˆàˆØˆ<Ø�6œC §
¡
›OÑ+ˆDá,‘R°$ˆ
Ù2™¸ˆÜ(¨¯©Ó4ò 	.‰OˆAˆ|Ùà# t§~¡~°hÀÀQÁÓ'HÐ&JÑJ�Ù#Ø×$Ñ$¸XÐ=Q h°Ñ%9ÐW_Ô`á"ØØØ(Ø%ØØØ˜!‘WØ-Ø"3ô
ˆGð ")¨¨! ÑˆH�hÚ Ø×!Ñ! '¨!¡*Õ-ð)	.ñ.  Ø× Ñ ¸Ð9M (¨HÑ!5ÐS[Ô\à—‘¨(Ð*>™hÀHÓMˆð —‘  1 aÓ(×3Ñ3Ó5ˆáØˆHáØÐ#Ü %Ñ%jÐP]Ô%jÓ j‘ä %Ñ%_ÐQ^Ô%_Ó _�áØÐ)ä"ñ #Ødnô#ó ‘
ô #Ñ"ZÈzÔ"ZÓZ�
áÜÑc V¨X°}ÀjÐ$QÔcÓcÐcäØ$¨8À=Ð]gô
ð 	
r9   r�   )NN)NNNNNNNNNNNN)r}   r~   r   r   r!  r%  r.  r5  r€   r>  rP  r   r$   ÚTensorÚboolrô   rð   ry   r�   r‚   s   @r8   rç   rç   T  s]  ø„ ôò(#ò-òò8 ð* òó ðó$ðL ð *.Ø.2Ø$(Ø)-Ø.2Ø.2Ø*.Ø-1Ø $Ø)-Ø,0Ø#'ñb
à—<‘< $Ñ&ðb
ð Ÿ™ tÑ+ðb
ð �l‰l˜TÑ!ð	b
ð
 —<‘< $Ñ&ðb
ð Ÿ™ tÑ+ðb
ð Ÿ™ tÑ+ðb
ð —L‘L 4Ñ'ðb
ð —|‘| dÑ*ðb
ð ˜‘+ðb
ð   $™;ðb
ð # T™kðb
ð ˜D‘[ðb
ð 
Ð!Ñ	!òb
ó ôb
r9   rç   zt
    XLNet Model with a language modeling head on top (linear layer with weights tied to the input embeddings).
    c            !       ó0  ‡ — e Zd ZddiZˆ fd„Zd„ Zd„ Z	 dd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 	 dde	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  dedz  dedz  dedz  dedz  dee	j                  z  deez  fd„«       Zed
ee	j                     de	j                  dee	j                     fd„«       Zˆ xZS )ÚXLNetLMHeadModelzlm_loss.weightz!transformer.word_embedding.weightc                 ó
  •— t         ‰| �  |«       |j                  | _        |j                  | _        t	        |«      | _        t        j                  |j                  |j                  d¬«      | _
        | j                  «        y )NTr¾   )r   r   r  r  rç   rà   r   r†   r   r  Úlm_lossr  r4   s     €r8   r   zXLNetLMHeadModel.__init__À  sb   ø€ Ü‰Ñ˜Ô Ø×)Ñ)ˆŒØ!×-Ñ-ˆÔä% fÓ-ˆÔÜ—y‘y §¡°×1BÑ1BÈÔNˆŒð 	�‰Õr9   c                 ó   — | j                   S r�   ©rŠ  r   s    r8   Úget_output_embeddingsz&XLNetLMHeadModel.get_output_embeddingsË  s   € Ø�|‰|Ðr9   c                 ó   — || _         y r�   rŒ  r#  s     r8   Úset_output_embeddingsz&XLNetLMHeadModel.set_output_embeddingsÎ  s	   € Ø%ˆ�r9   Nc                 óî  ‡— |j                   d   }t        j                  |dft        j                  |j                  ¬«      }dŠ|r$t        j
                  |d d …‰ d …f   |gd¬«      }nt        j
                  ||gd¬«      }|j                   d   }t        j                  |||ft        j                  |j                  ¬«      }	d|	d d …d d …df<   t        j                  |d|ft        j                  |j                  ¬«      }
d|
d d …ddf<   ||	|
|dœ}|rt        ˆfd	„|D «       «      |d
<   |j                  dd «       |j                  dd «       |j                  «       D ]  \  }}||vsŒ|||<   Œ |S )Nr   r   r@  r   rN   rD  r{   )rQ  rS  ro   rW  c              3   ó<   •K  — | ]  }|d ‰ …d d …d d …f   –— Œ y ­wr�   rõ   )ra  Ú
layer_pastÚoffsets     €r8   rc  zAXLNetLMHeadModel.prepare_inputs_for_generation.<locals>.<genexpr>û  s"   øè ø€ Ò(fÈ
¨°H°f°W°HºaÂ°NÕ)CÑ(fùs   ƒrn   rR  Ú	use_cache)
r?   r$   rr  rC   r<   ri   rF  rô   ÚpopÚitems)r5   rQ  Úpast_key_valuesrW  Úis_first_iterationry  Úeffective_batch_sizeÚdummy_tokenÚsequence_lengthrS  ro   Úmodel_inputsÚkeyÚvaluer“  s                 @r8   Úprepare_inputs_for_generationz.XLNetLMHeadModel.prepare_inputs_for_generationÑ  s~  ø€ ð  )Ÿ™¨qÑ1ÐÜ—k‘kÐ#7¸Ð";Ä5Ç:Á:ÐV_×VfÑVfÔgˆð
 ˆáÜŸ	™	 9ªQ°°±¨[Ñ#9¸;Ð"GÈQÔO‰IäŸ	™	 9¨kÐ":ÀÔBˆIð $Ÿ/™/¨!Ñ,ˆÜ—K‘KØ! ?°OÐDÌEÏKÉKÐ`i×`pÑ`pô
ˆ	ð "ˆ	’!’Q˜�(Ñô Ÿ™Ø! 1 oÐ6¼e¿k¹kÐR[×RbÑRbô
ˆð $'ˆ’q˜!˜R�xÑ ð #Ø"Ø,Ø ñ	
ˆñ Ü#(Ó(fÐVeÔ(fÓ#fˆL˜Ñ ð 	�
‰
Ð# TÔ*à�
‰
�; Ô%à Ÿ,™,›.ò 	*‰JˆC�Ø˜,Ò&Ø$)�˜SÒ!ð	*ð Ðr9   rQ  rR  rn   rS  ro   rT  rU  rV  ÚlabelsrW  rZ   rX  rY  Úlogits_to_keepr¤   c                 ó  — |�|n| j                   j                  } | j                  |f||||||||
|||dœ|¤Ž}|d   }t        |t        «      rt        | d«      n|}| j                  |dd…|dd…f   «      }d}|	�At        «       } ||j                  d|j                  d«      «      |	j                  d«      «      }|s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  |j                  ¬«      S )aÙ  
        mems (`list[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
            decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
            they have already been computed.

            `use_mems` has to be set to `True` to make use of `mems`.
        perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
            Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

            - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
            - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

            If not set, each token attends to all the others (full bidirectional attention). Only used during
            pretraining (to define factorization order) or for sequential decoding (generation).
        target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
            Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
            on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
            (generation).
        input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*):
            Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
            real tokens and 1 for padding which is kept for compatibility with the original code base.

            Mask values selected in `[0, 1]`:

            - 1 for tokens that are **masked**,
            - 0 for tokens that are **not masked**.

            You can only uses one of `input_mask` and `attention_mask`.
        labels (`torch.LongTensor` of shape `(batch_size, num_predict)`, *optional*):
            Labels for masked language modeling. `num_predict` corresponds to `target_mapping.shape[1]`. If
            `target_mapping` is `None`, then `num_predict` corresponds to `sequence_length`.

            The labels should correspond to the masked input words that should be predicted and depends on
            `target_mapping`. Note in order to perform standard auto-regressive language modeling a *<mask>* token has
            to be added to the `input_ids` (see the `prepare_inputs_for_generation` function and examples below)

            Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored, the loss
            is only computed for labels in `[0, ..., config.vocab_size]`
        use_mems (`bool`, *optional*):
            Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden
            states from previous forward passes to compute attention, which can significantly improve performance for
            sequential decoding tasks.

        Examples:

        ```python
        >>> from transformers import AutoTokenizer, XLNetLMHeadModel
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("xlnet/xlnet-large-cased")
        >>> model = XLNetLMHeadModel.from_pretrained("xlnet/xlnet-large-cased")

        >>> # We show how to setup inputs to predict a next token using a bi-directional context.
        >>> input_ids = torch.tensor(
        ...     tokenizer.encode("Hello, my dog is very <mask>", add_special_tokens=False)
        ... ).unsqueeze(
        ...     0
        ... )  # We will predict the masked token
        >>> perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float)
        >>> perm_mask[:, :, -1] = 1.0  # Previous tokens don't see last token
        >>> target_mapping = torch.zeros(
        ...     (1, 1, input_ids.shape[1]), dtype=torch.float
        ... )  # Shape [1, 1, seq_length] => let's predict one token
        >>> target_mapping[
        ...     0, 0, -1
        ... ] = 1.0  # Our first (and only) prediction will be the last token of the sequence (the masked token)

        >>> outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping)
        >>> next_token_logits = outputs[
        ...     0
        ... ]  # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size]

        >>> # The same way can the XLNetLMHeadModel be used to be trained by standard auto-regressive language modeling.
        >>> input_ids = torch.tensor(
        ...     tokenizer.encode("Hello, my dog is very <mask>", add_special_tokens=False)
        ... ).unsqueeze(
        ...     0
        ... )  # We will predict the masked token
        >>> labels = torch.tensor(tokenizer.encode("cute", add_special_tokens=False)).unsqueeze(0)
        >>> assert labels.shape[0] == 1, "only one word will be predicted"
        >>> perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float)
        >>> perm_mask[
        ...     :, :, -1
        ... ] = 1.0  # Previous tokens don't see last token as is done in standard auto-regressive lm training
        >>> target_mapping = torch.zeros(
        ...     (1, 1, input_ids.shape[1]), dtype=torch.float
        ... )  # Shape [1, 1, seq_length] => let's predict one token
        >>> target_mapping[
        ...     0, 0, -1
        ... ] = 1.0  # Our first (and only) prediction will be the last token of the sequence (the masked token)

        >>> outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping, labels=labels)
        >>> loss = outputs.loss
        >>> next_token_logits = (
        ...     outputs.logits
        ... )  # Logits have shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size]
        ```N©rR  rn   rS  ro   rT  rU  rV  rW  rZ   rX  rY  r   r{   r   ©rø   rù   rn   r¢   rò   )r6   rY  rà   rŠ   ÚintÚslicerŠ  r   ÚviewrÝ   r÷   rn   r¢   rò   )r5   rQ  rR  rn   rS  ro   rT  rU  rV  r   rW  rZ   rX  rY  r¡  ry  Útransformer_outputsr¢   Úslice_indicesrù   rø   Úloss_fctrf   s                          r8   ry   zXLNetLMHeadModel.forward  s8  € ðj &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà.˜d×.Ñ.Øð
à)ØØØ)Ø)Ø!Ø'ØØ/Ø!5Ø#ñ
ð ñ
Ðð  ,¨AÑ.ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐä'Ó)ˆHÙ˜FŸK™K¨¨F¯K©K¸«OÓ<¸f¿k¹kÈ"»oÓNˆDáØ�YÐ!4°Q°RÐ!8Ñ8ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä%ØØØ$×)Ñ)Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r9   Úbeam_idxc           	      ó~   — | D �cg c]-  }|j                  d|j                  |j                  «      «      ‘Œ/ c}S c c}w )zü
        This function is used to re-order the `mems` cache if [`~PreTrainedModel.beam_search`] or
        [`~PreTrainedModel.beam_sample`] is called. This is required to match `mems` with the correct beam_idx at every
        generation step.
        r   )rA   rs  r<   )rn   r«  r’  s      r8   Ú_reorder_cachezXLNetLMHeadModel._reorder_cache¦  s6   € ð ^bÖbÈz�
×'Ñ'¨¨8¯;©;°z×7HÑ7HÓ+IÕJÒbÐbùÒbs   …2:r|   )NNNNNNNNNNNNNr   )r}   r~   r   Ú_tied_weights_keysr   r�  r�  rŸ  r   r$   r…  r†  r¥  rô   r÷   ry   r€   ró   r­  r�   r‚   s   @r8   rˆ  rˆ  ¸  sÊ  ø„ ð +Ð,OÐPÐô	òò&ð RWó5ðn ð *.Ø.2Ø$(Ø)-Ø.2Ø.2Ø*.Ø-1Ø&*Ø $Ø)-Ø,0Ø#'Ø-.ñ[
à—<‘< $Ñ&ð[
ð Ÿ™ tÑ+ð[
ð �l‰l˜TÑ!ð	[
ð
 —<‘< $Ñ&ð[
ð Ÿ™ tÑ+ð[
ð Ÿ™ tÑ+ð[
ð —L‘L 4Ñ'ð[
ð —|‘| dÑ*ð[
ð —‘˜tÑ#ð[
ð ˜‘+ð[
ð   $™;ð[
ð # T™kð[
ð ˜D‘[ð[
ð ˜eŸl™lÑ*ð[
ð" 
Ð'Ñ	'ò#[
ó ð[
ðz ðc˜T %§,¡,Ñ/ð c¸5¿<¹<ð cÈDÐQV×Q]ÑQ]ÑL^ò có ôcr9   rˆ  z‘
    XLNet Model with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g.
    for GLUE tasks.
    c                   óŒ  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dedz  dedz  dedz  dedz  dee	z  fd„«       Z
ˆ xZS )ÚXLNetForSequenceClassificationc                 ó  •— t         ‰| �  |«       |j                  | _        || _        t	        |«      | _        t        |«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r�   )r   r   rÓ   r6   rç   rà   rÄ   Úsequence_summaryr   r†   r   Úlogits_projr  r4   s     €r8   r   z'XLNetForSequenceClassification.__init__·  sg   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØˆŒä% fÓ-ˆÔÜ 4°VÓ <ˆÔÜŸ9™9 V§^¡^°V×5FÑ5FÓGˆÔð 	�‰Õr9   NrQ  rR  rn   rS  ro   rT  rU  rV  r   rW  rZ   rX  rY  r¤   c                 ó`  — |�|n| j                   j                  } | j                  |f||||||||
|||dœ|¤Ž}|d   }| j                  |«      }| j	                  |«      }d}|	��‡| j                   j
                  €�| j                  dk(  rd| j                   _        nl| j                  dkD  rL|	j                  t        j                  k(  s|	j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j
                  dk(  rIt        «       }| j                  dk(  r& ||j                  «       |	j                  «       «      }nŒ |||	«      }n‚| j                   j
                  dk(  r=t        «       } ||j                  d| j                  «      |	j                  d«      «      }n,| j                   j
                  dk(  rt        «       } |||	«      }|s|f|dd z   }|�|f|z   S |S t!        |||j"                  |j$                  |j&                  ¬	«      S )
a.
  
        mems (`list[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
            decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
            they have already been computed.

            `use_mems` has to be set to `True` to make use of `mems`.
        perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
            Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

            - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
            - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

            If not set, each token attends to all the others (full bidirectional attention). Only used during
            pretraining (to define factorization order) or for sequential decoding (generation).
        target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
            Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
            on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
            (generation).
        input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*):
            Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
            real tokens and 1 for padding which is kept for compatibility with the original code base.

            Mask values selected in `[0, 1]`:

            - 1 for tokens that are **masked**,
            - 0 for tokens that are **not masked**.

            You can only uses one of `input_mask` and `attention_mask`.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        use_mems (`bool`, *optional*):
            Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden
            states from previous forward passes to compute attention, which can significantly improve performance for
            sequential decoding tasks.
        Nr£  r   r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr{   r¤  )r6   rY  rà   r²  r³  Úproblem_typerÓ   r=   r$   rC   r¥  r   r¦   r   r§  r   rü   rn   r¢   rò   )r5   rQ  rR  rn   rS  ro   rT  rU  rV  r   rW  rZ   rX  rY  ry  r¨  rf   rù   rø   rª  s                       r8   ry   z&XLNetForSequenceClassification.forwardÃ  s  € ðp &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà.˜d×.Ñ.Øð
à)ØØØ)Ø)Ø!Ø'ØØ/Ø!5Ø#ñ
ð ñ
Ðð % QÑ'ˆà×&Ñ& vÓ.ˆØ×!Ñ! &Ó)ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�áØ�YÐ!4°Q°RÐ!8Ñ8ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä3ØØØ$×)Ñ)Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r9   ©NNNNNNNNNNNNN)r}   r~   r   r   r   r$   r…  r†  rô   rü   ry   r�   r‚   s   @r8   r°  r°  °  sF  ø„ ô
ð ð *.Ø.2Ø$(Ø)-Ø.2Ø.2Ø*.Ø-1Ø&*Ø $Ø)-Ø,0Ø#'ñn
à—<‘< $Ñ&ðn
ð Ÿ™ tÑ+ðn
ð �l‰l˜TÑ!ð	n
ð
 —<‘< $Ñ&ðn
ð Ÿ™ tÑ+ðn
ð Ÿ™ tÑ+ðn
ð —L‘L 4Ñ'ðn
ð —|‘| dÑ*ðn
ð —‘˜tÑ#ðn
ð ˜‘+ðn
ð   $™;ðn
ð # T™kðn
ð ˜D‘[ðn
ð  
Ð5Ñ	5ò!n
ó ôn
r9   r°  c                   óŒ  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dedz  dedz  dedz  dedz  dee	z  fd„«       Z
ˆ xZS )ÚXLNetForTokenClassificationc                 óä   •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r�   )
r   r   rÓ   rç   rà   r   r†   r    Ú
classifierr  r4   s     €r8   r   z$XLNetForTokenClassification.__init__7  óT   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä% fÓ-ˆÔÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr9   NrQ  rR  rn   rS  ro   rT  rU  rV  r   rW  rZ   rX  rY  r¤   c                 ó¢  — |�|n| j                   j                  }| j                  |||||||||
|||¬«      }|d   }| j                  |«      }d}|	�<t	        «       } ||j                  d| j                  «      |	j                  d«      «      }|s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  |j                  ¬«      S )a×
  
        mems (`list[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
            decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
            they have already been computed.

            `use_mems` has to be set to `True` to make use of `mems`.
        perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
            Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

            - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
            - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

            If not set, each token attends to all the others (full bidirectional attention). Only used during
            pretraining (to define factorization order) or for sequential decoding (generation).
        target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
            Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
            on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
            (generation).
        input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*):
            Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
            real tokens and 1 for padding which is kept for compatibility with the original code base.

            Mask values selected in `[0, 1]`:

            - 1 for tokens that are **masked**,
            - 0 for tokens that are **not masked**.

            You can only uses one of `input_mask` and `attention_mask`.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices]`
            where *num_choices* is the size of the second dimension of the input tensors. (see *input_ids* above)
        use_mems (`bool`, *optional*):
            Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden
            states from previous forward passes to compute attention, which can significantly improve performance for
            sequential decoding tasks.emory states to speed up sequential decoding. If set to `True`, the model will use the hidden
            states from previous forward passes to compute attention, which can significantly improve performance for
            sequential decoding tasks.
        Nr£  r   r{   r   r¤  )r6   rY  rà   r½  r   r§  rÓ   rÿ   rn   r¢   rò   )r5   rQ  rR  rn   rS  ro   rT  rU  rV  r   rW  rZ   rX  rY  ry  rx   Úsequence_outputrù   rø   rª  rf   s                        r8   ry   z#XLNetForTokenClassification.forwardA  sû   € ðr &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà×"Ñ"ØØ)ØØØ)Ø)Ø!Ø'ØØ/Ø!5Ø#ð #ó 
ˆð " !™*ˆà—‘ Ó1ˆàˆØÐÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDáØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä0ØØØ—‘Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r9   r¹  )r}   r~   r   r   r   r$   r…  r†  rô   rÿ   ry   r�   r‚   s   @r8   r»  r»  5  sF  ø„ ôð ð *.Ø.2Ø$(Ø)-Ø.2Ø.2Ø*.Ø-1Ø&*Ø $Ø)-Ø,0Ø#'ñ\
à—<‘< $Ñ&ð\
ð Ÿ™ tÑ+ð\
ð �l‰l˜TÑ!ð	\
ð
 —<‘< $Ñ&ð\
ð Ÿ™ tÑ+ð\
ð Ÿ™ tÑ+ð\
ð —L‘L 4Ñ'ð\
ð —|‘| dÑ*ð\
ð —‘˜tÑ#ð\
ð ˜‘+ð\
ð   $™;ð\
ð # T™kð\
ð ˜D‘[ð\
ð  
Ð2Ñ	2ò!\
ó ô\
r9   r»  c                   óŒ  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dedz  dedz  dedz  dedz  dee	z  fd„«       Z
ˆ xZS )ÚXLNetForMultipleChoicec                 óÎ   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        t        j                  |j                  d«      | _	        | j                  «        y r”   )r   r   rç   rà   rÄ   r²  r   r†   r   r³  r  r4   s     €r8   r   zXLNetForMultipleChoice.__init__£  sM   ø€ Ü‰Ñ˜Ô ä% fÓ-ˆÔÜ 4°VÓ <ˆÔÜŸ9™9 V§^¡^°QÓ7ˆÔð 	�‰Õr9   NrQ  rT  rU  rR  rn   rS  ro   rV  r   rW  rZ   rX  rY  r¤   c                 óŠ  — |�|n| j                   j                  }|�|j                  d   n|j                  d   }|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�1|j                  d|j	                  d«      |j	                  d«      «      nd} | j
                  |f||||||||
|||dœ|¤Ž}|d   }| j                  |«      }| j                  |«      }|j                  d|«      }d}|	�"t        «       } |||	j                  d«      «      }|s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  |j                  ¬«      S )a&  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        input_mask (`torch.FloatTensor` of shape `batch_size, num_choices, sequence_length`, *optional*):
            Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
            real tokens and 1 for padding which is kept for compatibility with the original code base.

            Mask values selected in `[0, 1]`:

            - 1 for tokens that are **masked**,
            - 0 for tokens that are **not masked**.

            You can only uses one of `input_mask` and `attention_mask`.
        mems (`list[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
            decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
            they have already been computed.

            `use_mems` has to be set to `True` to make use of `mems`.
        perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
            Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

            - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
            - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

            If not set, each token attends to all the others (full bidirectional attention). Only used during
            pretraining (to define factorization order) or for sequential decoding (generation).
        target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
            Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
            on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
            (generation).
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
        use_mems (`bool`, *optional*):
            Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden
            states from previous forward passes to compute attention, which can significantly improve performance for
            sequential decoding tasks.
        Nr   r{   r³   )rT  rU  rR  rn   rS  ro   rV  rW  rZ   rX  rY  r   r¤  )r6   rY  r?   r§  rÝ   rà   r²  r³  r   r  rn   r¢   rò   )r5   rQ  rT  rU  rR  rn   rS  ro   rV  r   rW  rZ   rX  rY  ry  Únum_choicesÚflat_input_idsÚflat_token_type_idsÚflat_attention_maskÚflat_input_maskÚflat_inputs_embedsr¨  rf   rù   Úreshaped_logitsrø   rª  s                              r8   ry   zXLNetForMultipleChoice.forward­  s  € ðR &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà,5Ð,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆàCLÐCX˜Ÿ™¨¨I¯N©N¸2Ó,>Ô?Ð^bˆØR`ÐRl˜n×1Ñ1°"°n×6IÑ6IÈ"Ó6MÔNÐrvÐØR`ÐRl˜n×1Ñ1°"°n×6IÑ6IÈ"Ó6MÔNÐrvÐØFPÐF\˜*Ÿ/™/¨"¨j¯o©o¸bÓ.AÔBÐbfˆð Ð(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð /˜d×.Ñ.Øð
à.Ø&Ø.ØØØ)Ø,ØØ/Ø!5Ø#ñ
ð ñ
Ðð  % QÑ'ˆà×&Ñ& vÓ.ˆØ×!Ñ! &Ó)ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÐÜ'Ó)ˆHÙ˜O¨V¯[©[¸«_Ó=ˆDáØ%Ð'Ð*=¸a¸bÐ*AÑAˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä+ØØ"Ø$×)Ñ)Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r9   r¹  )r}   r~   r   r   r   r$   r…  r†  rô   r  ry   r�   r‚   s   @r8   rÂ  rÂ  ¡  sF  ø„ ôð ð *.Ø.2Ø*.Ø.2Ø$(Ø)-Ø.2Ø-1Ø&*Ø $Ø)-Ø,0Ø#'ñ{
à—<‘< $Ñ&ð{
ð Ÿ™ tÑ+ð{
ð —L‘L 4Ñ'ð	{
ð
 Ÿ™ tÑ+ð{
ð �l‰l˜TÑ!ð{
ð —<‘< $Ñ&ð{
ð Ÿ™ tÑ+ð{
ð —|‘| dÑ*ð{
ð —‘˜tÑ#ð{
ð ˜‘+ð{
ð   $™;ð{
ð # T™kð{
ð ˜D‘[ð{
ð  
Ð-Ñ	-ò!{
ó ô{
r9   rÂ  zÞ
    XLNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
    layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c            !       ó¬  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  dedz  dee	z  fd„«       Z
ˆ xZS )ÚXLNetForQuestionAnsweringSimplec                 óä   •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r�   )
r   r   rÓ   rç   rà   r   r†   r    Ú
qa_outputsr  r4   s     €r8   r   z(XLNetForQuestionAnsweringSimple.__init__3  r¾  r9   NrQ  rR  rn   rS  ro   rT  rU  rV  r°   Úend_positionsrW  rZ   rX  rY  r¤   c                 óH  — |�|n| j                   j                  } | j                  |f|||||||||||dœ|¤Ž}|d   }| j                  |«      }|j	                  dd¬«      \  }}|j                  d«      j                  «       }|j                  d«      j                  «       }d}|	�·|
�µt        |	j                  «       «      dkD  r|	j                  d«      }	t        |
j                  «       «      dkD  r|
j                  d«      }
|j                  d«      }|	j                  d|«      }	|
j                  d|«      }
t        |¬«      } |||	«      } |||
«      }||z   dz  }|s||f|dd z   }|�|f|z   S |S t        ||||j                  |j                  |j                  ¬	«      S )
r[  Nr£  r   r   r{   rN   )Úignore_indexr   )rø   r  r  rn   r¢   rò   )r6   rY  rà   rÏ  Úsplitr¦   r`  rv  rÝ   rH  r   r  rn   r¢   rò   )r5   rQ  rR  rn   rS  ro   rT  rU  rV  r°   rÐ  rW  rZ   rX  rY  ry  rx   rÀ  rù   r  r  Ú
total_lossÚignored_indexrª  Ú
start_lossÚend_lossrf   s                              r8   ry   z'XLNetForQuestionAnsweringSimple.forward=  sá  € ðj &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà"�$×"Ñ"Øð
à)ØØØ)Ø)Ø!Ø'ØØ/Ø!5Ø#ñ
ð ñ
ˆð  " !™*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJáØ" JÐ/°'¸!¸"°+Ñ=ˆFØ/9Ð/E�Z�M FÑ*ÐQÈ6ÐQä4ØØ%Ø!Ø—‘Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r9   )NNNNNNNNNNNNNN)r}   r~   r   r   r   r$   r…  r†  rô   r  ry   r�   r‚   s   @r8   rÍ  rÍ  ,  s]  ø„ ôð ð *.Ø.2Ø$(Ø)-Ø.2Ø.2Ø*.Ø-1Ø/3Ø-1Ø $Ø)-Ø,0Ø#'ñi
à—<‘< $Ñ&ði
ð Ÿ™ tÑ+ði
ð �l‰l˜TÑ!ð	i
ð
 —<‘< $Ñ&ði
ð Ÿ™ tÑ+ði
ð Ÿ™ tÑ+ði
ð —L‘L 4Ñ'ði
ð —|‘| dÑ*ði
ð Ÿ™¨Ñ,ði
ð —|‘| dÑ*ði
ð ˜‘+ði
ð   $™;ði
ð # T™kði
ð ˜D‘[ði
ð" 
Ð6Ñ	6ò#i
ó ôi
r9   rÍ  c            '       ó  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  dedz  dee	z  f$d„«       Z
ˆ xZS )ÚXLNetForQuestionAnsweringc                 ó  •— t         ‰| �  |«       |j                  | _        |j                  | _        t	        |«      | _        t        |«      | _        t        |«      | _	        t        |«      | _        | j                  «        y r�   )r   r   Ústart_n_topÚ	end_n_toprç   rà   rž   r  r©   r  r¼   Úanswer_classr  r4   s     €r8   r   z"XLNetForQuestionAnswering.__init__¬  sj   ø€ Ü‰Ñ˜Ô Ø!×-Ñ-ˆÔØ×)Ñ)ˆŒä% fÓ-ˆÔÜ2°6Ó:ˆÔÜ.¨vÓ6ˆŒÜ2°6Ó:ˆÔð 	�‰Õr9   NrQ  rR  rn   rS  ro   rT  rU  rV  r°   rÐ  Úis_impossiblerÀ   r£   rW  rZ   rX  rY  r¤   c                 ó~  — |�|n| j                   j                  } | j                  |f|||||||||||dœ|¤Ž}|d   }| j                  ||¬«      }|dd }|	�â|
�à|	|
||fD ]*  }|€Œ|j	                  «       dkD  sŒ|j                  d«       Œ, | j                  ||	|¬«      }t        «       } |||	«      } |||
«      }||z   dz  }|�;|�9| j                  ||	|¬	«      }t        j                  «       } |||«      }||d
z  z  }|s	|f|dd z   S t        ||j                  |j                  |j                  ¬«      S |j                  «       \  } }!}"t        j                   j#                  |d¬«      }#t%        j&                  |#| j(                  d¬«      \  }$}%|%j+                  d«      j-                  dd|"«      }&t%        j.                  |d|&«      }'|'j+                  d«      j-                  d|!dd«      }'|j+                  d«      j1                  |'«      }(|�|j+                  d«      nd}| j                  |(|'|¬«      }t        j                   j#                  |d¬«      })t%        j&                  |)| j2                  d¬«      \  }*}+|*j5                  d| j(                  | j2                  z  «      }*|+j5                  d| j(                  | j2                  z  «      }+t%        j6                  d||#«      }'| j                  ||'|¬«      }|s|$|%|*|+|f}||dd z   S t        |$|%|*|+||j                  |j                  |j                  ¬«      S )aß  
        mems (`list[torch.FloatTensor]` of length `config.n_layers`):
            Contains pre-computed hidden-states (see `mems` output below) . Can be used to speed up sequential
            decoding. The token ids which have their past given to this model should not be passed as `input_ids` as
            they have already been computed.

            `use_mems` has to be set to `True` to make use of `mems`.
        perm_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length, sequence_length)`, *optional*):
            Mask to indicate the attention pattern for each input token with values selected in `[0, 1]`:

            - if `perm_mask[k, i, j] = 0`, i attend to j in batch k;
            - if `perm_mask[k, i, j] = 1`, i does not attend to j in batch k.

            If not set, each token attends to all the others (full bidirectional attention). Only used during
            pretraining (to define factorization order) or for sequential decoding (generation).
        target_mapping (`torch.FloatTensor` of shape `(batch_size, num_predict, sequence_length)`, *optional*):
            Mask to indicate the output tokens to use. If `target_mapping[k, i, j] = 1`, the i-th predict in batch k is
            on the j-th token. Only used during pretraining for partial prediction or for sequential decoding
            (generation).
        input_mask (`torch.FloatTensor` of shape `batch_size, sequence_length`, *optional*):
            Mask to avoid performing attention on padding token indices. Negative of `attention_mask`, i.e. with 0 for
            real tokens and 1 for padding which is kept for compatibility with the original code base.

            Mask values selected in `[0, 1]`:

            - 1 for tokens that are **masked**,
            - 0 for tokens that are **not masked**.

            You can only uses one of `input_mask` and `attention_mask`.
        is_impossible (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels whether a question has an answer or no answer (SQuAD 2.0)
        cls_index (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the classification token to use as input for computing plausibility of the
            answer.
        p_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Optional mask of tokens which can't be in answers (e.g. [CLS], [PAD], ...). 1.0 means token should be
            masked. 0.0 mean token is not masked.
        use_mems (`bool`, *optional*):
            Whether to use memory states to speed up sequential decoding. If set to `True`, the model will use the hidden
            states from previous forward passes to compute attention, which can significantly improve performance for
            sequential decoding tasks.

        Example:

        ```python
        >>> from transformers import AutoTokenizer, XLNetForQuestionAnswering
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("xlnet/xlnet-base-cased")
        >>> model = XLNetForQuestionAnswering.from_pretrained("xlnet/xlnet-base-cased")

        >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(
        ...     0
        ... )  # Batch size 1
        >>> start_positions = torch.tensor([1])
        >>> end_positions = torch.tensor([3])
        >>> outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions)

        >>> loss = outputs.loss
        ```Nr£  r   )r£   r   r{   )r°   r£   r   )r°   rÀ   r   )rø   rn   r¢   rò   rN   r³   )r¯   r£   z
blh,bl->bh)r¯   rÀ   )r  r	  r
  r  r  rn   r¢   rò   )r6   rY  rà   r  rO   Úsqueeze_r  r   rÝ  r   r   r  rn   r¢   rò   rÝ   rR   rS   r$   ÚtopkrÛ  rÜ   r´   rµ   Ú	expand_asrÜ  r§  rP   ),r5   rQ  rR  rn   rS  ro   rT  rU  rV  r°   rÐ  rÞ  rÀ   r£   rW  rZ   rX  rY  ry  r¨  r¢   r  rx   rE   r  rª  rÖ  r×  rÔ  r  Úloss_fct_clsÚcls_lossr;  r¶   r·   Ústart_log_probsr  r	  Ústart_top_index_expr¯   Úhidden_states_expandedÚend_log_probsr
  r  s,                                               r8   ry   z!XLNetForQuestionAnswering.forward¹  s©  € ðd &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà.˜d×.Ñ.Øð
à)ØØØ)Ø)Ø!Ø'ØØ/Ø!5Ø#ñ
ð ñ
Ðð ,¨AÑ.ˆØ×(Ñ(¨¸vÐ(ÓFˆà% a bÐ)ˆàÐ&¨=Ð+Dà% }°iÀÐOò #�Ø‘= Q§U¡U£W¨q£[Ø—J‘J˜r•Nð#ð
 Ÿ™¨ÈÐ`f˜ÓgˆJä'Ó)ˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJàÐ$¨Ð)Bà!×.Ñ.¨}ÈoÐirÐ.Ós�
Ü!×3Ñ3Ó5�Ù'¨
°MÓB�ð ˜h¨™nÑ,�
áØ"�}Ð':¸1¸2Ð'>Ñ>Ð>ä6Ø#Ø,×1Ñ1Ø"5×"CÑ"CØ2×=Ñ=ô	ð ð +×/Ñ/Ó1‰NˆC��sÜ Ÿm™m×3Ñ3°LÀbÐ3ÓIˆOä38·:±:Ø ×!1Ñ!1°rô4Ñ0Ð ð #2×";Ñ";¸BÓ"?×"FÑ"FÀrÈ2ÈsÓ"SÐÜ Ÿ<™<¨°rÐ;NÓOˆLØ'×1Ñ1°!Ó4×;Ñ;¸BÀÀbÈ"ÓMˆLà%2×%<Ñ%<¸QÓ%?×%IÑ%IØó&Ð"ð .4Ð-?�V×%Ñ% bÔ)ÀTˆFØŸ™Ð)?ÈlÐci˜ÓjˆJÜŸM™M×1Ñ1°*À!Ð1ÓDˆMä/4¯z©zØ˜tŸ~™~°1ô0Ñ,Ð˜}ð !2× 6Ñ 6°r¸4×;KÑ;KÈdÏnÉnÑ;\Ó ]ÐØ)×.Ñ.¨r°4×3CÑ3CÀdÇnÁnÑ3TÓUˆMä Ÿ<™<Ø˜m¨_óˆLð ×*Ñ*Ø¨LÀIð +ó ˆJñ Ø.°ÐARÐTaÐcmÐn�ØÐ!4°Q°RÐ!8Ñ8Ð8ä6Ø(;Ø$3Ø&7Ø"/Ø)Ø,×1Ñ1Ø"5×"CÑ"CØ2×=Ñ=ô	ð 	r9   )NNNNNNNNNNNNNNNNN)r}   r~   r   r   r   r$   r…  r†  rô   r  ry   r�   r‚   s   @r8   rÙ  rÙ  ª  s¢  ø„ ôð ð *.Ø.2Ø$(Ø)-Ø.2Ø.2Ø*.Ø-1Ø/3Ø-1Ø-1Ø)-Ø&*Ø $Ø)-Ø,0Ø#'ñ%uà—<‘< $Ñ&ðuð Ÿ™ tÑ+ðuð �l‰l˜TÑ!ð	uð
 —<‘< $Ñ&ðuð Ÿ™ tÑ+ðuð Ÿ™ tÑ+ðuð —L‘L 4Ñ'ðuð —|‘| dÑ*ðuð Ÿ™¨Ñ,ðuð —|‘| dÑ*ðuð —|‘| dÑ*ðuð —<‘< $Ñ&ðuð —‘˜tÑ#ðuð ˜‘+ðuð    $™;ð!uð" # T™kð#uð$ ˜D‘[ð%uð( 
Ð0Ñ	0ò)uó ôur9   rÙ  )rÂ  rÙ  rÍ  r°  r»  rˆ  rç   rß   )8r§   Úcollections.abcr   Údataclassesr   r$   r   Útorch.nnr   r   r   Ú r
   rä   Úactivationsr   r   Ú
generationr   Úmodeling_utilsr   Úpytorch_utilsr   Úutilsr   r   r   Úconfiguration_xlnetr   Ú
get_loggerr}   ÚloggerÚModuler   r„   r’   rž   r©   r¼   rÄ   rß   rð   r÷   rü   rÿ   r  r  r  rç   rˆ  r°  r»  rÂ  rÍ  rÙ  Ú__all__rõ   r9   r8   ú<module>r÷     s8  ðñõ %Ý !ã Ý ß AÑ Aå &ß 1Ý )Ý -Ý 6ß 9Ñ 9Ý ,ð 
ˆ×	Ñ	˜HÓ	%€ôt˜RŸY™Yô tôn�r—y‘yô ô.-�—‘ô -ôb!˜RŸY™Yô !ôJB˜2Ÿ9™9ô BôL>˜RŸY™Yô >ôD`˜2Ÿ9™9ô `ðF ôW˜?ó Wó ðWñ2 ðôð
 ô<�{ó <ó óð<ñ& ðôð
 ô<˜[ó <ó óð<ñ, ðôð
 ô<¨;ó <ó óð<ñ& ðôð
 ô<¨ó <ó óð<ñ& ðôð
 ô< ;ó <ó óð<ñ* ðôð
 ô<¨Kó <ó óð<ñ, ðôð
 ô< kó <ó óð<ðB ô`
Ð%ó `
ó ð`
ñF ðôô
pcÐ+¨_ó pcóð
pcñf ðôô|
Ð%9ó |
óð|
ð~ ôh
Ð"6ó h
ó ðh
ðV ôG
Ð1ó G
ó ðG
ñT ðôôu
Ð&:ó u
óðu
ðp ôDÐ 4ó Dó ðDòN	�r9   