Ë
    ÜÓ^jÎ†  ã                   óf  — d Z ddlZddlZddlmZ ddlmZmZmZ ddlm	Z
 ddlmZmZ ddlmZ dd	lmZmZmZmZmZmZmZ dd
lmZ ddlmZmZ ddlmZ  ej>                  e «      Z!e G d„ de«      «       Z" G d„ dejF                  «      Z$ G d„ dejF                  «      Z% G d„ dejF                  «      Z& G d„ dejF                  «      Z' G d„ dejF                  «      Z( G d„ dejF                  «      Z) G d„ dejF                  «      Z* G d„ dejF                  «      Z+e G d „ d!e"«      «       Z, G d"„ d#e"«      Z- G d$„ d%ejF                  «      Z. ed&¬'«       G d(„ d)e"«      «       Z/e G d*„ d+e"«      «       Z0e G d,„ d-e"«      «       Z1 G d.„ d/ejF                  «      Z2e G d0„ d1e"«      «       Z3d2„ Z4g d3¢Z5y)4zPyTorch MPNet model.é    N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FNÚgelu)Úcreate_bidirectional_mask)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Úauto_docstringÚloggingé   )ÚMPNetConfigc                   óV   ‡ — e Zd ZU eed<   dZ ej                  «       ˆ fd„«       Zˆ xZ	S )ÚMPNetPreTrainedModelÚconfigÚmpnetc                 óX  •— t         ‰| �  |«       t        |t        «      r t	        j
                  |j                  «       yt        |t        «      rZt	        j                  |j                  t        j                  |j                  j                  d   «      j                  d«      «       yy)zInitialize the weightséÿÿÿÿ©r   r   N)ÚsuperÚ_init_weightsÚ
isinstanceÚMPNetLMHeadÚinitÚzeros_ÚbiasÚMPNetEmbeddingsÚcopy_Úposition_idsÚtorchÚarangeÚshapeÚexpand)ÚselfÚmoduleÚ	__class__s     €ús/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/mpnet/modeling_mpnet.pyr    z"MPNetPreTrainedModel._init_weights0   sv   ø€ ô 	‰Ñ˜fÔ%Ü�fœkÔ*Ü�K‰K˜Ÿ™Õ$Ü˜¤Ô0Ü�J‰J�v×*Ñ*¬E¯L©L¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÕhð 1ó    )
Ú__name__Ú
__module__Ú__qualname__r   Ú__annotations__Úbase_model_prefixr)   Úno_gradr    Ú__classcell__©r/   s   @r0   r   r   +   s+   ø… àÓØÐà€U‡]�]ƒ_óió ôir1   r   c                   ó,   ‡ — e Zd Zˆ fd„Zdd„Zd„ Zˆ xZS )r&   c                 óD  •— t         ‰| �  «        d| _        t        j                  |j
                  |j                  | j                  ¬«      | _        t        j                  |j                  |j                  | j                  ¬«      | _	        t        j                  |j                  |j                  ¬«      | _
        t        j                  |j                  «      | _        | j                  dt!        j"                  |j                  «      j%                  d«      d¬«       y )Nr   )Úpadding_idx©Úepsr(   r   F)Ú
persistent)r   Ú__init__r<   r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferr)   r*   r,   ©r-   r   r/   s     €r0   r@   zMPNetEmbeddings.__init__;   sÔ   ø€ Ü‰ÑÔØˆÔÜ!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_c×_oÑ_oÔpˆÔÜ#%§<¡<Ø×*Ñ*¨F×,>Ñ,>ÈD×L\ÑL\ô$
ˆÔ ô Ÿ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒØ×ÑØœEŸL™L¨×)GÑ)GÓH×OÑOÐPWÓXÐejð 	õ 	
r1   c                 óx  — |€*|�t        || j                  «      }n| j                  |«      }|�|j                  «       }n|j                  «       d d }|d   }|€| j                  d d …d |…f   }|€| j                  |«      }| j                  |«      }||z   }| j                  |«      }| j                  |«      }|S )Nr   r   )	Ú"create_position_ids_from_input_idsr<   Ú&create_position_ids_from_inputs_embedsÚsizer(   rD   rF   rG   rK   )	r-   Ú	input_idsr(   Úinputs_embedsÚkwargsÚinput_shapeÚ
seq_lengthrF   Ú
embeddingss	            r0   ÚforwardzMPNetEmbeddings.forwardI   sÒ   € ØÐØÐ$ÜAÀ)ÈT×M]ÑM]Ó^‘à#×JÑJÈ=ÓY�àÐ Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà  ‘^ˆ
àÐØ×,Ñ,ªQ°°°¨^Ñ<ˆLàÐ Ø ×0Ñ0°Ó;ˆMØ"×6Ñ6°|ÓDÐà"Ð%8Ñ8ˆ
Ø—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr1   c                 ó  — |j                  «       dd }|d   }t        j                  | j                  dz   || j                  z   dz   t        j                  |j
                  ¬«      }|j                  d«      j                  |«      S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr   r   )ÚdtypeÚdevicer   )rQ   r)   r*   r<   Úlongr[   Ú	unsqueezer,   )r-   rS   rU   Úsequence_lengthr(   s        r0   rP   z6MPNetEmbeddings.create_position_ids_from_inputs_embedsc   s€   € ð $×(Ñ(Ó*¨3¨BÐ/ˆØ% a™.ˆä—|‘|Ø×Ñ˜qÑ  /°D×4DÑ4DÑ"DÀqÑ"HÔPU×PZÑPZÐcp×cwÑcwô
ˆð ×%Ñ% aÓ(×/Ñ/°Ó<Ð<r1   )NNN)r2   r3   r4   r@   rX   rP   r8   r9   s   @r0   r&   r&   :   s   ø„ ô
óö4=r1   r&   c                   ó,   ‡ — e Zd Zˆ fd„Z	 	 	 dd„Zˆ xZS )ÚMPNetSelfAttentionc                 ó*  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  |j                  «      | _        t        j                  |j                   «      | _        y )Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú))r   r@   rC   Únum_attention_headsÚhasattrÚ
ValueErrorÚintÚattention_head_sizeÚall_head_sizer   ÚLinearÚqÚkÚvÚorI   Úattention_probs_dropout_probrK   rM   s     €r0   r@   zMPNetSelfAttention.__init__v   s:  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð
 $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä—‘˜6×-Ñ-¨t×/AÑ/AÓBˆŒÜ—‘˜6×-Ñ-¨t×/AÑ/AÓBˆŒÜ—‘˜6×-Ñ-¨t×/AÑ/AÓBˆŒÜ—‘˜6×-Ñ-¨v×/AÑ/AÓBˆŒä—z‘z &×"EÑ"EÓFˆ�r1   c                 ó‚  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
t        j                  ||	j	                  dd«      «      }|t        j                  | j                  «      z  }|�||z  }|�||z   }t        j                  j                  |d¬«      }| j                  |«      }t        j                  ||
«      }|j                  dddd«      j!                  «       }|j#                  «       d d | j$                  fz   } |j                  |Ž }| j'                  |«      }|r||f}|S |f}|S )Nr   r   é   éþÿÿÿ©Údimr   r   )r+   rh   rk   ÚviewÚ	transposerl   rm   r)   ÚmatmulÚmathÚsqrtr   Ú
functionalÚsoftmaxrK   ÚpermuteÚ
contiguousrQ   ri   rn   )r-   Úhidden_statesÚattention_maskÚposition_biasÚoutput_attentionsrT   rU   Úhidden_shaperk   rl   rm   Úattention_scoresÚattention_probsÚcÚnew_c_shapern   Úoutputss                    r0   rX   zMPNetSelfAttention.forward‰   s²  € ð $×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆØ�F‰F�=Ó!×&Ñ& |Ó4×>Ñ>¸qÀ!ÓDˆØ�F‰F�=Ó!×&Ñ& |Ó4×>Ñ>¸qÀ!ÓDˆØ�F‰F�=Ó!×&Ñ& |Ó4×>Ñ>¸qÀ!ÓDˆô !Ÿ<™<¨¨1¯;©;°r¸2Ó+>Ó?ÐØ+¬d¯i©i¸×8PÑ8PÓ.QÑQÐð Ð$Ø Ñ-ÐàÐ%Ø/°.Ñ@Ðô Ÿ-™-×/Ñ/Ð0@ÀbÐ/ÓIˆàŸ,™, Ó7ˆä�L‰L˜¨!Ó,ˆà�I‰I�a˜˜A˜qÓ!×,Ñ,Ó.ˆØ—f‘f“h˜s �m t×'9Ñ'9Ð&;Ñ;ˆØˆA�F‰F�KÐ ˆà�F‰F�1‹Iˆá*;�1�oÐ&ˆØˆð CDÀˆØˆr1   ©NNF©r2   r3   r4   r@   rX   r8   r9   s   @r0   r`   r`   u   s   ø„ ôGð, ØØ÷'r1   r`   c                   ó,   ‡ — e Zd Zˆ fd„Z	 	 	 dd„Zˆ xZS )ÚMPNetAttentionc                 óê   •— t         ‰| �  «        t        |«      | _        t	        j
                  |j                  |j                  ¬«      | _        t	        j                  |j                  «      | _
        y ©Nr=   )r   r@   r`   Úattnr   rG   rC   rH   rI   rJ   rK   rM   s     €r0   r@   zMPNetAttention.__init__´   sN   ø€ Ü‰ÑÔÜ& vÓ.ˆŒ	ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r1   c                 óŽ   — | j                  ||||¬«      }| j                  | j                  |d   «      |z   «      }|f|dd  z   }|S )N)r�   r   r   )rŽ   rG   rK   )	r-   r~   r   r€   r�   rT   Úself_outputsÚattention_outputr‡   s	            r0   rX   zMPNetAttention.forwardº   s_   € ð —y‘yØØØØ/ð	 !ó 
ˆð  Ÿ>™>¨$¯,©,°|ÀA±Ó*GÈ-Ñ*WÓXÐØ#Ð%¨°Q°RÐ(8Ñ8ˆØˆr1   rˆ   r‰   r9   s   @r0   r‹   r‹   ³   s   ø„ ô>ð ØØ÷r1   r‹   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚMPNetIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y ©N)r   r@   r   rj   rC   Úintermediate_sizeÚdenser!   Ú
hidden_actÚstrr	   Úintermediate_act_fnrM   s     €r0   r@   zMPNetIntermediate.__init__Ï   s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$r1   r~   Úreturnc                 óJ   — | j                  |«      }| j                  |«      }|S r•   )r—   rš   )r-   r~   s     r0   rX   zMPNetIntermediate.forward×   s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr1   ©r2   r3   r4   r@   r)   ÚTensorrX   r8   r9   s   @r0   r“   r“   Î   s#   ø„ ô9ð U§\¡\ð °e·l±l÷ r1   r“   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚMPNetOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y r�   )r   r@   r   rj   r–   rC   r—   rG   rH   rI   rJ   rK   rM   s     €r0   r@   zMPNetOutput.__init__ß   s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r1   r~   Úinput_tensorr›   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S r•   )r—   rK   rG   )r-   r~   r¢   s      r0   rX   zMPNetOutput.forwardå   s7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°|Ñ'CÓDˆØÐr1   r�   r9   s   @r0   r    r    Þ   s1   ø„ ô>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ r1   r    c                   ó,   ‡ — e Zd Zˆ fd„Z	 	 	 dd„Zˆ xZS )Ú
MPNetLayerc                 ó‚   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        t        |«      | _        y r•   )r   r@   r‹   Ú	attentionr“   Úintermediater    ÚoutputrM   s     €r0   r@   zMPNetLayer.__init__í   s3   ø€ Ü‰ÑÔÜ'¨Ó/ˆŒÜ-¨fÓ5ˆÔÜ! &Ó)ˆ�r1   c                 ó–   — | j                  ||||¬«      }|d   }|dd  }| j                  |«      }	| j                  |	|«      }
|
f|z   }|S )N)r€   r�   r   r   )r§   r¨   r©   )r-   r~   r   r€   r�   rT   Úself_attention_outputsr‘   r‡   Úintermediate_outputÚlayer_outputs              r0   rX   zMPNetLayer.forwardó   sr   € ð "&§¡ØØØ'Ø/ð	 "0ó "
Ðð 2°!Ñ4ÐØ(¨¨Ð,ˆà"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6Ð8HÓIˆØ�/ GÑ+ˆØˆr1   rˆ   r‰   r9   s   @r0   r¥   r¥   ì   s   ø„ ô*ð ØØ÷r1   r¥   c                   óŒ   ‡ — e Zd Zˆ fd„Z	 	 	 	 ddej
                  dej
                  dz  dededef
d„Zdd	„Ze	dd
„«       Z
ˆ xZS )ÚMPNetEncoderc                 óB  •— t         ‰| �  «        || _        |j                  | _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _
        t        j                  |j                  | j                  «      | _        y c c}w r•   )r   r@   r   rd   Ún_headsr   Ú
ModuleListÚrangeÚnum_hidden_layersr¥   ÚlayerrA   Úrelative_attention_num_bucketsÚrelative_attention_bias)r-   r   Ú_r/   s      €r0   r@   zMPNetEncoder.__init__  ss   ø€ Ü‰ÑÔØˆŒØ×1Ñ1ˆŒÜ—]‘]ÄÀf×F^ÑF^Ó@_Ö#`¸1¤J¨vÕ$6Ò#`ÓaˆŒ
Ü')§|¡|°F×4YÑ4YÐ[_×[gÑ[gÓ'hˆÕ$ùò $as   ÁBNr~   r   r�   Úoutput_hidden_statesÚreturn_dictc                 ó  — | j                  |«      }|rdnd }|rdnd }	t        | j                  «      D ]+  \  }
}|r||fz   } ||||fd|i|¤Ž}|d   }|sŒ#|	|d   fz   }	Œ- |r||fz   }|st        d„ |||	fD «       «      S t	        |||	¬«      S )N© r�   r   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr•   r¼   )Ú.0rm   s     r0   ú	<genexpr>z'MPNetEncoder.forward.<locals>.<genexpr>3  s   è ø€ Òh˜qÐZ[ÑZgœÑhùs   ‚Š)Úlast_hidden_stater~   Ú
attentions)Úcompute_position_biasÚ	enumeraterµ   Útupler   )r-   r~   r   r�   r¹   rº   rT   r€   Úall_hidden_statesÚall_attentionsÚiÚlayer_moduleÚlayer_outputss                r0   rX   zMPNetEncoder.forward  sã   € ð ×2Ñ2°=ÓAˆÙ"6™B¸DÐÙ0™°dˆÜ(¨¯©Ó4ò 	F‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!á(ØØØñð #4ð	ð
 ñˆMð *¨!Ñ,ˆMâ Ø!/°=ÀÑ3CÐ2EÑ!E‘ð	Fñ"  Ø 1°]Ð4DÑ DÐáÜÑh ]Ð4EÀ~Ð$VÔhÓhÐhÜØ+Ø+Ø%ô
ð 	
r1   c                 ód  — |j                  d«      |j                  d«      |j                  d«      }}}|�|d d …d d …d f   }|d d …d d d …f   }nXt        j                  |t        j                  ¬«      d d …d f   }t        j                  |t        j                  ¬«      d d d …f   }||z
  }	| j	                  |	|¬«      }
|
j                  |j                  «      }
| j                  |
«      }|j                  g d¢«      j                  d«      }|j                  |d||f«      j                  «       }|S )Nr   r   )rZ   )Únum_buckets)rq   r   r   r   )rQ   r)   r*   r\   Úrelative_position_bucketÚtor[   r·   r|   r]   r,   r}   )r-   Úxr(   rË   ÚbszÚqlenÚklenÚcontext_positionÚmemory_positionÚrelative_positionÚ	rp_bucketÚvaluess               r0   rÂ   z"MPNetEncoder.compute_position_bias:  s  € ØŸ&™& ›) Q§V¡V¨A£Y°·±°q³	�4ˆTˆØÐ#Ø+ªAªq°$¨JÑ7ÐØ*ª1¨d²A¨:Ñ6‰Oä$Ÿ|™|¨D¼¿
¹
ÔCÂAÀtÀGÑLÐÜ#Ÿl™l¨4´u·z±zÔBÀ4ÊÀ7ÑKˆOà+Ð.>Ñ>Ðà×1Ñ1Ð2CÐQ\Ð1Ó]ˆ	Ø—L‘L §¡Ó*ˆ	Ø×-Ñ-¨iÓ8ˆØ—‘¢	Ó*×4Ñ4°QÓ7ˆØ—‘  R¨¨tÐ4Ó5×@Ñ@ÓBˆØˆr1   c                 ó   — d}|  }|dz  }||dk  j                  t        j                  «      |z  z  }t        j                  |«      }|dz  }||k  }|t        j                  |j                  «       |z  «      t        j                  ||z  «      z  ||z
  z  j                  t        j                  «      z   }t        j                  |t        j                  ||dz
  «      «      }|t        j                  |||«      z  }|S )Nr   rq   r   )
rÍ   r)   r\   ÚabsÚlogÚfloatrx   ÚminÚ	full_likeÚwhere)rÔ   rË   Úmax_distanceÚretÚnÚ	max_exactÚis_smallÚval_if_larges           r0   rÌ   z%MPNetEncoder.relative_position_bucketL  sä   € àˆØÐˆà˜ÑˆØ��A‘�z‰zœ%Ÿ*™*Ó%¨Ñ3Ñ3ˆÜ�I‰I�a‹Lˆà 1Ñ$ˆ	Ø�y‘=ˆà Ü�I‰I�a—g‘g“i )Ñ+Ó,¬t¯x©x¸ÀyÑ8PÓ/QÑQÐU`ÐclÑUlÑmß
‰"ŒU�Z‰Z‹.ñˆô —y‘y ¬u¯©¸|È[Ð[\É_Ó/]Ó^ˆØŒu�{‰{˜8 Q¨Ó5Ñ5ˆØˆ
r1   )NFFF)Né    )rä   é€   )r2   r3   r4   r@   r)   rž   ÚboolrX   rÂ   ÚstaticmethodrÌ   r8   r9   s   @r0   r¯   r¯   
  ss   ø„ ôið /3Ø"'Ø%*Ø!ñ&
à—|‘|ð&
ð Ÿ™ tÑ+ð&
ð  ð	&
ð
 #ð&
ð ó&
óPð$ òó ôr1   r¯   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚMPNetPoolerc                 ó²   •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  «       | _        y r•   )r   r@   r   rj   rC   r—   ÚTanhÚ
activationrM   s     €r0   r@   zMPNetPooler.__init__c  s9   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'™'›)ˆ�r1   r~   r›   c                 ó\   — |d d …df   }| j                  |«      }| j                  |«      }|S ©Nr   )r—   rì   )r-   r~   Úfirst_token_tensorÚpooled_outputs       r0   rX   zMPNetPooler.forwardh  s6   € ð +ª1¨a¨4Ñ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐr1   r�   r9   s   @r0   ré   ré   b  s#   ø„ ô$ð
 U§\¡\ð °e·l±l÷ r1   ré   c                   ó  ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Ze	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e
dz  d
e
dz  de
dz  deej                     ez  fd„«       Zˆ xZS )Ú
MPNetModelc                 óº   •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        |rt        |«      nd| _        | j                  «        y)zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        N)
r   r@   r   r&   rW   r¯   Úencoderré   ÚpoolerÚ	post_init)r-   r   Úadd_pooling_layerr/   s      €r0   r@   zMPNetModel.__init__s  sM   ø€ ô
 	‰Ñ˜Ô ØˆŒä)¨&Ó1ˆŒÜ# FÓ+ˆŒÙ->”k &Ô)ÀDˆŒð 	�‰Õr1   c                 ó.   — | j                   j                  S r•   ©rW   rD   ©r-   s    r0   Úget_input_embeddingszMPNetModel.get_input_embeddings‚  s   € Ø�‰×.Ñ.Ð.r1   c                 ó&   — || j                   _        y r•   rù   )r-   Úvalues     r0   Úset_input_embeddingszMPNetModel.set_input_embeddings…  s   € Ø*/ˆ�‰Õ'r1   NrR   r   r(   rS   r�   r¹   rº   r›   c                 ó  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|�|�t	        d«      ‚|€|€t	        d«      ‚| j                  |||¬«      }	t        | j                   |	|¬«      }| j                  |	||||¬«      }
|
d   }| j                  �| j                  |«      nd }|s
||f|
dd  z   S t        |||
j                  |
j                  ¬«      S )	NzDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embeds)rR   r(   rS   )r   rS   r   )r   r�   r¹   rº   r   r   )rÀ   Úpooler_outputr~   rÁ   )r   r�   r¹   rº   rf   rW   r   rô   rõ   r   r~   rÁ   )r-   rR   r   r(   rS   r�   r¹   rº   rT   Úembedding_outputÚencoder_outputsÚsequence_outputrð   s                r0   rX   zMPNetModel.forwardˆ  s6  € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐ  ]Ð%>ÜÐcÓdÐdØÐ =Ð#8ÜÐTÓUÐUàŸ?™?°YÈ\Ðiv˜?ÓwÐä2Ø—;‘;Ø*Ø)ô
ˆð Ÿ,™,ØØ)Ø/Ø!5Ø#ð 'ó 
ˆð *¨!Ñ,ˆØ8<¿¹Ð8O˜Ÿ™ OÔ4ÐUYˆáØ# ]Ð3°oÀaÀbÐ6IÑIÐIä)Ø-Ø'Ø)×7Ñ7Ø&×1Ñ1ô	
ð 	
r1   )T)NNNNNNN)r2   r3   r4   r@   rû   rþ   r   r)   Ú
LongTensorÚFloatTensorræ   rÄ   rž   r   rX   r8   r9   s   @r0   rò   rò   q  sÒ   ø„ õò/ò0ð ð .2Ø37Ø04Ø26Ø)-Ø,0Ø#'ñ0
à×#Ñ# dÑ*ð0
ð ×)Ñ)¨DÑ0ð0
ð ×&Ñ&¨Ñ-ð	0
ð
 ×(Ñ(¨4Ñ/ð0
ð   $™;ð0
ð # T™kð0
ð ˜D‘[ð0
ð 
ˆu�|‰|Ñ	Ð9Ñ	9ò0
ó ô0
r1   rò   c                   ó0  ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 ddej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  deej                     ez  fd„«       Zˆ xZS )ÚMPNetForMaskedLMz'mpnet.embeddings.word_embeddings.weightzlm_head.bias)zlm_head.decoder.weightzlm_head.decoder.biasc                 óˆ   •— t         ‰| �  |«       t        |d¬«      | _        t	        |«      | _        | j                  «        y ©NF)r÷   )r   r@   rò   r   r"   Úlm_headrö   rM   s     €r0   r@   zMPNetForMaskedLM.__init__Â  s6   ø€ Ü‰Ñ˜Ô ä ¸%Ô@ˆŒ
Ü" 6Ó*ˆŒð 	�‰Õr1   c                 ó.   — | j                   j                  S r•   )r
  Údecoderrú   s    r0   Úget_output_embeddingsz&MPNetForMaskedLM.get_output_embeddingsË  s   € Ø�|‰|×#Ñ#Ð#r1   c                 ó\   — || j                   _        |j                  | j                   _        y r•   )r
  r  r%   )r-   Únew_embeddingss     r0   Úset_output_embeddingsz&MPNetForMaskedLM.set_output_embeddingsÎ  s    € Ø-ˆ�‰ÔØ*×/Ñ/ˆ�‰Õr1   NrR   r   r(   rS   Úlabelsr�   r¹   rº   r›   c	           	      ó–  — |�|n| j                   j                  }| j                  |||||||¬«      }
|
d   }| j                  |«      }d}|�Ft	        «       } ||j                  d| j                   j                  «      |j                  d«      «      }|s|f|
dd z   }|�|f|z   S |S t        |||
j                  |
j                  ¬«      S )a¢  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        N©r   r(   rS   r�   r¹   rº   r   r   rq   ©ÚlossÚlogitsr~   rÁ   )
r   rº   r   r
  r   ru   rB   r   r~   rÁ   )r-   rR   r   r(   rS   r  r�   r¹   rº   rT   r‡   r  Úprediction_scoresÚmasked_lm_lossÚloss_fctr©   s                   r0   rX   zMPNetForMaskedLM.forwardÒ  sô   € ð& &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—*‘*ØØ)Ø%Ø'Ø/Ø!5Ø#ð ó 
ˆð " !™*ˆØ ŸL™L¨Ó9ÐàˆØÐÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNáØ'Ð)¨G°A°B¨KÑ7ˆFØ3AÐ3M�^Ð%¨Ñ.ÐYÐSYÐYäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r1   ©NNNNNNNN)r2   r3   r4   Ú_tied_weights_keysr@   r  r  r   r)   r  r  ræ   rÄ   rž   r   rX   r8   r9   s   @r0   r  r  ¼  s÷   ø„ à"KØ .ñÐô
ò$ò0ð ð .2Ø37Ø04Ø26Ø*.Ø)-Ø,0Ø#'ñ/
à×#Ñ# dÑ*ð/
ð ×)Ñ)¨DÑ0ð/
ð ×&Ñ&¨Ñ-ð	/
ð
 ×(Ñ(¨4Ñ/ð/
ð × Ñ  4Ñ'ð/
ð   $™;ð/
ð # T™kð/
ð ˜D‘[ð/
ð 
ˆu�|‰|Ñ	˜~Ñ	-ò/
ó ô/
r1   r  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )r"   z5MPNet Head for masked and permuted language modeling.c                 ó°  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  d¬«      | _
        t        j                  t        j                  |j                  «      «      | _        y )Nr=   T)r%   )r   r@   r   rj   rC   r—   rG   rH   Ú
layer_normrB   r  Ú	Parameterr)   Úzerosr%   rM   s     €r0   r@   zMPNetLMHead.__init__  s‰   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒä—y‘y ×!3Ñ!3°V×5FÑ5FÈTÔRˆŒÜ—L‘L¤§¡¨V×->Ñ->Ó!?Ó@ˆ�	r1   c                 ó‚   — | j                  |«      }t        |«      }| j                  |«      }| j                  |«      }|S r•   )r—   r
   r  r  ©r-   ÚfeaturesrT   rÎ   s       r0   rX   zMPNetLMHead.forward  s;   € Ø�J‰J�xÓ ˆÜ�‹GˆØ�O‰O˜AÓˆð �L‰L˜‹Oˆàˆr1   ©r2   r3   r4   Ú__doc__r@   rX   r8   r9   s   @r0   r"   r"     s   ø„ Ù?ôAör1   r"   z�
    MPNet Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    )Úcustom_introc                   ó  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  d	edz  d
edz  de	ej                     ez  fd„«       Zˆ xZS )ÚMPNetForSequenceClassificationc                 óª   •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        t        |«      | _        | j                  «        y r	  )r   r@   Ú
num_labelsrò   r   ÚMPNetClassificationHeadÚ
classifierrö   rM   s     €r0   r@   z'MPNetForSequenceClassification.__init__"  sC   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ ¸%Ô@ˆŒ
Ü1°&Ó9ˆŒð 	�‰Õr1   NrR   r   r(   rS   r  r�   r¹   rº   r›   c	           	      ó  — |�|n| j                   j                  }| j                  |||||||¬«      }
|
d   }| j                  |«      }d}|��‡| j                   j                  €�| j
                  dk(  rd| j                   _        nl| j
                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j                  dk(  rIt        «       }| j
                  dk(  r& ||j                  «       |j                  «       «      }nŒ |||«      }n‚| j                   j                  dk(  r=t        «       } ||j                  d| j
                  «      |j                  d«      «      }n,| j                   j                  dk(  rt        «       } |||«      }|s|f|
d	d z   }|�|f|z   S |S t        |||
j                   |
j"                  ¬
«      S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Nr  r   r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr   rq   r  )r   rº   r   r,  Úproblem_typer*  rZ   r)   r\   rg   r   Úsqueezer   ru   r   r   r~   rÁ   ©r-   rR   r   r(   rS   r  r�   r¹   rº   rT   r‡   r  r  r  r  r©   s                   r0   rX   z&MPNetForSequenceClassification.forward,  sÔ  € ð( &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—*‘*ØØ)Ø%Ø'Ø/Ø!5Ø#ð ó 
ˆð " !™*ˆØ—‘ Ó1ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ÙØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r1   r  )r2   r3   r4   r@   r   r)   r  r  ræ   rÄ   rž   r   rX   r8   r9   s   @r0   r(  r(    së   ø„ ôð ð .2Ø37Ø04Ø26Ø*.Ø)-Ø,0Ø#'ñ@
à×#Ñ# dÑ*ð@
ð ×)Ñ)¨DÑ0ð@
ð ×&Ñ&¨Ñ-ð	@
ð
 ×(Ñ(¨4Ñ/ð@
ð × Ñ  4Ñ'ð@
ð   $™;ð@
ð # T™kð@
ð ˜D‘[ð@
ð 
ˆu�|‰|Ñ	Ð7Ñ	7ò@
ó ô@
r1   r(  c                   ó  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  d	edz  d
edz  de	ej                     ez  fd„«       Zˆ xZS )ÚMPNetForMultipleChoicec                 óö   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  «      | _        t	        j                  |j                  d«      | _
        | j                  «        y )Nr   )r   r@   rò   r   r   rI   rJ   rK   rj   rC   r,  rö   rM   s     €r0   r@   zMPNetForMultipleChoice.__init__r  sV   ø€ Ü‰Ñ˜Ô ä Ó'ˆŒ
Ü—z‘z &×"<Ñ"<Ó=ˆŒÜŸ)™) F×$6Ñ$6¸Ó:ˆŒð 	�‰Õr1   NrR   r   r(   rS   r  r�   r¹   rº   r›   c	           	      óþ  — |�|n| j                   j                  }|�|j                  d   n|j                  d   }
|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�1|j                  d|j	                  d«      |j	                  d«      «      nd}| j                  |||||||¬«      }|d   }| j                  |«      }| j                  |«      }|j                  d|
«      }d}|�t        «       } |||«      }|s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )a¸  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   r   rr   )r(   r   rS   r�   r¹   rº   rq   r  )r   rº   r+   ru   rQ   r   rK   r,  r   r   r~   rÁ   )r-   rR   r   r(   rS   r  r�   r¹   rº   rT   Únum_choicesÚflat_input_idsÚflat_position_idsÚflat_attention_maskÚflat_inputs_embedsr‡   rð   r  Úreshaped_logitsr  r  r©   s                         r0   rX   zMPNetForMultipleChoice.forward|  sµ  € ðH &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆØ,5Ð,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆàCLÐCX˜Ÿ™¨¨I¯N©N¸2Ó,>Ô?Ð^bˆØLXÐLd˜L×-Ñ-¨b°,×2CÑ2CÀBÓ2GÔHÐjnÐØR`ÐRl˜n×1Ñ1°"°n×6IÑ6IÈ"Ó6MÔNÐrvÐð Ð(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —*‘*ØØ*Ø.Ø,Ø/Ø!5Ø#ð ó 
ˆð   ™
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÐÜ'Ó)ˆHÙ˜O¨VÓ4ˆDáØ%Ð'¨'°!°"¨+Ñ5ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r1   r  )r2   r3   r4   r@   r   r)   r  r  ræ   rÄ   rž   r   rX   r8   r9   s   @r0   r5  r5  p  së   ø„ ôð ð .2Ø37Ø04Ø26Ø*.Ø)-Ø,0Ø#'ñL
à×#Ñ# dÑ*ðL
ð ×)Ñ)¨DÑ0ðL
ð ×&Ñ&¨Ñ-ð	L
ð
 ×(Ñ(¨4Ñ/ðL
ð × Ñ  4Ñ'ðL
ð   $™;ðL
ð # T™kðL
ð ˜D‘[ðL
ð 
ˆu�|‰|Ñ	Ð8Ñ	8òL
ó ôL
r1   r5  c                   ó  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  d	edz  d
edz  de	ej                     ez  fd„«       Zˆ xZS )ÚMPNetForTokenClassificationc                 ó0  •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r	  )r   r@   r*  rò   r   r   rI   rJ   rK   rj   rC   r,  rö   rM   s     €r0   r@   z$MPNetForTokenClassification.__init__Î  sk   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä ¸%Ô@ˆŒ
Ü—z‘z &×"<Ñ"<Ó=ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr1   NrR   r   r(   rS   r  r�   r¹   rº   r›   c	           	      ó¤  — |�|n| j                   j                  }| j                  |||||||¬«      }
|
d   }| j                  |«      }| j	                  |«      }d}|�<t        «       } ||j                  d| j                  «      |j                  d«      «      }|s|f|
dd z   }|�|f|z   S |S t        |||
j                  |
j                  ¬«      S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Nr  r   r   rq   r  )r   rº   r   rK   r,  r   ru   r*  r   r~   rÁ   r3  s                   r0   rX   z#MPNetForTokenClassification.forwardÙ  sð   € ð$ &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—*‘*ØØ)Ø%Ø'Ø/Ø!5Ø#ð ó 
ˆð " !™*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÐÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDáØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r1   r  )r2   r3   r4   r@   r   r)   r  r  ræ   rÄ   rž   r   rX   r8   r9   s   @r0   r?  r?  Ì  sà   ø„ ô	ð ð .2Ø37Ø04Ø26Ø*.Ø)-Ø,0Ø#'ñ0
à×#Ñ# dÑ*ð0
ð ×)Ñ)¨DÑ0ð0
ð ×&Ñ&¨Ñ-ð	0
ð
 ×(Ñ(¨4Ñ/ð0
ð × Ñ  4Ñ'ð0
ð   $™;ð0
ð # T™kð0
ð ˜D‘[ð0
ð 
ˆu�|‰|Ñ	Ð4Ñ	4ò0
ó ô0
r1   r?  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )r+  z-Head for sentence-level classification tasks.c                 ó&  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _
        y r•   )r   r@   r   rj   rC   r—   rI   rJ   rK   r*  Úout_projrM   s     €r0   r@   z MPNetClassificationHead.__init__  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü—z‘z &×"<Ñ"<Ó=ˆŒÜŸ	™	 &×"4Ñ"4°f×6GÑ6GÓHˆ�r1   c                 óÐ   — |d d …dd d …f   }| j                  |«      }| j                  |«      }t        j                  |«      }| j                  |«      }| j	                  |«      }|S rî   )rK   r—   r)   ÚtanhrD  r"  s       r0   rX   zMPNetClassificationHead.forward  sY   € Ø’Q˜š1�WÑˆØ�L‰L˜‹OˆØ�J‰J�q‹MˆÜ�J‰J�q‹MˆØ�L‰L˜‹OˆØ�M‰M˜!ÓˆØˆr1   r$  r9   s   @r0   r+  r+    s   ø„ Ù7ôIör1   r+  c                   ó:  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dedz  de	ej                     ez  fd„«       Zˆ xZS )ÚMPNetForQuestionAnsweringc                 óè   •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r	  )
r   r@   r*  rò   r   r   rj   rC   Ú
qa_outputsrö   rM   s     €r0   r@   z"MPNetForQuestionAnswering.__init__"  sU   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ ¸%Ô@ˆŒ
ÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr1   NrR   r   r(   rS   Ústart_positionsÚend_positionsr�   r¹   rº   r›   c
           	      ó$  — |	�|	n| j                   j                  }	| j                  |||||||	¬«      }|d   }| j                  |«      }|j	                  dd¬«      \  }}|j                  d«      j                  «       }|j                  d«      j                  «       }d }|�·|�µt        |j                  «       «      dkD  r|j                  d«      }t        |j                  «       «      dkD  r|j                  d«      }|j                  d«      }|j                  d|«      }|j                  d|«      }t        |¬«      } |||«      } |||«      }||z   dz  }|	s||f|dd  z   }|�|f|z   S |S t        ||||j                  |j                  ¬«      S )	Nr  r   r   r   rs   )Úignore_indexrq   )r  Ústart_logitsÚ
end_logitsr~   rÁ   )r   rº   r   rJ  Úsplitr2  r}   ÚlenrQ   Úclampr   r   r~   rÁ   )r-   rR   r   r(   rS   rK  rL  r�   r¹   rº   rT   r‡   r  r  rO  rP  Ú
total_lossÚignored_indexr  Ú
start_lossÚend_lossr©   s                         r0   rX   z!MPNetForQuestionAnswering.forward,  s¼  € ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—*‘*ØØ)Ø%Ø'Ø/Ø!5Ø#ð ó 
ˆð " !™*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJáØ" JÐ/°'¸!¸"°+Ñ=ˆFØ/9Ð/E�Z�M FÑ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r1   )	NNNNNNNNN)r2   r3   r4   r@   r   r)   r  r  ræ   rÄ   rž   r   rX   r8   r9   s   @r0   rH  rH     sø   ø„ ôð ð .2Ø37Ø04Ø26Ø37Ø15Ø)-Ø,0Ø#'ñ;
à×#Ñ# dÑ*ð;
ð ×)Ñ)¨DÑ0ð;
ð ×&Ñ&¨Ñ-ð	;
ð
 ×(Ñ(¨4Ñ/ð;
ð ×)Ñ)¨DÑ0ð;
ð ×'Ñ'¨$Ñ.ð;
ð   $™;ð;
ð # T™kð;
ð ˜D‘[ð;
ð 
ˆu�|‰|Ñ	Ð;Ñ	;ò;
ó ô;
r1   rH  c                 ó¸   — | j                  |«      j                  «       }t        j                  |d¬«      j	                  |«      |z  }|j                  «       |z   S )zñ
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's `utils.make_positions`. :param torch.Tensor x: :return torch.Tensor:
    r   rs   )Únerg   r)   ÚcumsumÚtype_asr\   )rR   r<   ÚmaskÚincremental_indicess       r0   rO   rO   k  sP   € ð �<‰<˜Ó$×(Ñ(Ó*€DÜŸ,™, t°Ô3×;Ñ;¸DÓAÀDÑHÐØ×#Ñ#Ó%¨Ñ3Ð3r1   )r  r5  rH  r(  r?  r¥   rò   r   )6r%  rx   r)   r   Útorch.nnr   r   r   Ú r   r#   Úactivationsr	   r
   Úmasking_utilsr   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_mpnetr   Ú
get_loggerr2   Úloggerr   ÚModuler&   r`   r‹   r“   r    r¥   r¯   ré   rò   r  r"   r(  r5  r?  r+  rH  rO   Ú__all__r¼   r1   r0   ú<module>rj     sÖ  ðñ ã ã Ý ß AÑ Aå &ß 'Ý 6÷÷ ñ õ .ß ,Ý ,ð 
ˆ×	Ñ	˜HÓ	%€ð ôi˜?ó ió ðiô8=�b—i‘iô 8=ôv;˜Ÿ™ô ;ô|�R—Y‘Yô ô6˜Ÿ	™	ô ô �"—)‘)ô ô�—‘ô ô<T�2—9‘9ô Tôp�"—)‘)ô ð ôG
Ð%ó G
ó ðG
ôTF
Ð+ô F
ôR�"—)‘)ô ñ, ðôôL
Ð%9ó L
óðL
ð^ ôX
Ð1ó X
ó ðX
ðv ô=
Ð"6ó =
ó ð=
ô@˜bŸi™iô ð& ôG
Ð 4ó G
ó ðG
òT4ò	�r1   