Ë
    ÛÓ^jÄß  ã                   ó®  — d Z ddlmZ ddlmZ ddlZddlmZ ddlmZm	Z	m
Z
 ddlmZ dd	lmZmZ dd
lmZmZmZ ddlmZ ddlmZmZ ddlmZ ddlmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ ddl%m&Z&m'Z' ddl(m)Z) ddl*m+Z+ ddl,m-Z-m.Z.m/Z/m0Z0 ddl1m2Z2m3Z3 ddl4m5Z5 ddl6m7Z7  e0jp                  e9«      Z: G d„ dejv                  «      Z<	 	 dVdejv                  dejz                  dejz                  dejz                  dejz                  dz  de>dz  de>d e)e.   fd!„Z? G d"„ d#ejv                  «      Z@ G d$„ d%ejv                  «      ZA G d&„ d'ejv                  «      ZB G d(„ d)ejv                  «      ZC G d*„ d+ejv                  «      ZD G d,„ d-ejv                  «      ZE G d.„ d/e«      ZF G d0„ d1ejv                  «      ZG G d2„ d3ejv                  «      ZH G d4„ d5ejv                  «      ZIe/ G d6„ d7e'«      «       ZJ e/d8¬9«      e G d:„ d;e-«      «       «       ZKe/ G d<„ d=eJ«      «       ZL G d>„ d?ejv                  «      ZM G d@„ dAejv                  «      ZN e/dB¬9«       G dC„ dDeJ«      «       ZO e/dE¬9«       G dF„ dGeJ«      «       ZP e/dH¬9«       G dI„ dJeJ«      «       ZQ e/dK¬9«       G dL„ dMeJ«      «       ZRe/ G dN„ dOeJ«      «       ZSe/ G dP„ dQeJ«      «       ZT e/dR¬9«       G dS„ dTeJe«      «       ZUg dU¢ZVy)WzPyTorch ELECTRA model.é    )ÚCallable)Ú	dataclassN)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FNÚget_activation)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)Ú"BaseModelOutputWithCrossAttentionsÚ)BaseModelOutputWithPastAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚModelOutputÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )ÚElectraConfigc                   óÊ   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  ded	ej                  fd
„Z
ˆ xZS )ÚElectraEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  |j                  ¬«      | _        t        j                  |j                  |j
                  «      | _	        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        | j#                  dt%        j&                  |j                  «      j)                  d«      d¬«       | j#                  dt%        j*                  | j,                  j/                  «       t$        j0                  ¬«      d¬«       y )	N)Úpadding_idx©ÚepsÚposition_ids©r'   éÿÿÿÿF)Ú
persistentÚtoken_type_ids©Údtype)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚembedding_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferÚtorchÚarangeÚexpandÚzerosr/   ÚsizeÚlong©ÚselfÚconfigÚ	__class__s     €úw/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/electra/modeling_electra.pyr7   zElectraEmbeddings.__init__;   s  ø€ Ü‰ÑÔÜ!Ÿ|™|¨F×,=Ñ,=¸v×?TÑ?TÐbh×buÑbuÔvˆÔÜ#%§<¡<°×0NÑ0NÐPV×PeÑPeÓ#fˆÔ Ü%'§\¡\°&×2HÑ2HÈ&×J_ÑJ_Ó%`ˆÔ"äŸ™ f×&;Ñ&;À×AVÑAVÔWˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒð 	×ÑØœEŸL™L¨×)GÑ)GÓH×OÑOÐPWÓXÐejð 	ô 	
ð 	×ÑØœeŸk™k¨$×*;Ñ*;×*@Ñ*@Ó*BÌ%Ï*É*ÔUÐbgð 	õ 	
ó    NÚ	input_idsr3   r/   Úinputs_embedsÚpast_key_values_lengthÚreturnc                 óŠ  — |�|j                  «       }n|j                  «       d d }|\  }}|€| j                  d d …|||z   …f   }|€št        | d«      rT| j                  j	                  |j
                  d   d«      }	t        j                  |	d|¬«      }	|	j	                  ||«      }n:t        j                  |t        j                  | j                  j                  ¬«      }|€| j                  |«      }| j                  |«      }
||
z   }| j                  |«      }||z   }| j                  |«      }| j                  |«      }|S )Nr1   r3   r   r'   )ÚdimÚindex)r5   Údevice)rK   r/   Úhasattrr3   rI   ÚshaperG   ÚgatherrJ   rL   rZ   r<   r@   r>   rA   rE   )rN   rS   r3   r/   rT   rU   Úinput_shapeÚ
batch_sizeÚ
seq_lengthÚbuffered_token_type_idsr@   Ú
embeddingsr>   s                rQ   ÚforwardzElectraEmbeddings.forwardM   sP  € ð Ð Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà!,Ñˆ
�JàÐØ×,Ñ,ªQÐ0FÈÐVlÑIlÐ0lÐ-lÑmˆLð
 Ð!Ü�tÐ-Ô.à*.×*=Ñ*=×*DÑ*DÀ\×EWÑEWÐXYÑEZÐ\^Ó*_Ð'Ü*/¯,©,Ð7NÐTUÐ]iÔ*jÐ'Ø!8×!?Ñ!?À
ÈJÓ!W‘ä!&§¡¨[ÄÇ
Á
ÐSW×SdÑSd×SkÑSkÔ!l�àÐ Ø ×0Ñ0°Ó;ˆMØ $× :Ñ :¸>Ó JÐØ"Ð%:Ñ:ˆ
à"×6Ñ6°|ÓDÐØÐ"5Ñ5ˆ
à—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐrR   )NNNNr   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r7   rG   Ú
LongTensorÚFloatTensorÚintÚTensorrc   Ú__classcell__©rP   s   @rQ   r*   r*   8   s‹   ø„ ÙQô
ð( .2Ø26Ø04Ø26Ø&'ñ(à×#Ñ# dÑ*ð(ð ×(Ñ(¨4Ñ/ð(ð ×&Ñ&¨Ñ-ð	(ð
 ×(Ñ(¨4Ñ/ð(ð !$ð(ð 
�‰÷(rR   r*   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingrE   Úkwargsc                 óœ  — |€|j                  d«      dz  }t        j                  ||j                  dd«      «      |z  }|�||z   }t        j
                  j                  |d¬«      }t        j
                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )Nr1   ç      à¿é   r	   ©rX   )ÚpÚtrainingr'   )
rK   rG   ÚmatmulÚ	transposer   Ú
functionalÚsoftmaxrE   rz   Ú
contiguous)
rn   ro   rp   rq   rr   rs   rE   rt   Úattn_weightsÚattn_outputs
             rQ   Úeager_attention_forwardr‚   y   s»   € ð €Ø—*‘*˜R“. DÑ(ˆô —<‘<  s§}¡}°Q¸Ó':Ó;¸gÑE€LàÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$rR   c                   ó–   ‡ — e Zd Zd	ˆ fd„	Z	 	 d
dej
                  dej                  dz  dedz  dee	   de
ej
                     f
d„Zˆ xZS )ÚElectraSelfAttentionNc                 ó@  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|| _        |j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _	        | j                  dz  | _
        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                   |j"                  «      | _        |j&                  | _        || _        || _        y ©Nr   r:   zThe hidden size (z6) is not a multiple of the number of attention heads (ú)rv   )r6   r7   Úhidden_sizeÚnum_attention_headsr[   Ú
ValueErrorrO   rj   Úattention_head_sizeÚall_head_sizers   r   ÚLinearro   rp   rq   rC   Úattention_probs_dropout_probrE   Ú
is_decoderÚ	is_causalÚ	layer_idx©rN   rO   r�   r‘   rP   s       €rQ   r7   zElectraSelfAttention.__init__—   sP  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð ˆŒà#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔØ×/Ñ/°Ñ5ˆŒä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z‘z &×"EÑ"EÓFˆŒà ×+Ñ+ˆŒØ"ˆŒØ"ˆ�rR   Úhidden_statesrr   Úpast_key_valuesrt   rV   c                 ó   — |j                   d d }g |¢d‘| j                  ‘­} | j                  |«      j                  |Ž j	                  dd«      } | j                  |«      j                  |Ž j	                  dd«      } | j                  |«      j                  |Ž j	                  dd«      }	|�>|}
t        |t        «      r|j                  }
|
j                  ||	| j                  «      \  }}	t        j                  | j                  j                  t         «      } || |||	|f| j"                  sdn| j$                  j&                  | j(                  dœ|¤Ž\  }} |j*                  g |¢d‘­Ž j-                  «       }||fS )Nr1   r'   rw   ç        ©rE   rs   )r\   r‹   ro   Úviewr|   rp   rq   Ú
isinstancer   Úself_attention_cacheÚupdater‘   r   Úget_interfacerO   Ú_attn_implementationr‚   rz   rE   ry   rs   Úreshaper   )rN   r“   rr   r”   rt   r^   Úhidden_shapeÚquery_layerÚ	key_layerÚvalue_layerÚcurrent_past_key_valuesÚattention_interfacer�   r€   s                 rQ   rc   zElectraSelfAttention.forward¯   s�  € ð $×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆð 5�d—j‘j Ó/×4Ñ4°lÐC×MÑMÈaÐQRÓSˆØ0�D—H‘H˜]Ó+×0Ñ0°,Ð?×IÑIÈ!ÈQÓOˆ	Ø4�d—j‘j Ó/×4Ñ4°lÐC×MÑMÈaÐQRÓSˆàÐ&à&5Ð#Ü˜/Ô+>Ô?Ø*9×*NÑ*NÐ'ð &=×%CÑ%CÀIÈ{Ð\`×\jÑ\jÓ%kÑ"ˆI�{ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$·,±,·.±.Ø—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ˜LÐ(Ð(rR   ©FN)NN©rd   re   rf   r7   rG   rk   ri   r   r   r!   Útuplerc   rl   rm   s   @rQ   r„   r„   –   sg   ø„ õ#ð6 48Ø(,ñ	')à—|‘|ð')ð ×)Ñ)¨DÑ0ð')ð  ™ð	')ð
 Ð+Ñ,ð')ð 
ˆu�|‰|Ñ	÷')rR   r„   c                   ó¶   ‡ — e Zd Zd
ˆ fd„	Z	 	 	 ddej
                  dej                  dz  dej                  dz  dedz  dee	   de
ej
                     fd	„Zˆ xZS )ÚElectraCrossAttentionNc                 ó  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|| _        |j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _	        | j                  dz  | _
        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                   |j"                  «      | _        || _        || _        y r†   )r6   r7   rˆ   r‰   r[   rŠ   rO   rj   r‹   rŒ   rs   r   r�   ro   rp   rq   rC   rŽ   rE   r�   r‘   r’   s       €rQ   r7   zElectraCrossAttention.__init__Û   sC  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð ˆŒà#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔØ×/Ñ/°Ñ5ˆŒä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z‘z &×"EÑ"EÓFˆŒà"ˆŒØ"ˆ�rR   r“   Úencoder_hidden_statesrr   r”   rt   rV   c                 óf  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }|�%|j
                  j                  | j                  «      nd}	|�]|	r[|j                  j                  | j                     j                  }
|j                  j                  | j                     j                  }nÅg |j                   d d ¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }
| j                  |«      j                  |«      j	                  dd«      }|�C|j                  j                  |
|| j                  «      \  }
}d|j
                  | j                  <   t        j                   | j"                  j$                  t&        «      } || ||
||f| j(                  sdn| j*                  j,                  | j.                  dœ|¤Ž\  }} |j0                  g |¢d‘­Ž j3                  «       }||fS )Nr1   r'   rw   FTr–   r—   )r\   r‹   ro   r˜   r|   Ú
is_updatedÚgetr‘   Úcross_attention_cacheÚlayersÚkeysÚvaluesrp   rq   r›   r   rœ   rO   r�   r‚   rz   rE   ry   rs   rž   r   )rN   r“   r«   rr   r”   rt   r^   rŸ   r    r­   r¡   r¢   Úkv_shaper¤   r�   r€   s                   rQ   rc   zElectraCrossAttention.forwardò   s  € ð $×)Ñ)¨#¨2Ð.ˆàC˜ÐC bÐC¨$×*BÑ*BÑCˆð —j‘j Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆàGVÐGb�_×/Ñ/×3Ñ3°D·N±NÔCÐhmˆ
ØÐ&©:à'×=Ñ=×DÑDÀTÇ^Á^ÑT×YÑYˆIØ)×?Ñ?×FÑFÀtÇ~Á~ÑV×]Ñ]‰KàXÐ.×4Ñ4°S°bÐ9ÐX¸2ÐX¸t×?WÑ?WÑXˆHØŸ™Ð!6Ó7×<Ñ<¸XÓF×PÑPÐQRÐTUÓVˆIØŸ*™*Ð%:Ó;×@Ñ@ÀÓJ×TÑTÐUVÐXYÓZˆKàÐ*à)8×)NÑ)N×)UÑ)UØ˜{¨D¯N©Nó*Ñ&�	˜;ð >B�×*Ñ*¨4¯>©>Ñ:ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$·,±,·.±.Ø—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ˜LÐ(Ð(rR   r¥   )NNN)rd   re   rf   r7   rG   rk   ri   r   r   r!   r§   rc   rl   rm   s   @rQ   r©   r©   Ú   s€   ø„ õ#ð4 ;?Ø37Ø6:ñ1)à—|‘|ð1)ð  %×0Ñ0°4Ñ7ð1)ð ×)Ñ)¨DÑ0ð	1)ð
 -¨tÑ3ð1)ð Ð+Ñ,ð1)ð 
ˆu�|‰|Ñ	÷1)rR   r©   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚElectraSelfOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _
        y ©Nr-   )r6   r7   r   r�   rˆ   ÚdenserA   rB   rC   rD   rE   rM   s     €rQ   r7   zElectraSelfOutput.__init__(  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�rR   r“   Úinput_tensorrV   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S ©N©r¸   rE   rA   ©rN   r“   r¹   s      rQ   rc   zElectraSelfOutput.forward.  ó7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°|Ñ'CÓDˆØÐrR   ©rd   re   rf   r7   rG   rk   rc   rl   rm   s   @rQ   rµ   rµ   '  ó1   ø„ ô>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ rR   rµ   c                   óÖ   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej
                  dej                  dz  dej                  dz  dej                  dz  dedz  dee	   d	e
ej
                     fd
„Zˆ xZS )ÚElectraAttentionNc                 óŒ   •— t         ‰| �  «        || _        |rt        nt        } ||||¬«      | _        t        |«      | _        y )N©r�   r‘   )r6   r7   Úis_cross_attentionr©   r„   rN   rµ   Úoutput)rN   rO   r�   r‘   rÅ   Úattention_classrP   s         €rQ   r7   zElectraAttention.__init__7  s=   ø€ Ü‰ÑÔØ"4ˆÔÙ3EÕ/ÔK_ˆÙ# F°iÈ9ÔUˆŒ	Ü'¨Ó/ˆ�rR   r“   rr   r«   Úencoder_attention_maskr”   rt   rV   c                 ó‚   — | j                   s|n|} | j                  |f|||dœ|¤Ž\  }}| j                  ||«      }||fS )N)r«   rr   r”   )rÅ   rN   rÆ   )	rN   r“   rr   r«   rÈ   r”   rt   Úattention_outputr€   s	            rQ   rc   zElectraAttention.forward>  sd   € ð 04×/FÒ/F™ÐLbˆØ)2¨¯©Øð*
à"7Ø)Ø+ñ	*
ð
 ñ*
Ñ&Ð˜,ð  Ÿ;™;Ð'7¸ÓGÐØ Ð-Ð-rR   )FNF©NNNNr¦   rm   s   @rQ   rÂ   rÂ   6  s—   ø„ õ0ð 48Ø:>Ø;?Ø(,ñ.à—|‘|ð.ð ×)Ñ)¨DÑ0ð.ð  %×0Ñ0°4Ñ7ð	.ð
 !&× 1Ñ 1°DÑ 8ð.ð  ™ð.ð Ð+Ñ,ð.ð 
ˆu�|‰|Ñ	÷.rR   rÂ   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚElectraIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y r»   )r6   r7   r   r�   rˆ   Úintermediate_sizer¸   r™   Ú
hidden_actÚstrr   Úintermediate_act_fnrM   s     €rQ   r7   zElectraIntermediate.__init__U  s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$rR   r“   rV   c                 óJ   — | j                  |«      }| j                  |«      }|S r»   )r¸   rÒ   )rN   r“   s     rQ   rc   zElectraIntermediate.forward]  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐrR   r¿   rm   s   @rQ   rÍ   rÍ   T  s#   ø„ ô9ð U§\¡\ð °e·l±l÷ rR   rÍ   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚElectraOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y r·   )r6   r7   r   r�   rÏ   rˆ   r¸   rA   rB   rC   rD   rE   rM   s     €rQ   r7   zElectraOutput.__init__e  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�rR   r“   r¹   rV   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S r»   r¼   r½   s      rQ   rc   zElectraOutput.forwardk  r¾   rR   r¿   rm   s   @rQ   rÕ   rÕ   d  rÀ   rR   rÕ   c                   óÖ   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej
                  dej                  dz  dej                  dz  dej                  dz  dedz  dee	   d	ej
                  fd
„Z
d„ Zˆ xZS )ÚElectraLayerNc                 ó„  •— t         ‰| �  «        |j                  | _        d| _        t	        ||j
                  |¬«      | _        |j
                  | _        |j                  | _        | j                  r.| j
                  st        | › d�«      ‚t	        |d|d¬«      | _	        t        |«      | _        t        |«      | _        y )Nr'   rÄ   z> should be used as a decoder model if cross attention is addedFT)r�   r‘   rÅ   )r6   r7   Úchunk_size_feed_forwardÚseq_len_dimrÂ   r�   Ú	attentionÚadd_cross_attentionrŠ   ÚcrossattentionrÍ   ÚintermediaterÕ   rÆ   )rN   rO   r‘   rP   s      €rQ   r7   zElectraLayer.__init__t  s­   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ)¨&¸F×<MÑ<MÐYbÔcˆŒØ ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ Ø×#Ò#Ø—?’?Ü  D 6Ð)gÐ!hÓiÐiÜ"2ØØØ#Ø#'ô	#ˆDÔô 0°Ó7ˆÔÜ# FÓ+ˆ�rR   r“   rr   r«   rÈ   r”   rt   rV   c                 ó   —  | j                   ||fd|i|¤Ž\  }}|}	| j                  r:|�8t        | d«      st        d| › d�«      ‚ | j                  |d ||fd|i|¤Ž\  }
}|
}	t        | j                  | j                  | j                  |	«      }|S )Nr”   rß   z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`)	rÝ   r�   r[   rŠ   rß   r   Úfeed_forward_chunkrÛ   rÜ   )rN   r“   rr   r«   rÈ   r”   rt   Úself_attention_outputÚ_rÊ   Úcross_attention_outputÚlayer_outputs               rQ   rc   zElectraLayer.forward‡  så   € ð $2 4§>¡>ØØñ$
ð ,ð$
ð ñ	$
Ñ Ð˜qð 1Ðà�?Š?Ð4Ð@Ü˜4Ð!1Ô2Ü Ø=¸d¸Vð DDð Dóð ð
 )<¨×(;Ñ(;Ø%ØØ%Ø&ñ	)ð
 !0ð)ð ñ)Ñ%Ð" Að  6Ðä0Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUeó
ˆð ÐrR   c                 óL   — | j                  |«      }| j                  ||«      }|S r»   )rà   rÆ   )rN   rÊ   Úintermediate_outputræ   s       rQ   râ   zElectraLayer.feed_forward_chunk®  s,   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6Ð8HÓIˆØÐrR   r»   rË   )rd   re   rf   r7   rG   rk   ri   r   r   r!   rc   râ   rl   rm   s   @rQ   rÙ   rÙ   s  s˜   ø„ õ,ð, 48Ø:>Ø;?Ø(,ñ%à—|‘|ð%ð ×)Ñ)¨DÑ0ð%ð  %×0Ñ0°4Ñ7ð	%ð
 !&× 1Ñ 1°DÑ 8ð%ð  ™ð%ð Ð+Ñ,ð%ð 
�‰ó%öNrR   rÙ   c                   óæ   ‡ — e Zd Zˆ fd„Z	 	 	 	 	 ddej
                  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  d	e	e
   d
eej
                     ez  fd„Zˆ xZS )ÚElectraEncoderc           	      óÆ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ c}«      | _        y c c}w )N)r‘   )	r6   r7   rO   r   Ú
ModuleListÚrangeÚnum_hidden_layersrÙ   Úlayer)rN   rO   ÚirP   s      €rQ   r7   zElectraEncoder.__init__¶  sF   ø€ Ü‰ÑÔØˆŒÜ—]‘]ÌuÐU[×UmÑUmÓOnÖ#oÈ!¤L°À1Ö$EÒ#oÓpˆ�
ùÒ#os   ½ANr“   rr   r«   rÈ   r”   Ú	use_cachert   rV   c                 ó„   — t        | j                  «      D ]  \  }}	 |	|||f||dœ|¤Ž}Œ t        ||r|¬«      S d ¬«      S )N)rÈ   r”   ©Úlast_hidden_stater”   )Ú	enumeraterï   r   )
rN   r“   rr   r«   rÈ   r”   rñ   rt   rð   Úlayer_modules
             rQ   rc   zElectraEncoder.forward»  sn   € ô  )¨¯©Ó4ò 	‰OˆAˆ|Ù(ØØØ%ðð (>Ø /ñð ñ‰Mð	ô 9Ø+Ù/8˜Oô
ð 	
à>Bô
ð 	
rR   )NNNNN)rd   re   rf   r7   rG   rk   ri   r   Úboolr   r!   r§   r   rc   rl   rm   s   @rQ   rê   rê   µ  s¯   ø„ ôqð 48Ø:>Ø;?Ø(,Ø!%ñ
à—|‘|ð
ð ×)Ñ)¨DÑ0ð
ð  %×0Ñ0°4Ñ7ð	
ð
 !&× 1Ñ 1°DÑ 8ð
ð  ™ð
ð ˜$‘;ð
ð Ð+Ñ,ð
ð 
ˆu�|‰|Ñ	ÐHÑ	H÷
rR   rê   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚElectraDiscriminatorPredictionszEPrediction module for the discriminator, made up of two dense layers.c                 ó  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        |j                  «      | _        t        j                  |j                  d«      | _	        || _
        y ©Nr'   )r6   r7   r   r�   rˆ   r¸   r   rÐ   Ú
activationÚdense_predictionrO   rM   s     €rQ   r7   z(ElectraDiscriminatorPredictions.__init__Ø  s^   ø€ Ü‰ÑÔä—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü(¨×):Ñ):Ó;ˆŒÜ "§	¡	¨&×*<Ñ*<¸aÓ @ˆÔØˆ�rR   c                 óŠ   — | j                  |«      }| j                  |«      }| j                  |«      j                  d«      }|S )Nr1   )r¸   rü   rý   Úsqueeze)rN   Údiscriminator_hidden_statesr“   Úlogitss       rQ   rc   z'ElectraDiscriminatorPredictions.forwardà  s?   € ØŸ
™
Ð#>Ó?ˆØŸ™¨Ó6ˆØ×&Ñ& }Ó5×=Ñ=¸bÓAˆàˆrR   ©rd   re   rf   rg   r7   rc   rl   rm   s   @rQ   rù   rù   Õ  s   ø„ ÙOôörR   rù   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚElectraGeneratorPredictionszAPrediction module for the generator, made up of two dense layers.c                 ó   •— t         ‰| �  «        t        d«      | _        t	        j
                  |j                  |j                  ¬«      | _        t	        j                  |j                  |j                  «      | _
        y )NÚgelur-   )r6   r7   r   rü   r   rA   r:   rB   r�   rˆ   r¸   rM   s     €rQ   r7   z$ElectraGeneratorPredictions.__init__ë  sV   ø€ Ü‰ÑÔä(¨Ó0ˆŒÜŸ™ f×&;Ñ&;À×AVÑAVÔWˆŒÜ—Y‘Y˜v×1Ñ1°6×3HÑ3HÓIˆ�
rR   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r»   )r¸   rü   rA   )rN   Úgenerator_hidden_statesr“   s      rQ   rc   z#ElectraGeneratorPredictions.forwardò  s3   € ØŸ
™
Ð#:Ó;ˆØŸ™¨Ó6ˆØŸ™ }Ó5ˆàÐrR   r  rm   s   @rQ   r  r  è  s   ø„ ÙKôJörR   r  c                   óF   ‡ — e Zd ZeZdZdZdZdZdZ	dZ
eeedœZˆ fd„Zˆ xZS )ÚElectraPreTrainedModelÚelectraT)r“   Ú
attentionsÚcross_attentionsc                 ó6  •— t         ‰| �  |«       t        |t        «      ryt	        j
                  |j                  t        j                  |j                  j                  d   «      j                  d«      «       t	        j                  |j                  «       y y )Nr1   r0   )r6   Ú_init_weightsr™   r*   ÚinitÚcopy_r/   rG   rH   r\   rI   Úzeros_r3   )rN   rn   rP   s     €rQ   r  z$ElectraPreTrainedModel._init_weights	  sm   ø€ Ü‰Ñ˜fÔ%Ü�fÔ/Ô0Ü�J‰J�v×*Ñ*¬E¯L©L¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÔhÜ�K‰K˜×-Ñ-Õ.ð 1rR   )rd   re   rf   r(   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendrÙ   r„   r©   Ú_can_record_outputsr  rl   rm   s   @rQ   r
  r
  ú  sF   ø„ à €LØ!ÐØ&*Ð#ØÐØ€NØÐØ"&Ðà%Ø*Ø1ñÐ÷/ð /rR   r
  z3
    Output type of [`ElectraForPreTraining`].
    )Úcustom_introc                   ó¾   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                     dz  ed<   dZe
ej                     dz  ed<   y)ÚElectraForPreTrainingOutputa+  
    loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
        Total loss of the ELECTRA objective.
    logits (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
        Prediction scores of the head (scores for each token before SoftMax).
    NÚlossr  r“   r  )rd   re   rf   rg   r  rG   ri   Ú__annotations__r  r“   r§   r  © rR   rQ   r  r    sg   … ñð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø'+€FˆE×Ñ Ñ$Ó+Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ô6rR   r  c                   ó˜  ‡ — e Zd Zˆ fd„Zd„ Zd„ Zeee	 	 	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  dee	j                     dz  dedz  dee   dee	j                     ez  fd„«       «       «       Zd„ Zˆ xZS )ÚElectraModelc                 ó0  •— t         ‰| �  |«       t        |«      | _        |j                  |j
                  k7  r/t        j                  |j                  |j
                  «      | _        t        |«      | _
        || _        d| _        | j                  «        y )NF)r6   r7   r*   rb   r:   rˆ   r   r�   Úembeddings_projectrê   ÚencoderrO   Úgradient_checkpointingÚ	post_initrM   s     €rQ   r7   zElectraModel.__init__&  st   ø€ Ü‰Ñ˜Ô Ü+¨FÓ3ˆŒà× Ñ  F×$6Ñ$6Ò6Ü&(§i¡i°×0EÑ0EÀv×GYÑGYÓ&ZˆDÔ#ä% fÓ-ˆŒØˆŒØ&+ˆÔ#à�‰ÕrR   c                 ó.   — | j                   j                  S r»   ©rb   r<   ©rN   s    rQ   Úget_input_embeddingsz!ElectraModel.get_input_embeddings3  s   € Ø�‰×.Ñ.Ð.rR   c                 ó&   — || j                   _        y r»   r)  )rN   rq   s     rQ   Úset_input_embeddingsz!ElectraModel.set_input_embeddings6  s   € Ø*/ˆ�‰Õ'rR   NrS   rr   r3   r/   rT   r«   rÈ   r”   rñ   rt   rV   c
           
      ó’  — |d u |d uz  rt        d«      ‚| j                  j                  r|	�|	n| j                  j                  }	nd}	|	rd|€b|€| j                  j                  r4t        t        | j                  ¬«      t        | j                  ¬«      «      nt        | j                  ¬«      }|�|j                  «       nd}| j                  |||||¬«      }t        | d«      r| j                  |«      }| j                  |||||¬«      \  }} | j                  |f|||||	|dœ|
¤Ž}t        |j                  |j                  ¬	«      S )
Nz:You must specify exactly one of input_ids or inputs_embedsF)rO   r   )rS   r/   r3   rT   rU   r$  )rr   rÈ   Úembedding_outputr«   r”   )rr   r«   rÈ   r”   rñ   r/   ró   )rŠ   rO   r�   rñ   Úis_encoder_decoderr   r   Úget_seq_lengthrb   r[   r$  Ú_create_attention_masksr%  r   rô   r”   )rN   rS   rr   r3   r/   rT   r«   rÈ   r”   rñ   rt   rU   r/  Úencoder_outputss                 rQ   rc   zElectraModel.forward9  sw  € ð  ˜Ð -°tÐ";Ò<ÜÐYÓZÐZà�;‰;×!Ò!Ø%.Ð%:™	ÀÇÁ×@UÑ@U‰IàˆIá˜Ð0ð )Ð4¸¿¹×8VÒ8Vô $¤L¸¿¹Ô$DÄlÐZ^×ZeÑZeÔFfÔgä!¨¯©Ô5ð ð FUÐE` ×!?Ñ!?Ô!AÐfgÐàŸ?™?ØØ%Ø)Ø'Ø#9ð +ó 
Ðô �4Ð-Ô.Ø#×6Ñ6Ð7GÓHÐà15×1MÑ1MØ)Ø#9Ø-Ø"7Ø+ð 2Nó 2
Ñ.ˆÐ.ð '˜$Ÿ,™,Øð	
à)Ø"7Ø#9Ø+ØØ%ñ	
ð ñ	
ˆô 9Ø-×?Ñ?Ø+×;Ñ;ô
ð 	
rR   c                 óÐ   — | j                   j                  rt        | j                   |||¬«      }nt        | j                   ||¬«      }|�t        | j                   |||¬«      }||fS )N)rO   rT   rr   r”   )rO   rT   rr   )rO   rT   rr   r«   )rO   r�   r   r   )rN   rr   rÈ   r/  r«   r”   s         rQ   r2  z$ElectraModel._create_attention_masks}  su   € ð �;‰;×!Ò!Ü/Ø—{‘{Ø.Ø-Ø /ô	‰Nô 7Ø—{‘{Ø.Ø-ôˆNð "Ð-Ü%>Ø—{‘{Ø.Ø5Ø&;ô	&Ð"ð Ð5Ð5Ð5rR   )	NNNNNNNNN)rd   re   rf   r7   r+  r-  r%   r&   r"   rG   rk   Úlistri   r÷   r   r!   r§   r   rc   r2  rl   rm   s   @rQ   r"  r"  $  s-  ø„ ôò/ò0ð  ØØð *.Ø.2Ø.2Ø,0Ø-1Ø59Ø6:Ø:>Ø!%ñ>
à—<‘< $Ñ&ð>
ð Ÿ™ tÑ+ð>
ð Ÿ™ tÑ+ð	>
ð
 —l‘l TÑ)ð>
ð —|‘| dÑ*ð>
ð  %Ÿ|™|¨dÑ2ð>
ð !&§¡¨tÑ 3ð>
ð ˜e×/Ñ/Ñ0°4Ñ7ð>
ð ˜$‘;ð>
ð Ð+Ñ,ð>
ð 
ˆu�|‰|Ñ	ÐAÑ	Aò>
ó ó ó  ð>
öB6rR   r"  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚElectraClassificationHeadz-Head for sentence-level classification tasks.c                 óz  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        |j                  �|j                  n|j                  }t        d«      | _	        t        j                  |«      | _        t        j                  |j                  |j                  «      | _        y )Nr  )r6   r7   r   r�   rˆ   r¸   Úclassifier_dropoutrD   r   rü   rC   rE   Ú
num_labelsÚout_proj©rN   rO   r9  rP   s      €rQ   r7   z"ElectraClassificationHead.__init__¡  s‹   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
à)/×)BÑ)BÐ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô )¨Ó0ˆŒÜ—z‘zÐ"4Ó5ˆŒÜŸ	™	 &×"4Ñ"4°f×6GÑ6GÓHˆ�rR   c                 óÈ   — |d d …dd d …f   }| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|S )Nr   )rE   r¸   rü   r;  )rN   Úfeaturesrt   Úxs       rQ   rc   z!ElectraClassificationHead.forward«  sZ   € Ø’Q˜š1�WÑˆØ�L‰L˜‹OˆØ�J‰J�q‹MˆØ�O‰O˜AÓˆØ�L‰L˜‹OˆØ�M‰M˜!ÓˆØˆrR   r  rm   s   @rQ   r7  r7  ž  s   ø„ Ù7ôIörR   r7  c                   ó‚   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej                  dej                  dz  dej                  fd„Z	ˆ xZ
S )
ÚElectraSequenceSummaryaÏ  
    Compute a single vector summary of a sequence hidden states.

    Args:
        config ([`ElectraConfig`]):
            The config used by the model. Relevant arguments in the config class of the model are (refer to the actual
            config class of your model for the default values it uses):

            - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are:

                - `"last"` -- Take the last token hidden state (like XLNet)
                - `"first"` -- Take the first token hidden state (like Bert)
                - `"mean"` -- Take the mean of all tokens hidden states
                - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2)
                - `"attn"` -- Not implemented now, use multi-head attention

            - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
            - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
              (otherwise to `config.hidden_size`).
            - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
              another string or `None` will add no activation.
            - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
            - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.
    rO   c                 óf  •— t         ‰| �  «        t        |dd«      | _        | j                  dk(  rt        ‚t        j                  «       | _        t        |d«      rq|j                  ret        |d«      r(|j                  r|j                  dkD  r|j                  }n|j                  }t        j                  |j                  |«      | _        t        |dd «      }|rt        |«      nt        j                  «       | _        t        j                  «       | _        t        |d«      r3|j"                  dkD  r$t        j$                  |j"                  «      | _        t        j                  «       | _        t        |d	«      r5|j(                  dkD  r%t        j$                  |j(                  «      | _        y y y )
NÚsummary_typeÚlastÚattnÚsummary_use_projÚsummary_proj_to_labelsr   Úsummary_activationÚsummary_first_dropoutÚsummary_last_dropout)r6   r7   ÚgetattrrC  ÚNotImplementedErrorr   ÚIdentityÚsummaryr[   rF  rG  r:  rˆ   r�   r   rü   Úfirst_dropoutrI  rC   Úlast_dropoutrJ  )rN   rO   Únum_classesÚactivation_stringrP   s       €rQ   r7   zElectraSequenceSummary.__init__Ð  sU  ø€ Ü‰ÑÔä# F¨N¸FÓCˆÔØ×Ñ Ò&ô &Ð%ä—{‘{“}ˆŒÜ�6Ð-Ô.°6×3JÒ3JÜ�vÐ7Ô8¸V×=ZÒ=ZÐ_e×_pÑ_pÐstÒ_tØ$×/Ñ/‘à$×0Ñ0�ÜŸ9™9 V×%7Ñ%7¸ÓEˆDŒLä# FÐ,@À$ÓGÐÙIZ¤NÐ3DÔ$EÔ`b×`kÑ`kÓ`mˆŒäŸ[™[›]ˆÔÜ�6Ð2Ô3¸×8TÑ8TÐWXÒ8XÜ!#§¡¨F×,HÑ,HÓ!IˆDÔäŸK™K›MˆÔÜ�6Ð1Ô2°v×7RÑ7RÐUVÒ7VÜ "§
¡
¨6×+FÑ+FÓ GˆDÕð 8WÐ2rR   Nr“   Ú	cls_indexrV   c                 óü  — | j                   dk(  r|dd…df   }�n| j                   dk(  r|dd…df   }�n| j                   dk(  r|j                  d¬«      }ná| j                   d	k(  r½|€At        j                  |d
dd…dd…f   |j                  d   dz
  t        j
                  ¬«      }nX|j                  d«      j                  d«      }|j                  d|j                  «       dz
  z  |j                  d«      fz   «      }|j                  d|«      j                  d«      }n| j                   dk(  rt        ‚| j                  «      }| j                  |«      }| j                  |«      }| j!                  |«      }|S )ak  
        Compute a single vector summary of a sequence hidden states.

        Args:
            hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`):
                The hidden states of the last layer.
            cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*):
                Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token.

        Returns:
            `torch.FloatTensor`: The summary of the sequence hidden states.
        rD  Nr1   Úfirstr   Úmeanr'   rx   rS  .éþÿÿÿr4   )r1   rE  )rC  rV  rG   Ú	full_liker\   rL   Ú	unsqueezerI   rX   rK   r]   rÿ   rL  rO  rN  rü   rP  )rN   r“   rS  rÆ   s       rQ   rc   zElectraSequenceSummary.forwardí  sn  € ð ×Ñ Ò&Ø"¢1 b 5Ñ)ŠFØ×Ñ 'Ò)Ø"¢1 a 4Ñ(ŠFØ×Ñ &Ò(Ø"×'Ñ'¨AÐ'Ó.‰FØ×Ñ +Ò-ØÐ Ü!ŸO™OØ! # r¨ rª1 *Ñ-Ø!×'Ñ'¨Ñ+¨aÑ/ÜŸ*™*ô‘	ð &×/Ñ/°Ó3×=Ñ=¸bÓA�	Ø%×,Ñ,¨U°i·m±m³oÈÑ6IÑ-JÈm×N`ÑN`ÐacÓNdÐMfÑ-fÓg�	à"×)Ñ)¨"¨iÓ8×@Ñ@ÀÓD‰FØ×Ñ &Ò(Ü%Ð%à×#Ñ# FÓ+ˆØ—‘˜fÓ%ˆØ—‘ Ó(ˆØ×"Ñ" 6Ó*ˆàˆrR   r»   )rd   re   rf   rg   r(   r7   rG   ri   rh   rc   rl   rm   s   @rQ   rA  rA  ¶  sQ   ø„ ñð2H˜}õ Hð< VZñ)Ø"×.Ñ.ð)Ø;@×;KÑ;KÈdÑ;Rð)à	×	Ñ	÷)rR   rA  zŸ
    ELECTRA Model transformer with a sequence classification/regression head on top (a linear layer on top of the
    pooled output) e.g. for GLUE tasks.
    c                   ó*  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ee	   d
e
ej                     ez  fd„«       «       Zˆ xZS )Ú ElectraForSequenceClassificationc                 ó´   •— t         ‰| �  |«       |j                  | _        || _        t	        |«      | _        t        |«      | _        | j                  «        y r»   )	r6   r7   r:  rO   r"  r  r7  Ú
classifierr'  rM   s     €rQ   r7   z)ElectraForSequenceClassification.__init__   sH   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØˆŒÜ# FÓ+ˆŒÜ3°FÓ;ˆŒð 	�‰ÕrR   NrS   rr   r3   r/   rT   Úlabelsrt   rV   c           	      ó¾  —  | j                   |f||||ddœ|¤Ž}|d   }	| j                  |	«      }
d}|��‡| j                  j                  €�| j                  dk(  rd| j                  _        nl| j                  dkD  rL|j
                  t        j                  k(  s|j
                  t        j                  k(  rd| j                  _        nd| j                  _        | j                  j                  dk(  rIt        «       }| j                  dk(  r& ||
j                  «       |j                  «       «      }nŒ ||
|«      }n‚| j                  j                  dk(  r=t        «       } ||
j                  d	| j                  «      |j                  d	«      «      }n,| j                  j                  dk(  rt        «       } ||
|«      }t        ||
|j                  |j                   ¬
«      S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        T©rr   r3   r/   rT   Úreturn_dictr   Nr'   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr1   ©r  r  r“   r  )r  r]  rO   Úproblem_typer:  r5   rG   rL   rj   r   rÿ   r   r˜   r   r   r“   r  )rN   rS   rr   r3   r/   rT   r^  rt   r   Úsequence_outputr  r  Úloss_fcts                rQ   rc   z(ElectraForSequenceClassification.forward*  s�  € ð$ '3 d§l¡lØð'
à)Ø)Ø%Ø'Øñ'
ð ñ'
Ð#ð 6°aÑ8ˆØ—‘ Ó1ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ä'ØØØ5×CÑCØ2×=Ñ=ô	
ð 	
rR   ©NNNNNN)rd   re   rf   r7   r$   r"   rG   rk   r   r!   r§   r   rc   rl   rm   s   @rQ   r[  r[    sÑ   ø„ ôð Øð *.Ø.2Ø.2Ø,0Ø-1Ø&*ñ9
à—<‘< $Ñ&ð9
ð Ÿ™ tÑ+ð9
ð Ÿ™ tÑ+ð	9
ð
 —l‘l TÑ)ð9
ð —|‘| dÑ*ð9
ð —‘˜tÑ#ð9
ð Ð+Ñ,ð9
ð 
ˆu�|‰|Ñ	Ð7Ñ	7ò9
ó ó ô9
rR   r[  zÊ
    Electra model with a binary classification head on top as used during pretraining for identifying generated tokens.

    It is recommended to load the discriminator checkpoint into that model.
    c                   ó*  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ee	   d
e
ej                     ez  fd„«       «       Zˆ xZS )ÚElectraForPreTrainingc                 ó„   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        | j                  «        y r»   )r6   r7   r"  r  rù   Údiscriminator_predictionsr'  rM   s     €rQ   r7   zElectraForPreTraining.__init__p  s3   ø€ Ü‰Ñ˜Ô ä# FÓ+ˆŒÜ)HÈÓ)PˆÔ&à�‰ÕrR   NrS   rr   r3   r/   rT   r^  rt   rV   c           	      ó  —  | j                   |f||||ddœ|¤Ž}|d   }	| j                  |	«      }
d}|�«t        j                  «       }|�a|j	                  d|	j
                  d   «      dk(  }|
j	                  d|	j
                  d   «      |   }||   } |||j                  «       «      }n4 ||
j	                  d|	j
                  d   «      |j                  «       «      }t        ||
|j                  |j                  ¬«      S )am  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the ELECTRA loss. Input should be a sequence of tokens (see `input_ids` docstring)
            Indices should be in `[0, 1]`:

            - 0 indicates the token is an original token,
            - 1 indicates the token was replaced.

        Examples:

        ```python
        >>> from transformers import ElectraForPreTraining, AutoTokenizer
        >>> import torch

        >>> discriminator = ElectraForPreTraining.from_pretrained("google/electra-base-discriminator")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/electra-base-discriminator")

        >>> sentence = "The quick brown fox jumps over the lazy dog"
        >>> fake_sentence = "The quick brown fox fake over the lazy dog"

        >>> fake_tokens = tokenizer.tokenize(fake_sentence, add_special_tokens=True)
        >>> fake_inputs = tokenizer.encode(fake_sentence, return_tensors="pt")
        >>> discriminator_outputs = discriminator(fake_inputs)
        >>> predictions = torch.round((torch.sign(discriminator_outputs[0]) + 1) / 2)

        >>> fake_tokens
        ['[CLS]', 'the', 'quick', 'brown', 'fox', 'fake', 'over', 'the', 'lazy', 'dog', '[SEP]']

        >>> predictions.squeeze().tolist()
        [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0]
        ```Tr`  r   Nr1   r'   re  )
r  rm  r   r   r˜   r\   Úfloatr  r“   r  )rN   rS   rr   r3   r/   rT   r^  rt   r   Údiscriminator_sequence_outputr  r  rh  Úactive_lossÚactive_logitsÚactive_labelss                   rQ   rc   zElectraForPreTraining.forwardx  s/  € ðV '3 d§l¡lØð'
à)Ø)Ø%Ø'Øñ'
ð ñ'
Ð#ð )DÀAÑ(FÐ%à×/Ñ/Ð0MÓNˆàˆØÐÜ×+Ñ+Ó-ˆHØÐ)Ø,×1Ñ1°"Ð6S×6YÑ6YÐZ[Ñ6\Ó]ÐabÑb�Ø &§¡¨BÐ0M×0SÑ0SÐTUÑ0VÓ WÐXcÑ d�Ø & {Ñ 3�Ù ¨}×/BÑ/BÓ/DÓE‘á §¡¨BÐ0M×0SÑ0SÐTUÑ0VÓ WÐY_×YeÑYeÓYgÓh�ä*ØØØ5×CÑCØ2×=Ñ=ô	
ð 	
rR   ri  )rd   re   rf   r7   r$   r"   rG   rk   r   r!   r§   r  rc   rl   rm   s   @rQ   rk  rk  h  sÛ   ø„ ôð Øð *.Ø.2Ø.2Ø,0Ø-1Ø&*ñF
à—<‘< $Ñ&ðF
ð Ÿ™ tÑ+ðF
ð Ÿ™ tÑ+ð	F
ð
 —l‘l TÑ)ðF
ð —|‘| dÑ*ðF
ð —‘˜tÑ#ðF
ð Ð+Ñ,ðF
ð 
ˆu�|‰|Ñ	Ð:Ñ	:òF
ó ó ôF
rR   rk  zý
    Electra model with a language modeling head on top.

    Even though both the discriminator and generator may be loaded into this model, the generator is the only model of
    the two to have been trained for the masked language modeling task.
    c                   ó>  ‡ — e Zd ZddiZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 dde	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  dee   dee	j                     ez  fd„«       «       Zˆ xZS )ÚElectraForMaskedLMúgenerator_lm_head.weightú)electra.embeddings.word_embeddings.weightc                 óâ   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        t        j                  |j                  |j                  «      | _
        | j                  «        y r»   )r6   r7   r"  r  r  Úgenerator_predictionsr   r�   r:   r9   Úgenerator_lm_headr'  rM   s     €rQ   r7   zElectraForMaskedLM.__init__Î  sR   ø€ Ü‰Ñ˜Ô ä# FÓ+ˆŒÜ%@ÀÓ%HˆÔ"ä!#§¡¨6×+@Ñ+@À&×BSÑBSÓ!TˆÔà�‰ÕrR   c                 ó   — | j                   S r»   ©rz  r*  s    rQ   Úget_output_embeddingsz(ElectraForMaskedLM.get_output_embeddingsØ  ó   € Ø×%Ñ%Ð%rR   c                 ó   — || _         y r»   r|  )rN   r<   s     rQ   Úset_output_embeddingsz(ElectraForMaskedLM.set_output_embeddingsÛ  s
   € Ø!0ˆÕrR   NrS   rr   r3   r/   rT   r^  rt   rV   c           	      óp  —  | j                   |f||||ddœ|¤Ž}|d   }	| j                  |	«      }
| j                  |
«      }
d}|�Pt        j                  «       } ||
j                  d| j                  j                  «      |j                  d«      «      }t        ||
|j                  |j                  ¬«      S )a¢  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        Tr`  r   Nr1   re  )r  ry  rz  r   r   r˜   rO   r9   r   r“   r  )rN   rS   rr   r3   r/   rT   r^  rt   r  Úgenerator_sequence_outputÚprediction_scoresr  rh  s                rQ   rc   zElectraForMaskedLM.forwardÞ  sÖ   € ð$ #/ $§,¡,Øð#
à)Ø)Ø%Ø'Øñ#
ð ñ#
Ðð %<¸AÑ$>Ð!à ×6Ñ6Ð7PÓQÐØ ×2Ñ2Ð3DÓEÐàˆàÐÜ×*Ñ*Ó,ˆHÙÐ-×2Ñ2°2°t·{±{×7MÑ7MÓNÐPV×P[ÑP[Ð\^ÓP_Ó`ˆDäØØ$Ø1×?Ñ?Ø.×9Ñ9ô	
ð 	
rR   ri  )rd   re   rf   Ú_tied_weights_keysr7   r}  r€  r$   r"   rG   rk   r   r!   r§   r   rc   rl   rm   s   @rQ   ru  ru  Ã  sè   ø„ ð 5Ð6aÐbÐôò&ò1ð Øð *.Ø.2Ø.2Ø,0Ø-1Ø&*ñ)
à—<‘< $Ñ&ð)
ð Ÿ™ tÑ+ð)
ð Ÿ™ tÑ+ð	)
ð
 —l‘l TÑ)ð)
ð —|‘| dÑ*ð)
ð —‘˜tÑ#ð)
ð Ð+Ñ,ð)
ð 
ˆu�|‰|Ñ	˜~Ñ	-ò)
ó ó ô)
rR   ru  z‰
    Electra model with a token classification head on top.

    Both the discriminator and generator may be loaded into this model.
    c                   ó*  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ee	   d
e
ej                     ez  fd„«       «       Zˆ xZS )ÚElectraForTokenClassificationc                 ó`  •— t         ‰| �  |«       |j                  | _        t        |«      | _        |j
                  �|j
                  n|j                  }t        j                  |«      | _	        t        j                  |j                  |j                  «      | _        | j                  «        y r»   )r6   r7   r:  r"  r  r9  rD   r   rC   rE   r�   rˆ   r]  r'  r<  s      €rQ   r7   z&ElectraForTokenClassification.__init__  s†   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä# FÓ+ˆŒà)/×)BÑ)BÐ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z‘zÐ"4Ó5ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒà�‰ÕrR   NrS   rr   r3   r/   rT   r^  rt   rV   c           	      óH  —  | j                   |f||||ddœ|¤Ž}|d   }	| j                  |	«      }	| j                  |	«      }
d}|�<t        «       } ||
j	                  d| j
                  «      |j	                  d«      «      }t        ||
|j                  |j                  ¬«      S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Tr`  r   Nr1   re  )	r  rE   r]  r   r˜   r:  r   r“   r  )rN   rS   rr   r3   r/   rT   r^  rt   r   rp  r  r  rh  s                rQ   rc   z%ElectraForTokenClassification.forward!  s¾   € ð  '3 d§l¡lØð'
à)Ø)Ø%Ø'Øñ'
ð ñ'
Ð#ð )DÀAÑ(FÐ%à(,¯©Ð5RÓ(SÐ%Ø—‘Ð!>Ó?ˆàˆØÐÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ5×CÑCØ2×=Ñ=ô	
ð 	
rR   ri  )rd   re   rf   r7   r$   r"   rG   rk   r   r!   r§   r   rc   rl   rm   s   @rQ   r†  r†    sÑ   ø„ ôð Øð *.Ø.2Ø.2Ø,0Ø-1Ø&*ñ&
à—<‘< $Ñ&ð&
ð Ÿ™ tÑ+ð&
ð Ÿ™ tÑ+ð	&
ð
 —l‘l TÑ)ð&
ð —|‘| dÑ*ð&
ð —‘˜tÑ#ð&
ð Ð+Ñ,ð&
ð 
ˆu�|‰|Ñ	Ð4Ñ	4ò&
ó ó ô&
rR   r†  c                   óR  ‡ — e Zd ZeZdZˆ fd„Zee	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  dee   dee	j                     ez  fd„«       «       Zˆ xZS )ÚElectraForQuestionAnsweringr  c                 óä   •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r»   )
r6   r7   r:  r"  r  r   r�   rˆ   Ú
qa_outputsr'  rM   s     €rQ   r7   z$ElectraForQuestionAnswering.__init__Q  sS   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä# FÓ+ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÕrR   NrS   rr   r3   r/   rT   Ústart_positionsÚend_positionsrt   rV   c           	      óÆ  —  | j                   |f||||ddœ|¤Ž}	|	d   }
| j                  |
«      }|j                  dd¬«      \  }}|j                  d«      j	                  «       }|j                  d«      j	                  «       }d }|�·|�µt        |j                  «       «      dkD  r|j                  d«      }t        |j                  «       «      dkD  r|j                  d«      }|j                  d«      }|j                  d|«      }|j                  d|«      }t        |¬«      } |||«      } |||«      }||z   dz  }t        ||||	j                  |	j                  ¬	«      S )
NTr`  r   r'   r1   rx   )Úignore_indexrw   )r  Ústart_logitsÚ
end_logitsr“   r  )r  rŒ  Úsplitrÿ   r   ÚlenrK   Úclampr   r   r“   r  )rN   rS   rr   r3   r/   rT   r�  rŽ  rt   r   rg  r  r‘  r’  Ú
total_lossÚignored_indexrh  Ú
start_lossÚend_losss                      rQ   rc   z#ElectraForQuestionAnswering.forward[  s‚  € ð '3 d§l¡lØð'
à)Ø)Ø%Ø'Øñ'
ð ñ'
Ð#ð 6°aÑ8ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJä+ØØ%Ø!Ø5×CÑCØ2×=Ñ=ô
ð 	
rR   )NNNNNNN)rd   re   rf   r(   r  r  r7   r$   r"   rG   rk   r   r!   r§   r   rc   rl   rm   s   @rQ   rŠ  rŠ  L  sò   ø„ à €LØ!Ðôð Øð *.Ø.2Ø.2Ø,0Ø-1Ø/3Ø-1ñ3
à—<‘< $Ñ&ð3
ð Ÿ™ tÑ+ð3
ð Ÿ™ tÑ+ð	3
ð
 —l‘l TÑ)ð3
ð —|‘| dÑ*ð3
ð Ÿ™¨Ñ,ð3
ð —|‘| dÑ*ð3
ð Ð+Ñ,ð3
ð 
ˆu�|‰|Ñ	Ð;Ñ	;ò3
ó ó ô3
rR   rŠ  c                   ó*  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ee	   d
e
ej                     ez  fd„«       «       Zˆ xZS )ÚElectraForMultipleChoicec                 óÎ   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        t        j                  |j                  d«      | _	        | j                  «        y rû   )r6   r7   r"  r  rA  Úsequence_summaryr   r�   rˆ   r]  r'  rM   s     €rQ   r7   z!ElectraForMultipleChoice.__init__•  sM   ø€ Ü‰Ñ˜Ô ä# FÓ+ˆŒÜ 6°vÓ >ˆÔÜŸ)™) F×$6Ñ$6¸Ó:ˆŒð 	�‰ÕrR   NrS   rr   r3   r/   rT   r^  rt   rV   c           	      óì  — |�|j                   d   n|j                   d   }|�!|j                  d|j                  d«      «      nd}|�!|j                  d|j                  d«      «      nd}|�!|j                  d|j                  d«      «      nd}|�!|j                  d|j                  d«      «      nd}|�1|j                  d|j                  d«      |j                  d«      «      nd} | j                  |f||||ddœ|¤Ž}	|	d   }
| j	                  |
«      }| j                  |«      }|j                  d|«      }d}|�t        «       } |||«      }t        |||	j                  |	j                  ¬«      S )	a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr'   r1   rW  Tr`  r   re  )
r\   r˜   rK   r  r�  r]  r   r   r“   r  )rN   rS   rr   r3   r/   rT   r^  rt   Únum_choicesr   rg  Úpooled_outputr  Úreshaped_logitsr  rh  s                   rQ   rc   z ElectraForMultipleChoice.forwardŸ  s¢  € ðT -6Ð,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆà>GÐ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÐG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ð(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð '3 d§l¡lØð'
à)Ø)Ø%Ø'Øñ'
ð ñ'
Ð#ð 6°aÑ8ˆà×-Ñ-¨oÓ>ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÐÜ'Ó)ˆHÙ˜O¨VÓ4ˆDä(ØØ"Ø5×CÑCØ2×=Ñ=ô	
ð 	
rR   ri  )rd   re   rf   r7   r$   r"   rG   rk   r   r!   r§   r   rc   rl   rm   s   @rQ   r›  r›  “  sÛ   ø„ ôð Øð *.Ø.2Ø.2Ø,0Ø-1Ø&*ñN
à—<‘< $Ñ&ðN
ð Ÿ™ tÑ+ðN
ð Ÿ™ tÑ+ð	N
ð
 —l‘l TÑ)ðN
ð —|‘| dÑ*ðN
ð —‘˜tÑ#ðN
ð Ð+Ñ,ðN
ð 
ˆu�|‰|Ñ	Ð8Ñ	8òN
ó ó ôN
rR   r›  zS
    ELECTRA Model with a `language modeling` head on top for CLM fine-tuning.
    c                   ó¶  ‡ — e Zd ZddiZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 dde	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  dedz  dedz  dee	j                  z  dee   dee	j                     ez  fd„«       «       Zˆ xZS )ÚElectraForCausalLMrv  rw  c                 ó$  •— t         ‰| �  |«       |j                  st        j	                  d«       t        |«      | _        t        |«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y )NzOIf you want to use `ElectraForCausalLM` as a standalone, add `is_decoder=True.`)r6   r7   r�   ÚloggerÚwarningr"  r  r  ry  r   r�   r:   r9   rz  r'  rM   s     €rQ   r7   zElectraForCausalLM.__init__ú  sh   ø€ Ü‰Ñ˜Ô à× Ò Ü�N‰NÐlÔmä# FÓ+ˆŒÜ%@ÀÓ%HˆÔ"Ü!#§¡¨6×+@Ñ+@À&×BSÑBSÓ!TˆÔà�‰ÕrR   c                 ó   — | j                   S r»   r|  r*  s    rQ   r}  z(ElectraForCausalLM.get_output_embeddings  r~  rR   c                 ó   — || _         y r»   r|  )rN   Únew_embeddingss     rQ   r€  z(ElectraForCausalLM.set_output_embeddings	  s
   € Ø!/ˆÕrR   NrS   rr   r3   r/   rT   r«   rÈ   r^  r”   rñ   Úlogits_to_keeprt   rV   c                 ó¼  — |�d}
 | j                   |f|||||||	|
ddœ	|¤Ž}|j                  }t        |t        «      rt	        | d«      n|}| j                  | j                  |dd…|dd…f   «      «      }d}|�* | j                  d||| j                  j                  dœ|¤Ž}t        |||j                  |j                  |j                  |j                  ¬«      S )a3  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`

        Example:

        ```python
        >>> from transformers import AutoTokenizer, ElectraForCausalLM, ElectraConfig
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("google/electra-base-generator")
        >>> config = ElectraConfig.from_pretrained("google/electra-base-generator")
        >>> config.is_decoder = True
        >>> model = ElectraForCausalLM.from_pretrained("google/electra-base-generator", config=config)

        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> prediction_logits = outputs.logits
        ```NFT)	rr   r3   r/   rT   r«   rÈ   r”   rñ   ra  )r  r^  r9   )r  r  r”   r“   r  r  r   )r  rô   r™   rj   Úslicerz  ry  Úloss_functionrO   r9   r   r”   r“   r  r  )rN   rS   rr   r3   r/   rT   r«   rÈ   r^  r”   rñ   rª  rt   Úoutputsr“   Úslice_indicesr  r  s                     rQ   rc   zElectraForCausalLM.forward  s
  € ðN ÐØˆIà=I¸T¿\¹\Øð>
à)Ø)Ø%Ø'Ø"7Ø#9Ø+ØØñ>
ð ñ>
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ×'Ñ'¨×(BÑ(BÀ=ÒQRÐTaÒcdÐQdÑCeÓ(fÓgˆàˆØÐØ%�4×%Ñ%Ðp¨V¸FÈtÏ{É{×OeÑOeÑpÐioÑpˆDä0ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô
ð 	
rR   )NNNNNNNNNNr   )rd   re   rf   r„  r7   r}  r€  r$   r"   rG   rk   r   r÷   rj   r   r!   r§   r   rc   rl   rm   s   @rQ   r£  r£  ò  s\  ø„ ð 5Ð6aÐbÐô
ò&ò0ð Øð *.Ø.2Ø.2Ø,0Ø-1Ø59Ø6:Ø&*Ø(,Ø!%Ø-.ñF
à—<‘< $Ñ&ðF
ð Ÿ™ tÑ+ðF
ð Ÿ™ tÑ+ð	F
ð
 —l‘l TÑ)ðF
ð —|‘| dÑ*ðF
ð  %Ÿ|™|¨dÑ2ðF
ð !&§¡¨tÑ 3ðF
ð —‘˜tÑ#ðF
ð  ™ðF
ð ˜$‘;ðF
ð ˜eŸl™lÑ*ðF
ð Ð+Ñ,ðF
ð 
ˆu�|‰|Ñ	Ð@Ñ	@òF
ó ó ôF
rR   r£  )	r£  ru  r›  rk  rŠ  r[  r†  r"  r
  )Nr–   )Wrg   Úcollections.abcr   Údataclassesr   rG   r   Útorch.nnr   r   r   Ú r
   r  Úactivationsr   r   Úcache_utilsr   r   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr    r!   r"   r#   Úutils.genericr$   r%   Úutils.output_capturingr&   Úconfiguration_electrar(   Ú
get_loggerrd   r¥  ÚModuler*   rk   ro  r‚   r„   r©   rµ   rÂ   rÍ   rÕ   rÙ   rê   rù   r  r
  r  r"  r7  rA  r[  rk  ru  r†  rŠ  r›  r£  Ú__all__r   rR   rQ   ú<module>rÄ     sJ  ðñ å $Ý !ã Ý ß AÑ Aå &ß 1ß CÑ CÝ )ß JÝ 9÷	÷ 	ó 	÷ GÝ &Ý 6÷ó ÷ JÝ 5Ý 0ð 
ˆ×	Ñ	˜HÓ	%€ô=˜Ÿ	™	ô =ðN !Øñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ñ(ó%ô:@)˜2Ÿ9™9ô @)ôHI)˜BŸI™Iô I)ôZ˜Ÿ	™	ô ô.�r—y‘yô .ô<˜"Ÿ)™)ô ô �B—I‘Iô ô>Ð-ô >ôD
�R—Y‘Yô 
ô@ b§i¡iô ô& "§)¡)ô ð$ ô/˜_ó /ó ð/ñ* ðôð
 ô7 +ó 7ó óð7ð ôv6Ð)ó v6ó ðv6ôr §	¡	ô ô0`˜RŸY™Yô `ñF ðôôF
Ð'=ó F
óðF
ñR ðôôQ
Ð2ó Q
óðQ
ñh ðôô>
Ð/ó >
óð>
ñB ðôô6
Ð$:ó 6
óð6
ðr ôC
Ð"8ó C
ó ðC
ðL ô[
Ð5ó [
ó ð[
ñ| ðôô
]
Ð/°ó ]
óð
]
ò@
�rR   