Ë
    ÛÓ^jÁÁ  ã                   óî  — d dl mZ d dlZd dlmZ d dlmZmZmZ ddlm	Z
 ddlmZmZ ddlmZmZmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZmZmZmZmZm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3  e,jh                  e5«      Z6 G d„ dejn                  «      Z8	 	 dIdejn                  dejr                  dejr                  dejr                  dejr                  dz  de:dz  de:de&e*   fd„Z; G d„ d ejn                  «      Z< G d!„ d"ejn                  «      Z= G d#„ d$ejn                  «      Z> G d%„ d&ejn                  «      Z? G d'„ d(ejn                  «      Z@ G d)„ d*ejn                  «      ZA G d+„ d,e«      ZBe+ G d-„ d.e$«      «       ZC G d/„ d0ejn                  «      ZD G d1„ d2ejn                  «      ZEe+ G d3„ d4eC«      «       ZF G d5„ d6ejn                  «      ZG G d7„ d8ejn                  «      ZH e+d9¬:«       G d;„ d<eCe«      «       ZIe+ G d=„ d>eC«      «       ZJ e+d?¬:«       G d@„ dAeC«      «       ZKe+ G dB„ dCeC«      «       ZLe+ G dD„ dEeC«      «       ZMe+ G dF„ dGeC«      «       ZNg dH¢ZOy)Jé    )ÚCallableN)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FNÚgelu)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚ!CausalLMOutputWithCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚTransformersKwargsÚauto_docstringÚlogging)Úcan_return_tupleÚmerge_with_config_defaults)Úcapture_outputsé   )ÚData2VecTextConfigc                   óì   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  ded	ej                  fd
„Z
ed„ «       Zedd„«       Zˆ xZS )ÚData2VecTextEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 óT  •— t         ‰| �  «        t        j                  |j                  |j
                  |j                  ¬«      | _        t        j                  |j                  |j
                  «      | _	        t        j                  |j
                  |j                  ¬«      | _
        t        j                  |j                  «      | _        | j                  dt!        j"                  |j$                  «      j'                  d«      d¬«       | j                  dt!        j(                  | j*                  j-                  «       t         j.                  ¬«      d¬«       |j                  | _        t        j                  |j$                  |j
                  | j0                  ¬«      | _        y )	N)Úpadding_idx©ÚepsÚposition_ids©r$   éÿÿÿÿF)Ú
persistentÚtoken_type_ids)Údtype)ÚsuperÚ__init__ÚnnÚ	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferÚtorchÚarangeÚmax_position_embeddingsÚexpandÚzerosr,   ÚsizeÚlongr)   Úposition_embeddings©ÚselfÚconfigÚ	__class__s     €ú~/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_text.pyr3   zData2VecTextEmbeddings.__init__:   s4  ø€ Ü‰ÑÔÜ!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ%'§\¡\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"äŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒà×ÑØœEŸL™L¨×)GÑ)GÓH×OÑOÐPWÓXÐejð 	ô 	
ð 	×ÑØœeŸk™k¨$×*;Ñ*;×*@Ñ*@Ó*BÌ%Ï*É*ÔUÐbgð 	ô 	
ð "×.Ñ.ˆÔÜ#%§<¡<Ø×*Ñ*¨F×,>Ñ,>ÈD×L\ÑL\ô$
ˆÕ ó    NÚ	input_idsr0   r,   Úinputs_embedsÚpast_key_values_lengthÚreturnc                 ó  — |€<|�| j                  || j                  |«      }n| j                  || j                  «      }|�|j                  «       }n|j                  «       d d }|\  }}|€³t	        | d«      rm| j
                  j                  |j                  «      j                  |j                  d   d«      }	t        j                  |	d|¬«      }	|	j                  ||«      }n:t        j                  |t        j                  | j                  j                  ¬«      }|€| j                  |«      }| j!                  |«      }
||
z   }| j#                  |«      }||z   }| j%                  |«      }| j'                  |«      }|S )Nr.   r0   r   r$   )ÚdimÚindex©r1   Údevice)Ú"create_position_ids_from_input_idsr)   Ú&create_position_ids_from_inputs_embedsrG   Úhasattrr0   ÚtorX   rE   ÚshaperB   ÚgatherrF   rH   r,   r9   r;   rI   r<   r@   )rK   rP   r0   r,   rQ   rR   Úinput_shapeÚ
batch_sizeÚ
seq_lengthÚbuffered_token_type_idsr;   Ú
embeddingsrI   s                rN   ÚforwardzData2VecTextEmbeddings.forwardN   s€  € ð ÐØÐ$à#×FÑFØ˜t×/Ñ/Ð1Gó ‘ð  $×JÑJÈ=ÐZ^×ZjÑZjÓk�àÐ Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà!,Ñˆ
�Jð
 Ð!Ü�tÐ-Ô.à*.×*=Ñ*=×*@Ñ*@À×ATÑATÓ*U×*\Ñ*\Ð]i×]oÑ]oÐpqÑ]rÐtvÓ*wÐ'Ü*/¯,©,Ð7NÐTUÐ]iÔ*jÐ'Ø!8×!?Ñ!?À
ÈJÓ!W‘ä!&§¡¨[ÄÇ
Á
ÐSW×SdÑSd×SkÑSkÔ!l�àÐ Ø ×0Ñ0°Ó;ˆMØ $× :Ñ :¸>Ó JÐØ"Ð%:Ñ:ˆ
à"×6Ñ6°|ÓDÐØÐ"5Ñ5ˆ
à—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐrO   c                 óæ   — | j                  «       dd }|d   }t        j                  |dz   ||z   dz   t        j                  | j                  ¬«      }|j                  d«      j                  |«      S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr.   r$   rW   r   )rG   rB   rC   rH   rX   Ú	unsqueezerE   )rQ   r)   r_   Úsequence_lengthr,   s        rN   rZ   z=Data2VecTextEmbeddings.create_position_ids_from_inputs_embeds~   sp   € ð $×(Ñ(Ó*¨3¨BÐ/ˆØ% a™.ˆä—|‘|Ø˜!‰O˜_¨{Ñ:¸QÑ>ÄeÇjÁjÐYf×YmÑYmô
ˆð ×%Ñ% aÓ(×/Ñ/°Ó<Ð<rO   c                 ó¾   — | j                  |«      j                  «       }t        j                  |d¬«      j	                  |«      |z   |z  }|j                  «       |z   S )a  
        Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
        are ignored. This is modified from fairseq's `utils.make_positions`.

        Args:
            x: torch.Tensor x:

        Returns: torch.Tensor
        r$   ©rU   )ÚneÚintrB   ÚcumsumÚtype_asrH   )rP   r)   rR   ÚmaskÚincremental_indicess        rN   rY   z9Data2VecTextEmbeddings.create_position_ids_from_input_ids�   sW   € ð �|‰|˜KÓ(×,Ñ,Ó.ˆÜ$Ÿ|™|¨D°aÔ8×@Ñ@ÀÓFÐI_Ñ_ÐcgÑgÐØ"×'Ñ'Ó)¨KÑ7Ð7rO   )NNNNr   )r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r3   rB   Ú
LongTensorÚFloatTensorrk   ÚTensorrd   ÚstaticmethodrZ   rY   Ú__classcell__©rM   s   @rN   r'   r'   7   s´   ø„ ÙQô
ð, .2Ø26Ø04Ø26Ø&'ñ.à×#Ñ# dÑ*ð.ð ×(Ñ(¨4Ñ/ð.ð ×&Ñ&¨Ñ-ð	.ð
 ×(Ñ(¨4Ñ/ð.ð !$ð.ð 
�‰ó.ð` ñ=ó ð=ð" ò8ó ô8rO   r'   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingr@   Úkwargsc                 óœ  — |€|j                  d«      dz  }t        j                  ||j                  dd«      «      |z  }|�||z   }t        j
                  j                  |d¬«      }t        j
                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )Nr.   ç      à¿é   r   ri   )ÚpÚtrainingr$   )
rG   rB   ÚmatmulÚ	transposer4   Ú
functionalÚsoftmaxr@   r…   Ú
contiguous)
rz   r{   r|   r}   r~   r   r@   r€   Úattn_weightsÚattn_outputs
             rN   Úeager_attention_forwardr�   ¡   s»   € ð €Ø—*‘*˜R“. DÑ(ˆô —<‘<  s§}¡}°Q¸Ó':Ó;¸gÑE€LàÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$rO   c                   ó–   ‡ — e Zd Zd	ˆ fd„	Z	 	 d
dej
                  dej                  dz  dedz  dee	   de
ej
                     f
d„Zˆ xZS )ÚData2VecTextSelfAttentionNc                 ó@  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|| _        |j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _	        | j                  dz  | _
        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                   |j"                  «      | _        |j&                  | _        || _        || _        y ©Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)r‚   )r2   r3   r7   Únum_attention_headsr[   Ú
ValueErrorrL   rk   Úattention_head_sizeÚall_head_sizer   r4   ÚLinearr{   r|   r}   r>   Úattention_probs_dropout_probr@   Ú
is_decoderÚ	is_causalÚ	layer_idx©rK   rL   r›   rœ   rM   s       €rN   r3   z"Data2VecTextSelfAttention.__init__¾   sP  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð ˆŒà#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔØ×/Ñ/°Ñ5ˆŒä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z‘z &×"EÑ"EÓFˆŒà ×+Ñ+ˆŒØ"ˆŒØ"ˆ�rO   Úhidden_statesr~   Úpast_key_valuesr€   rS   c                 ó   — |j                   d d }g |¢d‘| j                  ‘­} | j                  |«      j                  |Ž j	                  dd«      } | j                  |«      j                  |Ž j	                  dd«      } | j                  |«      j                  |Ž j	                  dd«      }	|�>|}
t        |t        «      r|j                  }
|
j                  ||	| j                  «      \  }}	t        j                  | j                  j                  t         «      } || |||	|f| j"                  sdn| j$                  j&                  | j(                  dœ|¤Ž\  }} |j*                  g |¢d‘­Ž j-                  «       }||fS )Nr.   r$   rƒ   ç        ©r@   r   )r]   r–   r{   Úviewr‡   r|   r}   Ú
isinstancer   Úself_attention_cacheÚupdaterœ   r   Úget_interfacerL   Ú_attn_implementationr�   r…   r@   r„   r   ÚreshaperŠ   )rK   rž   r~   rŸ   r€   r_   Úhidden_shapeÚquery_layerÚ	key_layerÚvalue_layerÚcurrent_past_key_valuesÚattention_interfacerŒ   r‹   s                 rN   rd   z!Data2VecTextSelfAttention.forwardÖ   s�  € ð $×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆð 5�d—j‘j Ó/×4Ñ4°lÐC×MÑMÈaÐQRÓSˆØ0�D—H‘H˜]Ó+×0Ñ0°,Ð?×IÑIÈ!ÈQÓOˆ	Ø4�d—j‘j Ó/×4Ñ4°lÐC×MÑMÈaÐQRÓSˆàÐ&à&5Ð#Ü˜/Ô+>Ô?Ø*9×*NÑ*NÐ'ð &=×%CÑ%CÀIÈ{Ð\`×\jÑ\jÓ%kÑ"ˆI�{ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$·,±,·.±.Ø—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ˜LÐ(Ð(rO   ©FN)NN©rp   rq   rr   r3   rB   rv   ru   r   r   r   Útuplerd   rx   ry   s   @rN   r�   r�   ½   sg   ø„ õ#ð6 48Ø(,ñ	')à—|‘|ð')ð ×)Ñ)¨DÑ0ð')ð  ™ð	')ð
 Ð+Ñ,ð')ð 
ˆu�|‰|Ñ	÷')rO   r�   c                   ó¶   ‡ — e Zd Zd
ˆ fd„	Z	 	 	 ddej
                  dej                  dz  dej                  dz  dedz  dee	   de
ej
                     fd	„Zˆ xZS )ÚData2VecTextCrossAttentionNc                 ó  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|| _        |j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _	        | j                  dz  | _
        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                   |j"                  «      | _        || _        || _        y r‘   )r2   r3   r7   r”   r[   r•   rL   rk   r–   r—   r   r4   r˜   r{   r|   r}   r>   r™   r@   r›   rœ   r�   s       €rN   r3   z#Data2VecTextCrossAttention.__init__  sC  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð ˆŒà#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔØ×/Ñ/°Ñ5ˆŒä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z‘z &×"EÑ"EÓFˆŒà"ˆŒØ"ˆ�rO   rž   Úencoder_hidden_statesr~   rŸ   r€   rS   c                 óf  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }|�%|j
                  j                  | j                  «      nd}	|�]|	r[|j                  j                  | j                     j                  }
|j                  j                  | j                     j                  }nÅg |j                   d d ¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }
| j                  |«      j                  |«      j	                  dd«      }|�C|j                  j                  |
|| j                  «      \  }
}d|j
                  | j                  <   t        j                   | j"                  j$                  t&        «      } || ||
||f| j(                  sdn| j*                  j,                  | j.                  dœ|¤Ž\  }} |j0                  g |¢d‘­Ž j3                  «       }||fS )Nr.   r$   rƒ   FTr¡   r¢   )r]   r–   r{   r£   r‡   Ú
is_updatedÚgetrœ   Úcross_attention_cacheÚlayersÚkeysÚvaluesr|   r}   r¦   r   r§   rL   r¨   r�   r…   r@   r„   r   r©   rŠ   )rK   rž   r¶   r~   rŸ   r€   r_   rª   r«   r¸   r¬   r­   Úkv_shaper¯   rŒ   r‹   s                   rN   rd   z"Data2VecTextCrossAttention.forward  s  € ð $×)Ñ)¨#¨2Ð.ˆàC˜ÐC bÐC¨$×*BÑ*BÑCˆð —j‘j Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆàGVÐGb�_×/Ñ/×3Ñ3°D·N±NÔCÐhmˆ
ØÐ&©:à'×=Ñ=×DÑDÀTÇ^Á^ÑT×YÑYˆIØ)×?Ñ?×FÑFÀtÇ~Á~ÑV×]Ñ]‰KàXÐ.×4Ñ4°S°bÐ9ÐX¸2ÐX¸t×?WÑ?WÑXˆHØŸ™Ð!6Ó7×<Ñ<¸XÓF×PÑPÐQRÐTUÓVˆIØŸ*™*Ð%:Ó;×@Ñ@ÀÓJ×TÑTÐUVÐXYÓZˆKàÐ*à)8×)NÑ)N×)UÑ)UØ˜{¨D¯N©Nó*Ñ&�	˜;ð >B�×*Ñ*¨4¯>©>Ñ:ä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}š}‘C°$·,±,·.±.Ø—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ˜LÐ(Ð(rO   r°   )NNN)rp   rq   rr   r3   rB   rv   ru   r   r   r   r²   rd   rx   ry   s   @rN   r´   r´      s€   ø„ õ#ð4 ;?Ø37Ø6:ñ1)à—|‘|ð1)ð  %×0Ñ0°4Ñ7ð1)ð ×)Ñ)¨DÑ0ð	1)ð
 -¨tÑ3ð1)ð Ð+Ñ,ð1)ð 
ˆu�|‰|Ñ	÷1)rO   r´   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚData2VecTextSelfOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _
        y ©Nr*   )r2   r3   r4   r˜   r7   Údenser<   r=   r>   r?   r@   rJ   s     €rN   r3   zData2VecTextSelfOutput.__init__M  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�rO   rž   Úinput_tensorrS   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S ©N©rÃ   r@   r<   ©rK   rž   rÄ   s      rN   rd   zData2VecTextSelfOutput.forwardS  ó7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°|Ñ'CÓDˆØÐrO   ©rp   rq   rr   r3   rB   rv   rd   rx   ry   s   @rN   rÀ   rÀ   L  ó1   ø„ ô>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ rO   rÀ   c                   óÖ   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej
                  dej                  dz  dej                  dz  dej                  dz  dedz  dee	   d	e
ej
                     fd
„Zˆ xZS )ÚData2VecTextAttentionNc                 óŒ   •— t         ‰| �  «        || _        |rt        nt        } ||||¬«      | _        t        |«      | _        y )N©r›   rœ   )r2   r3   Úis_cross_attentionr´   r�   rK   rÀ   Úoutput)rK   rL   r›   rœ   rÐ   Úattention_classrM   s         €rN   r3   zData2VecTextAttention.__init__[  s=   ø€ Ü‰ÑÔØ"4ˆÔÙ8JÕ4ÔPiˆÙ# F°iÈ9ÔUˆŒ	Ü,¨VÓ4ˆ�rO   rž   r~   r¶   Úencoder_attention_maskrŸ   r€   rS   c                 ó‚   — | j                   s|n|} | j                  |f|||dœ|¤Ž\  }}| j                  ||«      }||fS )N)r¶   r~   rŸ   )rÐ   rK   rÑ   )	rK   rž   r~   r¶   rÓ   rŸ   r€   Úattention_outputr‹   s	            rN   rd   zData2VecTextAttention.forwardb  sd   € ð 04×/FÒ/F™ÐLbˆØ)2¨¯©Øð*
à"7Ø)Ø+ñ	*
ð
 ñ*
Ñ&Ð˜,ð  Ÿ;™;Ð'7¸ÓGÐØ Ð-Ð-rO   )FNF©NNNNr±   ry   s   @rN   rÍ   rÍ   Z  s—   ø„ õ5ð 48Ø:>Ø;?Ø(,ñ.à—|‘|ð.ð ×)Ñ)¨DÑ0ð.ð  %×0Ñ0°4Ñ7ð	.ð
 !&× 1Ñ 1°DÑ 8ð.ð  ™ð.ð Ð+Ñ,ð.ð 
ˆu�|‰|Ñ	÷.rO   rÍ   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚData2VecTextIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y rÆ   )r2   r3   r4   r˜   r7   Úintermediate_sizerÃ   r¤   Ú
hidden_actÚstrr	   Úintermediate_act_fnrJ   s     €rN   r3   z!Data2VecTextIntermediate.__init__x  s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$rO   rž   rS   c                 óJ   — | j                  |«      }| j                  |«      }|S rÆ   )rÃ   rÝ   )rK   rž   s     rN   rd   z Data2VecTextIntermediate.forward€  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐrO   rÊ   ry   s   @rN   rØ   rØ   w  s#   ø„ ô9ð U§\¡\ð °e·l±l÷ rO   rØ   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚData2VecTextOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y rÂ   )r2   r3   r4   r˜   rÚ   r7   rÃ   r<   r=   r>   r?   r@   rJ   s     €rN   r3   zData2VecTextOutput.__init__‡  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�rO   rž   rÄ   rS   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S rÆ   rÇ   rÈ   s      rN   rd   zData2VecTextOutput.forward�  rÉ   rO   rÊ   ry   s   @rN   rà   rà   †  rË   rO   rà   c                   óÖ   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 ddej
                  dej                  dz  dej                  dz  dej                  dz  dedz  dee	   d	ej
                  fd
„Z
d„ Zˆ xZS )ÚData2VecTextLayerNc                 ó„  •— t         ‰| �  «        |j                  | _        d| _        t	        ||j
                  |¬«      | _        |j
                  | _        |j                  | _        | j                  r.| j
                  st        | › d�«      ‚t	        |d|d¬«      | _	        t        |«      | _        t        |«      | _        y )Nr$   rÏ   z> should be used as a decoder model if cross attention is addedFT)r›   rœ   rÐ   )r2   r3   Úchunk_size_feed_forwardÚseq_len_dimrÍ   rš   Ú	attentionÚadd_cross_attentionr•   ÚcrossattentionrØ   Úintermediaterà   rÑ   )rK   rL   rœ   rM   s      €rN   r3   zData2VecTextLayer.__init__•  s­   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ.¨vÀ×ARÑARÐ^gÔhˆŒØ ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ Ø×#Ò#Ø—?’?Ü  D 6Ð)gÐ!hÓiÐiÜ"7ØØØ#Ø#'ô	#ˆDÔô 5°VÓ<ˆÔÜ(¨Ó0ˆ�rO   rž   r~   r¶   rÓ   rŸ   r€   rS   c                 ó   —  | j                   ||fd|i|¤Ž\  }}|}	| j                  r:|�8t        | d«      st        d| › d�«      ‚ | j                  |d ||fd|i|¤Ž\  }
}|
}	t        | j                  | j                  | j                  |	«      }|S )NrŸ   rê   z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`)	rè   rš   r[   r•   rê   r   Úfeed_forward_chunkræ   rç   )rK   rž   r~   r¶   rÓ   rŸ   r€   Úself_attention_outputÚ_rÕ   Úcross_attention_outputÚlayer_outputs               rN   rd   zData2VecTextLayer.forward¨  så   € ð $2 4§>¡>ØØñ$
ð ,ð$
ð ñ	$
Ñ Ð˜qð 1Ðà�?Š?Ð4Ð@Ü˜4Ð!1Ô2Ü Ø=¸d¸Vð DDð Dóð ð
 )<¨×(;Ñ(;Ø%ØØ%Ø&ñ	)ð
 !0ð)ð ñ)Ñ%Ð" Að  6Ðä0Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUeó
ˆð ÐrO   c                 óL   — | j                  |«      }| j                  ||«      }|S rÆ   )rë   rÑ   )rK   rÕ   Úintermediate_outputrñ   s       rN   rí   z$Data2VecTextLayer.feed_forward_chunkÏ  s,   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6Ð8HÓIˆØÐrO   rÆ   rÖ   )rp   rq   rr   r3   rB   rv   ru   r   r   r   rd   rí   rx   ry   s   @rN   rä   rä   ”  s˜   ø„ õ1ð, 48Ø:>Ø;?Ø(,ñ%à—|‘|ð%ð ×)Ñ)¨DÑ0ð%ð  %×0Ñ0°4Ñ7ð	%ð
 !&× 1Ñ 1°DÑ 8ð%ð  ™ð%ð Ð+Ñ,ð%ð 
�‰ó%öNrO   rä   c                   óN   ‡ — e Zd ZeZdZdZddgZdZdZ	dZ
dZeeedœZˆ fd„Zˆ xZS )ÚData2VecTextPreTrainedModelÚdata2vec_textTÚData2VecTextForTextEmbeddingsrä   )rž   Ú
attentionsÚcross_attentionsc                 ó6  •— t         ‰| �  |«       t        |t        «      ryt	        j
                  |j                  t        j                  |j                  j                  d   «      j                  d«      «       t	        j                  |j                  «       y y )Nr.   r-   )r2   Ú_init_weightsr¤   r'   ÚinitÚcopy_r,   rB   rC   r]   rE   Úzeros_r0   )rK   rz   rM   s     €rN   rû   z)Data2VecTextPreTrainedModel._init_weightså  sm   ø€ Ü‰Ñ˜fÔ%Ü�fÔ4Ô5Ü�J‰J�v×*Ñ*¬E¯L©L¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÔhÜ�K‰K˜×-Ñ-Õ.ð 6rO   )rp   rq   rr   r%   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendrä   r�   r´   Ú_can_record_outputsrû   rx   ry   s   @rN   rõ   rõ   Õ  sR   ø„ à%€LØ'ÐØ&*Ð#Ø8Ð:MÐNÐØÐØ€NØÐØ"&Ðà*Ø/Ø6ñÐ÷/ð /rO   rõ   c                   óæ   ‡ — e Zd Zˆ fd„Z	 	 	 	 	 ddej
                  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  d	e	e
   d
eej
                     ez  fd„Zˆ xZS )ÚData2VecTextEncoderc           	      óÆ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ c}«      | _        y c c}w )N)rœ   )	r2   r3   rL   r4   Ú
ModuleListÚrangeÚnum_hidden_layersrä   Úlayer)rK   rL   ÚirM   s      €rN   r3   zData2VecTextEncoder.__init__í  sH   ø€ Ü‰ÑÔØˆŒÜ—]‘]ÔTYÐZ`×ZrÑZrÓTsÖ#tÈqÔ$5°fÈÖ$JÒ#tÓuˆ�
ùÒ#ts   ½ANrž   r~   r¶   rÓ   rŸ   Ú	use_cacher€   rS   c                 ó„   — t        | j                  «      D ]  \  }}	 |	|||f||dœ|¤Ž}Œ t        ||r|¬«      S d ¬«      S )N)rÓ   rŸ   )Úlast_hidden_staterŸ   )Ú	enumerater  r   )
rK   rž   r~   r¶   rÓ   rŸ   r  r€   r  Úlayer_modules
             rN   rd   zData2VecTextEncoder.forwardò  sn   € ô  )¨¯©Ó4ò 	‰OˆAˆ|Ù(ØØØ%ðð (>Ø /ñð ñ‰Mð	ô 9Ø+Ù/8˜Oô
ð 	
à>Bô
ð 	
rO   )NNNNN)rp   rq   rr   r3   rB   rv   ru   r   Úboolr   r   r²   r   rd   rx   ry   s   @rN   r	  r	  ì  s¯   ø„ ôvð 48Ø:>Ø;?Ø(,Ø!%ñ
à—|‘|ð
ð ×)Ñ)¨DÑ0ð
ð  %×0Ñ0°4Ñ7ð	
ð
 !&× 1Ñ 1°DÑ 8ð
ð  ™ð
ð ˜$‘;ð
ð Ð+Ñ,ð
ð 
ˆu�|‰|Ñ	ÐHÑ	H÷
rO   r	  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚData2VecTextPoolerc                 ó²   •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  «       | _        y rÆ   )r2   r3   r4   r˜   r7   rÃ   ÚTanhÚ
activationrJ   s     €rN   r3   zData2VecTextPooler.__init__  s9   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'™'›)ˆ�rO   rž   rS   c                 ó\   — |d d …df   }| j                  |«      }| j                  |«      }|S ©Nr   )rÃ   r  )rK   rž   Úfirst_token_tensorÚpooled_outputs       rN   rd   zData2VecTextPooler.forward  s6   € ð +ª1¨a¨4Ñ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐrO   rÊ   ry   s   @rN   r  r    s#   ø„ ô$ð
 U§\¡\ð °e·l±l÷ rO   r  c                   óˆ  ‡ — e Zd ZddgZdˆ fd„	Zd„ Zd„ Zeee		 	 	 	 	 	 	 	 	 dde
j                  dz  de
j                  dz  d	e
j                  dz  d
e
j                  dz  de
j                  dz  de
j                  dz  de
j                  dz  dedz  dedz  dee   dee
j                     ez  fd„«       «       «       Zd„ Zˆ xZS )ÚData2VecTextModelr'   rä   c                 óÈ   •— t         ‰| �  |«       || _        d| _        t	        |«      | _        t        |«      | _        |rt        |«      nd| _	        | j                  «        y)zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        FN)r2   r3   rL   Úgradient_checkpointingr'   rc   r	  Úencoderr  ÚpoolerÚ	post_init)rK   rL   Úadd_pooling_layerrM   s      €rN   r3   zData2VecTextModel.__init__  sV   ø€ ô
 	‰Ñ˜Ô ØˆŒØ&+ˆÔ#ä0°Ó8ˆŒÜ*¨6Ó2ˆŒá4EÔ(¨Ô0È4ˆŒð 	�‰ÕrO   c                 ó.   — | j                   j                  S rÆ   ©rc   r9   ©rK   s    rN   Úget_input_embeddingsz&Data2VecTextModel.get_input_embeddings0  s   € Ø�‰×.Ñ.Ð.rO   c                 ó&   — || j                   _        y rÆ   r(  )rK   r}   s     rN   Úset_input_embeddingsz&Data2VecTextModel.set_input_embeddings3  s   € Ø*/ˆ�‰Õ'rO   NrP   r~   r0   r,   rQ   r¶   rÓ   rŸ   r  r€   rS   c
           
      óœ  — |d u |d uz  rt        d«      ‚| j                  j                  r|	�|	n| j                  j                  }	nd}	|	rd|€b|€| j                  j                  r4t        t        | j                  ¬«      t        | j                  ¬«      «      nt        | j                  ¬«      }|�|j                  «       nd}| j                  |||||¬«      }| j                  |||||¬«      \  }} | j                  |f|||||	|dœ|
¤Ž}|j                  }| j                  �| j                  |«      nd }t        |||j                  ¬«      S )	Nz:You must specify exactly one of input_ids or inputs_embedsF)rL   r   )rP   r,   r0   rQ   rR   )r~   rÓ   Úembedding_outputr¶   rŸ   )r~   r¶   rÓ   rŸ   r  r,   )r  Úpooler_outputrŸ   )r•   rL   rš   r  Úis_encoder_decoderr   r   Úget_seq_lengthrc   Ú_create_attention_masksr#  r  r$  r   rŸ   )rK   rP   r~   r0   r,   rQ   r¶   rÓ   rŸ   r  r€   rR   r.  Úencoder_outputsÚsequence_outputr  s                   rN   rd   zData2VecTextModel.forward6  s|  € ð  ˜Ð -°tÐ";Ò<ÜÐYÓZÐZà�;‰;×!Ò!Ø%.Ð%:™	ÀÇÁ×@UÑ@U‰IàˆIá˜Ð0ð )Ð4¸¿¹×8VÒ8Vô $¤L¸¿¹Ô$DÄlÐZ^×ZeÑZeÔFfÔgä!¨¯©Ô5ð ð FUÐE` ×!?Ñ!?Ô!AÐfgÐàŸ?™?ØØ%Ø)Ø'Ø#9ð +ó 
Ðð 26×1MÑ1MØ)Ø#9Ø-Ø"7Ø+ð 2Nó 2
Ñ.ˆÐ.ð '˜$Ÿ,™,Øð	
à)Ø"7Ø#9Ø+ØØ%ñ	
ð ñ	
ˆð *×;Ñ;ˆØ8<¿¹Ð8O˜Ÿ™ OÔ4ÐUYˆä;Ø-Ø'Ø+×;Ñ;ô
ð 	
rO   c                 óÐ   — | j                   j                  rt        | j                   |||¬«      }nt        | j                   ||¬«      }|�t        | j                   |||¬«      }||fS )N)rL   rQ   r~   rŸ   )rL   rQ   r~   )rL   rQ   r~   r¶   )rL   rš   r   r   )rK   r~   rÓ   r.  r¶   rŸ   s         rN   r2  z)Data2VecTextModel._create_attention_masksz  su   € ð �;‰;×!Ò!Ü/Ø—{‘{Ø.Ø-Ø /ô	‰Nô 7Ø—{‘{Ø.Ø-ôˆNð "Ð-Ü%>Ø—{‘{Ø.Ø5Ø&;ô	&Ð"ð Ð5Ð5Ð5rO   )T)	NNNNNNNNN)rp   rq   rr   r  r3   r*  r,  r"   r#   r   rB   rv   r   r  r   r   r²   r   rd   r2  rx   ry   s   @rN   r   r     s-  ø„ à1Ð3FÐGÐõò"/ò0ð  ØØð *.Ø.2Ø.2Ø,0Ø-1Ø59Ø6:Ø(,Ø!%ñ?
à—<‘< $Ñ&ð?
ð Ÿ™ tÑ+ð?
ð Ÿ™ tÑ+ð	?
ð
 —l‘l TÑ)ð?
ð —|‘| dÑ*ð?
ð  %Ÿ|™|¨dÑ2ð?
ð !&§¡¨tÑ 3ð?
ð  ™ð?
ð ˜$‘;ð?
ð Ð+Ñ,ð?
ð 
ˆu�|‰|Ñ	ÐKÑ	Kò?
ó ó ó  ð?
öB6rO   r   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚData2VecTextLMHeadz/Data2VecText Head for masked language modeling.c                 ó¬  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  «      | _
        t        j                  t        j                  |j                  «      «      | _        y rÂ   )r2   r3   r4   r˜   r7   rÃ   r<   r=   Ú
layer_normr6   ÚdecoderÚ	ParameterrB   rF   ÚbiasrJ   s     €rN   r3   zData2VecTextLMHead.__init__ž  s‡   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒä—y‘y ×!3Ñ!3°V×5FÑ5FÓGˆŒÜ—L‘L¤§¡¨V×->Ñ->Ó!?Ó@ˆ�	rO   c                 ó‚   — | j                  |«      }t        |«      }| j                  |«      }| j                  |«      }|S rÆ   )rÃ   r
   r9  r:  ©rK   Úfeaturesr€   Úxs       rN   rd   zData2VecTextLMHead.forward¦  s;   € Ø�J‰J�xÓ ˆÜ�‹GˆØ�O‰O˜AÓˆð �L‰L˜‹OˆàˆrO   ©rp   rq   rr   rs   r3   rd   rx   ry   s   @rN   r7  r7  ›  s   ø„ Ù9ôAörO   r7  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚData2VecTextClassificationHeadz-Head for sentence-level classification tasks.c                 óZ  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        |j                  �|j                  n|j                  }t        j                  |«      | _	        t        j                  |j                  |j                  «      | _        y rÆ   )r2   r3   r4   r˜   r7   rÃ   Úclassifier_dropoutr?   r>   r@   Ú
num_labelsÚout_proj©rK   rL   rE  rM   s      €rN   r3   z'Data2VecTextClassificationHead.__init__´  s   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
à)/×)BÑ)BÐ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z‘zÐ"4Ó5ˆŒÜŸ	™	 &×"4Ñ"4°f×6GÑ6GÓHˆ�rO   c                 óÐ   — |d d …dd d …f   }| j                  |«      }| j                  |«      }t        j                  |«      }| j                  |«      }| j	                  |«      }|S r  )r@   rÃ   rB   ÚtanhrG  r>  s       rN   rd   z&Data2VecTextClassificationHead.forward½  sY   € Ø’Q˜š1�WÑˆØ�L‰L˜‹OˆØ�J‰J�q‹MˆÜ�J‰J�q‹MˆØ�L‰L˜‹OˆØ�M‰M˜!ÓˆØˆrO   rA  ry   s   @rN   rC  rC  ±  s   ø„ Ù7ôIörO   rC  zX
    Data2VecText Model with a `language modeling` head on top for CLM fine-tuning.
    )Úcustom_introc                   ó¾  ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 	 	 	 dde	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  deee	j                        dz  dedz  dee	j                  z  dee   deez  fd„«       «       Zˆ xZS )ÚData2VecTextForCausalLMú/data2vec_text.embeddings.word_embeddings.weightúlm_head.bias©zlm_head.decoder.weightzlm_head.decoder.biasc                 óÊ   •— t         ‰| �  |«       |j                  st        j	                  d«       t        |d¬«      | _        t        |«      | _        | j                  «        y )NzTIf you want to use `Data2VecTextLMHeadModel` as a standalone, add `is_decoder=True.`F©r&  ©
r2   r3   rš   ÚloggerÚwarningr   rö   r7  Úlm_headr%  rJ   s     €rN   r3   z Data2VecTextForCausalLM.__init__Ò  sM   ø€ Ü‰Ñ˜Ô à× Ò Ü�N‰NÐqÔrä.¨vÈÔOˆÔÜ)¨&Ó1ˆŒð 	�‰ÕrO   c                 ó.   — | j                   j                  S rÆ   ©rV  r:  r)  s    rN   Úget_output_embeddingsz-Data2VecTextForCausalLM.get_output_embeddingsÞ  ó   € Ø�|‰|×#Ñ#Ð#rO   c                 ó&   — || j                   _        y rÆ   rX  ©rK   Únew_embeddingss     rN   Úset_output_embeddingsz-Data2VecTextForCausalLM.set_output_embeddingsá  ó   € Ø-ˆ�‰ÕrO   NrP   r~   r0   r,   rQ   r¶   rÓ   ÚlabelsrŸ   r  Úlogits_to_keepr€   rS   c                 óž  — |�d}
 | j                   |f|||||||	|
ddœ	|¤Ž}|j                  }t        |t        «      rt	        | d«      n|}| j                  |dd…|dd…f   «      }d}|�* | j                  d||| j                  j                  dœ|¤Ž}t        |||j                  |j                  |j                  |j                  ¬«      S )aA  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Data2VecTextForCausalLM, Data2VecTextConfig
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("facebook/data2vec-text-base")
        >>> config = Data2VecTextConfig.from_pretrained("facebook/data2vec-text-base")
        >>> config.is_decoder = True
        >>> model = Data2VecTextForCausalLM.from_pretrained("facebook/data2vec-text-base", config=config)

        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> prediction_logits = outputs.logits
        ```NFT)	r~   r0   r,   rQ   r¶   rÓ   rŸ   r  Úreturn_dict)Úlogitsr`  r6   )Úlossrd  rŸ   rž   rø   rù   © )rö   r  r¤   rk   ÚslicerV  Úloss_functionrL   r6   r   rŸ   rž   rø   rù   )rK   rP   r~   r0   r,   rQ   r¶   rÓ   r`  rŸ   r  ra  r€   Úoutputsrž   Úslice_indicesrd  re  s                     rN   rd   zData2VecTextForCausalLM.forwardä  sÿ   € ðN ÐØˆIà@RÀ×@RÑ@RØðA
à)Ø)Ø%Ø'Ø"7Ø#9Ø+ØØñA
ð ñA
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ%Ðp¨V¸FÈtÏ{É{×OeÑOeÑpÐioÑpˆDä0ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø$×5Ñ5ô
ð 	
rO   )NNNNNNNNNNr   )rp   rq   rr   Ú_tied_weights_keysr3   rY  r^  r!   r   rB   rt   ru   r²   r  rk   rv   r   r   r   rd   rx   ry   s   @rN   rM  rM  Ç  sw  ø„ ð #TØ .ñÐô

ò$ò.ð Øð .2Ø37Ø26Ø04Ø26Ø:>Ø;?Ø*.ØBFØ!%Ø-.ñF
à×#Ñ# dÑ*ðF
ð ×)Ñ)¨DÑ0ðF
ð ×(Ñ(¨4Ñ/ð	F
ð
 ×&Ñ&¨Ñ-ðF
ð ×(Ñ(¨4Ñ/ðF
ð  %×0Ñ0°4Ñ7ðF
ð !&× 1Ñ 1°DÑ 8ðF
ð × Ñ  4Ñ'ðF
ð ˜u U×%6Ñ%6Ñ7Ñ8¸4Ñ?ðF
ð ˜$‘;ðF
ð ˜eŸl™lÑ*ðF
ð Ð+Ñ,ðF
ð 
Ð2Ñ	2òF
ó ó ôF
rO   rM  c                   óf  ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 	 	 dde	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  dee   deez  fd„«       «       Zˆ xZS )ÚData2VecTextForMaskedLMrN  rO  rP  c                 óÊ   •— t         ‰| �  |«       |j                  rt        j	                  d«       t        |d¬«      | _        t        |«      | _        | j                  «        y )NzsIf you want to use `Data2VecTextForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.FrR  rS  rJ   s     €rN   r3   z Data2VecTextForMaskedLM.__init__6  sS   ø€ Ü‰Ñ˜Ô à×ÒÜ�N‰Nð1ôô
 /¨vÈÔOˆÔÜ)¨&Ó1ˆŒð 	�‰ÕrO   c                 ó.   — | j                   j                  S rÆ   rX  r)  s    rN   rY  z-Data2VecTextForMaskedLM.get_output_embeddingsE  rZ  rO   c                 ó&   — || j                   _        y rÆ   rX  r\  s     rN   r^  z-Data2VecTextForMaskedLM.set_output_embeddingsH  r_  rO   NrP   r~   r0   r,   rQ   r¶   rÓ   r`  r€   rS   c	                 ót  —  | j                   |f||||||ddœ|	¤Ž}
|
d   }| j                  |«      }d}|�at        «       }|j                  |j                  «      } ||j                  d| j                  j                  «      |j                  d«      «      }t        |||
j                  |
j                  ¬«      S )a¢  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        T)r~   r0   r,   rQ   r¶   rÓ   rc  r   Nr.   ©re  rd  rž   rø   )rö   rV  r   r\   rX   r£   rL   r6   r   rž   rø   )rK   rP   r~   r0   r,   rQ   r¶   rÓ   r`  r€   ri  r4  Úprediction_scoresÚmasked_lm_lossÚloss_fcts                  rN   rd   zData2VecTextForMaskedLM.forwardK  sÕ   € ð( %�$×$Ñ$Øð

à)Ø)Ø%Ø'Ø"7Ø#9Øñ

ð ñ

ˆð " !™*ˆØ ŸL™L¨Ó9ÐàˆØÐÜ'Ó)ˆHà—Y‘YÐ0×7Ñ7Ó8ˆFÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
rO   )NNNNNNNN)rp   rq   rr   rk  r3   rY  r^  r!   r   rB   rt   ru   r   r   r²   r   rd   rx   ry   s   @rN   rm  rm  /  s  ø„ ð #TØ .ñÐô
ò$ò.ð Øð .2Ø37Ø26Ø04Ø26Ø:>Ø;?Ø*.ñ,
à×#Ñ# dÑ*ð,
ð ×)Ñ)¨DÑ0ð,
ð ×(Ñ(¨4Ñ/ð	,
ð
 ×&Ñ&¨Ñ-ð,
ð ×(Ñ(¨4Ñ/ð,
ð  %×0Ñ0°4Ñ7ð,
ð !&× 1Ñ 1°DÑ 8ð,
ð × Ñ  4Ñ'ð,
ð Ð+Ñ,ð,
ð 
�Ñ	ò,
ó ó ô,
rO   rm  z¤
    Data2VecText Model transformer with a sequence classification/regression head on top (a linear layer on top of the
    pooled output) e.g. for GLUE tasks.
    c                   ó  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
   d
eez  fd„«       «       Zˆ xZS )Ú%Data2VecTextForSequenceClassificationc                 ó¸   •— t         ‰| �  |«       |j                  | _        || _        t	        |d¬«      | _        t        |«      | _        | j                  «        y ©NFrR  )	r2   r3   rF  rL   r   rö   rC  Ú
classifierr%  rJ   s     €rN   r3   z.Data2VecTextForSequenceClassification.__init__ƒ  sK   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØˆŒä.¨vÈÔOˆÔÜ8¸Ó@ˆŒð 	�‰ÕrO   NrP   r~   r0   r,   rQ   r`  r€   rS   c           	      óô  —  | j                   |f||||ddœ|¤Ž}|d   }	| j                  |	«      }
d}|��¢|j                  |
j                  «      }| j                  j
                  €�| j                  dk(  rd| j                  _        nl| j                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                  _        nd| j                  _        | j                  j
                  dk(  rIt        «       }| j                  dk(  r& ||
j                  «       |j                  «       «      }nŒ ||
|«      }n‚| j                  j
                  dk(  r=t        «       } ||
j                  d	| j                  «      |j                  d	«      «      }n,| j                  j
                  dk(  rt        «       } ||
|«      }t!        ||
|j"                  |j$                  ¬
«      S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        T©r~   r0   r,   rQ   rc  r   Nr$   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr.   rr  )rö   rz  r\   rX   rL   Úproblem_typerF  r1   rB   rH   rk   r   Úsqueezer   r£   r   r   rž   rø   ©rK   rP   r~   r0   r,   rQ   r`  r€   ri  r4  rd  re  ru  s                rN   rd   z-Data2VecTextForSequenceClassification.forwardŽ  s¯  € ð$ %�$×$Ñ$Øð
à)Ø)Ø%Ø'Øñ
ð ñ
ˆð " !™*ˆØ—‘ Ó1ˆàˆØÑØ—Y‘Y˜vŸ}™}Ó-ˆFà�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rO   ©NNNNNN)rp   rq   rr   r3   r!   r   rB   rt   ru   r   r   r²   r   rd   rx   ry   s   @rN   rw  rw  |  sÔ   ø„ ô	ð Øð .2Ø37Ø26Ø04Ø26Ø*.ñ:
à×#Ñ# dÑ*ð:
ð ×)Ñ)¨DÑ0ð:
ð ×(Ñ(¨4Ñ/ð	:
ð
 ×&Ñ&¨Ñ-ð:
ð ×(Ñ(¨4Ñ/ð:
ð × Ñ  4Ñ'ð:
ð Ð+Ñ,ð:
ð 
Ð)Ñ	)ò:
ó ó ô:
rO   rw  c                   ó  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
   d
eez  fd„«       «       Zˆ xZS )ÚData2VecTextForMultipleChoicec                 óö   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  «      | _        t	        j                  |j                  d«      | _
        | j                  «        y )Nr$   )r2   r3   r   rö   r4   r>   r?   r@   r˜   r7   rz  r%  rJ   s     €rN   r3   z&Data2VecTextForMultipleChoice.__init__Ï  sW   ø€ Ü‰Ñ˜Ô ä.¨vÓ6ˆÔÜ—z‘z &×"<Ñ"<Ó=ˆŒÜŸ)™) F×$6Ñ$6¸Ó:ˆŒð 	�‰ÕrO   NrP   r0   r~   r`  r,   rQ   r€   rS   c           	      ó"  — |�|j                   d   n|j                   d   }|�!|j                  d|j                  d«      «      nd}	|�!|j                  d|j                  d«      «      nd}
|�!|j                  d|j                  d«      «      nd}|�!|j                  d|j                  d«      «      nd}|�1|j                  d|j                  d«      |j                  d«      «      nd} | j                  |	f|
|||ddœ|¤Ž}|d   }| j	                  |«      }| j                  |«      }|j                  d|«      }d}|�.t        «       }|j                  |j                  «      } |||«      }t        |||j                  |j                  ¬«      S )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        Nr$   r.   éþÿÿÿT)r,   r0   r~   rQ   rc  rr  )r]   r£   rG   rö   r@   rz  r   r\   rX   r   rž   rø   )rK   rP   r0   r~   r`  r,   rQ   r€   Únum_choicesÚflat_input_idsÚflat_position_idsÚflat_token_type_idsÚflat_attention_maskÚflat_inputs_embedsri  r  rd  Úreshaped_logitsre  ru  s                       rN   rd   z%Data2VecTextForMultipleChoice.forwardÙ  s·  € ðT -6Ð,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆàCLÐCX˜Ÿ™¨¨I¯N©N¸2Ó,>Ô?Ð^bˆØLXÐLd˜L×-Ñ-¨b°,×2CÑ2CÀBÓ2GÔHÐjnÐØR`ÐRl˜n×1Ñ1°"°n×6IÑ6IÈ"Ó6MÔNÐrvÐØR`ÐRl˜n×1Ñ1°"°n×6IÑ6IÈ"Ó6MÔNÐrvÐð Ð(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð %�$×$Ñ$Øð
à*Ø.Ø.Ø,Øñ
ð ñ
ˆð   ™
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÐÜ'Ó)ˆHà—Y‘Y˜×5Ñ5Ó6ˆFÙ˜O¨VÓ4ˆDä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
rO   rƒ  )rp   rq   rr   r3   r!   r   rB   rt   ru   r   r   r²   r   rd   rx   ry   s   @rN   r…  r…  Í  sÞ   ø„ ôð Øð .2Ø26Ø37Ø*.Ø04Ø26ñO
à×#Ñ# dÑ*ðO
ð ×(Ñ(¨4Ñ/ðO
ð ×)Ñ)¨DÑ0ð	O
ð
 × Ñ  4Ñ'ðO
ð ×&Ñ&¨Ñ-ðO
ð ×(Ñ(¨4Ñ/ðO
ð Ð+Ñ,ðO
ð 
Ð*Ñ	*òO
ó ó ôO
rO   r…  c                   ó  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
   d
eez  fd„«       «       Zˆ xZS )Ú"Data2VecTextForTokenClassificationc                 ód  •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        |j
                  �|j
                  n|j                  }t        j                  |«      | _	        t        j                  |j                  |j                  «      | _        | j                  «        y ry  )r2   r3   rF  r   rö   rE  r?   r4   r>   r@   r˜   r7   rz  r%  rH  s      €rN   r3   z+Data2VecTextForTokenClassification.__init__/  s‹   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä.¨vÈÔOˆÔà)/×)BÑ)BÐ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z‘zÐ"4Ó5ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÕrO   NrP   r~   r0   r,   rQ   r`  r€   rS   c           	      ó~  —  | j                   |f||||ddœ|¤Ž}|d   }	| j                  |	«      }	| j                  |	«      }
d}|�Wt        «       }|j	                  |
j
                  «      } ||
j                  d| j                  «      |j                  d«      «      }t        ||
|j                  |j                  ¬«      S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Tr|  r   Nr.   rr  )rö   r@   rz  r   r\   rX   r£   rF  r   rž   rø   r‚  s                rN   rd   z*Data2VecTextForTokenClassification.forward=  sË   € ð  %�$×$Ñ$Øð
à)Ø)Ø%Ø'Øñ
ð ñ
ˆð " !™*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÐÜ'Ó)ˆHà—Y‘Y˜vŸ}™}Ó-ˆFÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rO   rƒ  )rp   rq   rr   r3   r!   r   rB   rt   ru   r   r   r²   r   rd   rx   ry   s   @rN   r‘  r‘  -  sÔ   ø„ ôð Øð .2Ø37Ø26Ø04Ø26Ø*.ñ)
à×#Ñ# dÑ*ð)
ð ×)Ñ)¨DÑ0ð)
ð ×(Ñ(¨4Ñ/ð	)
ð
 ×&Ñ&¨Ñ-ð)
ð ×(Ñ(¨4Ñ/ð)
ð × Ñ  4Ñ'ð)
ð Ð+Ñ,ð)
ð 
Ð&Ñ	&ò)
ó ó ô)
rO   r‘  c                   ó0  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
e	e
   deez  fd„«       «       Zˆ xZS )Ú Data2VecTextForQuestionAnsweringc                 óè   •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y ry  )
r2   r3   rF  r   rö   r4   r˜   r7   Ú
qa_outputsr%  rJ   s     €rN   r3   z)Data2VecTextForQuestionAnswering.__init__m  sV   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä.¨vÈÔOˆÔÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÕrO   NrP   r~   r0   r,   rQ   Ústart_positionsÚend_positionsr€   rS   c           	      óÆ  —  | j                   |f||||ddœ|¤Ž}	|	d   }
| j                  |
«      }|j                  dd¬«      \  }}|j                  d«      j	                  «       }|j                  d«      j	                  «       }d }|�·|�µt        |j                  «       «      dkD  r|j                  d«      }t        |j                  «       «      dkD  r|j                  d«      }|j                  d«      }|j                  d|«      }|j                  d|«      }t        |¬«      } |||«      } |||«      }||z   dz  }t        ||||	j                  |	j                  ¬	«      S )
NTr|  r   r$   r.   ri   )Úignore_indexrƒ   )re  Ústart_logitsÚ
end_logitsrž   rø   )rö   r—  Úsplitr�  rŠ   ÚlenrG   Úclampr   r   rž   rø   )rK   rP   r~   r0   r,   rQ   r˜  r™  r€   ri  r4  rd  rœ  r�  Ú
total_lossÚignored_indexru  Ú
start_lossÚend_losss                      rN   rd   z(Data2VecTextForQuestionAnswering.forwardw  s‚  € ð %�$×$Ñ$Øð
à)Ø)Ø%Ø'Øñ
ð ñ
ˆð " !™*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
rO   )NNNNNNN)rp   rq   rr   r3   r!   r   rB   rt   ru   r   r   r²   r   rd   rx   ry   s   @rN   r•  r•  k  sì   ø„ ôð Øð .2Ø37Ø26Ø04Ø26Ø37Ø15ñ3
à×#Ñ# dÑ*ð3
ð ×)Ñ)¨DÑ0ð3
ð ×(Ñ(¨4Ñ/ð	3
ð
 ×&Ñ&¨Ñ-ð3
ð ×(Ñ(¨4Ñ/ð3
ð ×)Ñ)¨DÑ0ð3
ð ×'Ñ'¨$Ñ.ð3
ð Ð+Ñ,ð3
ð 
Ð-Ñ	-ò3
ó ó ô3
rO   r•  )rM  rm  r…  r•  rw  r‘  r   rõ   )Nr¡   )PÚcollections.abcr   rB   Útorch.nnr4   r   r   r   Ú r   rü   Úactivationsr	   r
   Úcache_utilsr   r   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r    Úutils.genericr!   r"   Úutils.output_capturingr#   Úconfiguration_data2vec_textr%   Ú
get_loggerrp   rT  ÚModuler'   rv   Úfloatr�   r�   r´   rÀ   rÍ   rØ   rà   rä   rõ   r	  r  r   r7  rC  rM  rm  rw  r…  r‘  r•  Ú__all__rf  rO   rN   ú<module>r¹     sÂ  ðõ* %ã Ý ß AÑ Aå &ß 'ß CÑ CÝ )ß JÝ 9÷	÷ 	ó 	÷ GÝ &Ý 6ß @Ñ @ß IÝ 5Ý ;ð 
ˆ×	Ñ	˜HÓ	%€ôg8˜RŸY™Yô g8ð` !Øñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ñ(ó%ô8@) §	¡	ô @)ôFI) §¡ô I)ôX˜RŸY™Yô ô.˜BŸI™Iô .ô:˜rŸy™yô ô˜Ÿ™ô ô>Ð2ô >ðB ô/ /ó /ó ð/ô,
˜"Ÿ)™)ô 
ô@˜Ÿ™ô ð ô|6Ð3ó |6ó ð|6ô~˜Ÿ™ô ô, R§Y¡Yô ñ, ðôô
`
Ð9¸?ó `
óð
`
ðF ôI
Ð9ó I
ó ðI
ñX ðôôH
Ð,Gó H
óðH
ðV ô\
Ð$?ó \
ó ð\
ð~ ô:
Ð)Dó :
ó ð:
ðz ô@
Ð'Bó @
ó ð@
òF	�rO   