
    ^j                        d dl mZ d dlZd dlmZ d dlmZmZmZ ddlm	Z
 ddlmZmZ ddlmZmZmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZmZmZmZmZm Z m!Z! ddl"m#Z#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/ ddl0m1Z1 ddl2m3Z3  e,jh                  e5      Z6 G d dejn                        Z8	 	 dIdejn                  dejr                  dejr                  dejr                  dejr                  dz  de:dz  de:de&e*   fdZ; G d d ejn                        Z< G d! d"ejn                        Z= G d# d$ejn                        Z> G d% d&ejn                        Z? G d' d(ejn                        Z@ G d) d*ejn                        ZA G d+ d,e      ZBe+ G d- d.e$             ZC G d/ d0ejn                        ZD G d1 d2ejn                        ZEe+ G d3 d4eC             ZF G d5 d6ejn                        ZG G d7 d8ejn                        ZH e+d9:       G d; d<eCe             ZIe+ G d= d>eC             ZJ e+d?:       G d@ dAeC             ZKe+ G dB dCeC             ZLe+ G dD dEeC             ZMe+ G dF dGeC             ZNg dHZOy)J    )CallableN)BCEWithLogitsLossCrossEntropyLossMSELoss   )initialization)ACT2FNgelu)CacheDynamicCacheEncoderDecoderCache)GenerationMixin)create_bidirectional_maskcreate_causal_mask)GradientCheckpointingLayer))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentions!CausalLMOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)apply_chunking_to_forward)TransformersKwargsauto_docstringlogging)can_return_tuplemerge_with_config_defaults)capture_outputs   )Data2VecTextConfigc                        e Zd ZdZ fdZ	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  ded	ej                  fd
Z
ed        Zedd       Z xZS )Data2VecTextEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 T   t         |           t        j                  |j                  |j
                  |j                        | _        t        j                  |j                  |j
                        | _	        t        j                  |j
                  |j                        | _
        t        j                  |j                        | _        | j                  dt!        j"                  |j$                        j'                  d      d       | j                  dt!        j(                  | j*                  j-                         t         j.                        d       |j                  | _        t        j                  |j$                  |j
                  | j0                        | _        y )	N)padding_idxepsposition_idsr$   F)
persistenttoken_type_ids)dtype)super__init__nn	Embedding
vocab_sizehidden_sizepad_token_idword_embeddingstype_vocab_sizetoken_type_embeddings	LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_buffertorcharangemax_position_embeddingsexpandzerosr,   sizelongr)   position_embeddingsselfconfig	__class__s     ~/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_text.pyr3   zData2VecTextEmbeddings.__init__:   s4   !||F,=,=v?Q?Q_e_r_rs%'\\&2H2H&J\J\%]"f&8&8f>S>STzz&"<"<=ELL)G)GHOOPWXej 	 	
 	ekk$*;*;*@*@*B%**Ubg 	 	
 "..#%<<**F,>,>DL\L\$
     N	input_idsr0   r,   inputs_embedspast_key_values_lengthreturnc                    |<|| j                  || j                  |      }n| j                  || j                        }||j                         }n|j                         d d }|\  }}|t	        | d      rm| j
                  j                  |j                        j                  |j                  d   d      }	t        j                  |	d|      }	|	j                  ||      }n:t        j                  |t        j                  | j                  j                        }|| j                  |      }| j!                  |      }
||
z   }| j#                  |      }||z   }| j%                  |      }| j'                  |      }|S )Nr.   r0   r   r$   )dimindexr1   device)"create_position_ids_from_input_idsr)   &create_position_ids_from_inputs_embedsrG   hasattrr0   torX   rE   shaperB   gatherrF   rH   r,   r9   r;   rI   r<   r@   )rK   rP   r0   r,   rQ   rR   input_shape
batch_size
seq_lengthbuffered_token_type_idsr;   
embeddingsrI   s                rN   forwardzData2VecTextEmbeddings.forwardN   s    $#FFt//1G   $JJ=Z^ZjZjk #..*K',,.s3K!,
J
 !t-.*.*=*=*@*@ATAT*U*\*\]i]o]opq]rtv*w'*/,,7NTU]i*j'!8!?!?
J!W!&[

SWSdSdSkSk!l  00;M $ : :> J"%::
"66|D"55
^^J/
\\*-
rO   c                     | j                         dd }|d   }t        j                  |dz   ||z   dz   t        j                  | j                        }|j                  d      j                  |      S )z
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr.   r$   rW   r   )rG   rB   rC   rH   rX   	unsqueezerE   )rQ   r)   r_   sequence_lengthr,   s        rN   rZ   z=Data2VecTextEmbeddings.create_position_ids_from_inputs_embeds~   sp     $((*3B/%a.||!O_{:Q>ejjYfYmYm
 %%a(//<<rO   c                     | j                  |      j                         }t        j                  |d      j	                  |      |z   |z  }|j                         |z   S )a  
        Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
        are ignored. This is modified from fairseq's `utils.make_positions`.

        Args:
            x: torch.Tensor x:

        Returns: torch.Tensor
        r$   rU   )neintrB   cumsumtype_asrH   )rP   r)   rR   maskincremental_indicess        rN   rY   z9Data2VecTextEmbeddings.create_position_ids_from_input_ids   sW     ||K(,,.$||Da8@@FI__cgg"'')K77rO   )NNNNr   )r   )__name__
__module____qualname____doc__r3   rB   
LongTensorFloatTensorrk   Tensorrd   staticmethodrZ   rY   __classcell__rM   s   @rN   r'   r'   7   s    Q
, .2260426&'.##d*. ((4/. &&-	.
 ((4/. !$. 
.` = =" 8 8rO   r'   modulequerykeyvalueattention_maskscalingr@   kwargsc                    ||j                  d      dz  }t        j                  ||j                  dd            |z  }|||z   }t        j
                  j                  |d      }t        j
                  j                  ||| j                        }t        j                  ||      }	|	j                  dd      j                         }	|	|fS )Nr.            r   ri   )ptrainingr$   )
rG   rB   matmul	transposer4   
functionalsoftmaxr@   r   
contiguous)
rz   r{   r|   r}   r~   r   r@   r   attn_weightsattn_outputs
             rN   eager_attention_forwardr      s     **R.D( <<s}}Q':;gEL!#n4==((2(>L==((6??([L,,|U3K''1-88:K$$rO   c                        e Zd Zd	 fd	Z	 	 d
dej
                  dej                  dz  dedz  dee	   de
ej
                     f
dZ xZS )Data2VecTextSelfAttentionNc                 @   t         |           |j                  |j                  z  dk7  r2t	        |d      s&t        d|j                   d|j                   d      || _        |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _	        | j                  dz  | _
        t        j                  |j                  | j                        | _        t        j                  |j                  | j                        | _        t        j                  |j                  | j                        | _        t        j                   |j"                        | _        |j&                  | _        || _        || _        y Nr   embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()r   )r2   r3   r7   num_attention_headsr[   
ValueErrorrL   rk   attention_head_sizeall_head_sizer   r4   Linearr{   r|   r}   r>   attention_probs_dropout_probr@   
is_decoder	is_causal	layer_idxrK   rL   r   r   rM   s       rN   r3   z"Data2VecTextSelfAttention.__init__   sP    : ::a?PVXhHi#F$6$6#7 8 445Q8  #)#=#= #&v'9'9F<V<V'V#W !558P8PP//5YYv1143E3EF
99V//1C1CDYYv1143E3EF
zz&"E"EF ++""rO   hidden_statesr~   past_key_valuesr   rS   c                     |j                   d d }g |d| j                  } | j                  |      j                  | j	                  dd      } | j                  |      j                  | j	                  dd      } | j                  |      j                  | j	                  dd      }	|>|}
t        |t              r|j                  }
|
j                  ||	| j                        \  }}	t        j                  | j                  j                  t               } || |||	|f| j"                  sdn| j$                  j&                  | j(                  d|\  }} |j*                  g |d j-                         }||fS )Nr.   r$   r           r@   r   )r]   r   r{   viewr   r|   r}   
isinstancer   self_attention_cacheupdater   r   get_interfacerL   _attn_implementationr   r   r@   r   r   reshaper   )rK   r   r~   r   r   r_   hidden_shapequery_layer	key_layervalue_layercurrent_past_key_valuesattention_interfacer   r   s                 rN   rd   z!Data2VecTextSelfAttention.forward   s    $))#2.CCbC$*B*BC 5djj/44lCMMaQRS0DHH]+00,?II!QO	4djj/44lCMMaQRS&&5#/+>?*9*N*N' &=%C%CI{\`\j\j%k"I{(?(M(MKK,,.E)
 %8	%
  $}}C$,,..LL	%
 	%
!\ *k));;;;FFHL((rO   FN)NNrp   rq   rr   r3   rB   rv   ru   r   r   r   tuplerd   rx   ry   s   @rN   r   r      sg    #6 48(,	')||') ))D0') 	')
 +,') 
u||	')rO   r   c                        e Zd Zd
 fd	Z	 	 	 ddej
                  dej                  dz  dej                  dz  dedz  dee	   de
ej
                     fd	Z xZS )Data2VecTextCrossAttentionNc                    t         |           |j                  |j                  z  dk7  r2t	        |d      s&t        d|j                   d|j                   d      || _        |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _	        | j                  dz  | _
        t        j                  |j                  | j                        | _        t        j                  |j                  | j                        | _        t        j                  |j                  | j                        | _        t        j                   |j"                        | _        || _        || _        y r   )r2   r3   r7   r   r[   r   rL   rk   r   r   r   r4   r   r{   r|   r}   r>   r   r@   r   r   r   s       rN   r3   z#Data2VecTextCrossAttention.__init__  sC    : ::a?PVXhHi#F$6$6#7 8 445Q8  #)#=#= #&v'9'9F<V<V'V#W !558P8PP//5YYv1143E3EF
99V//1C1CDYYv1143E3EF
zz&"E"EF""rO   r   encoder_hidden_statesr~   r   r   rS   c                 f   |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }|%|j
                  j                  | j                        nd}	|]|	r[|j                  j                  | j                     j                  }
|j                  j                  | j                     j                  }ng |j                   d d d| j                  }| j                  |      j                  |      j	                  dd      }
| j                  |      j                  |      j	                  dd      }|C|j                  j                  |
|| j                        \  }
}d|j
                  | j                  <   t        j                   | j"                  j$                  t&              } || ||
||f| j(                  sdn| j*                  j,                  | j.                  d|\  }} |j0                  g |d j3                         }||fS )Nr.   r$   r   FTr   r   )r]   r   r{   r   r   
is_updatedgetr   cross_attention_cachelayerskeysvaluesr|   r}   r   r   r   rL   r   r   r   r@   r   r   r   r   )rK   r   r   r~   r   r   r_   r   r   r   r   r   kv_shaper   r   r   s                   rN   rd   z"Data2VecTextCrossAttention.forward  s    $))#2.CCbC$*B*BC jj/44\BLLQPQRGVGb_//33DNNChm
&:'==DDT^^TYYI)??FFt~~V]]KX.44Sb9X2Xt?W?WXH!67<<XFPPQRTUVI**%:;@@JTTUVXYZK*)8)N)N)U)U{DNN*&	; >B**4>>:(?(M(MKK,,.E)
 %8	%
  $}}C$,,..LL	%
 	%
!\ *k));;;;FFHL((rO   r   )NNN)rp   rq   rr   r3   rB   rv   ru   r   r   r   r   rd   rx   ry   s   @rN   r   r      s    #4 ;?376:1)||1)  %00471) ))D0	1)
 -t31) +,1) 
u||	1)rO   r   c                   n     e Zd Z fdZdej
                  dej
                  dej
                  fdZ xZS )Data2VecTextSelfOutputc                 (   t         |           t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                        | _
        y Nr*   )r2   r3   r4   r   r7   denser<   r=   r>   r?   r@   rJ   s     rN   r3   zData2VecTextSelfOutput.__init__M  s`    YYv1163E3EF
f&8&8f>S>STzz&"<"<=rO   r   input_tensorrS   c                 r    | j                  |      }| j                  |      }| j                  ||z         }|S Nr   r@   r<   rK   r   r   s      rN   rd   zData2VecTextSelfOutput.forwardS  7    

=1]3}|'CDrO   rp   rq   rr   r3   rB   rv   rd   rx   ry   s   @rN   r   r   L  1    >U\\  RWR^R^ rO   r   c                        e Zd Zd fd	Z	 	 	 	 ddej
                  dej                  dz  dej                  dz  dej                  dz  dedz  dee	   d	e
ej
                     fd
Z xZS )Data2VecTextAttentionNc                     t         |           || _        |rt        nt        } ||||      | _        t        |      | _        y )Nr   r   )r2   r3   is_cross_attentionr   r   rK   r   output)rK   rL   r   r   r   attention_classrM   s         rN   r3   zData2VecTextAttention.__init__[  s=    "48J4Pi#Fi9U	,V4rO   r   r~   r   encoder_attention_maskr   r   rS   c                     | j                   s|n|} | j                  |f|||d|\  }}| j                  ||      }||fS )N)r   r~   r   )r   rK   r   )	rK   r   r~   r   r   r   r   attention_outputr   s	            rN   rd   zData2VecTextAttention.forwardb  sd     04/F/FLb)2*
"7)+	*

 *
&,  ;;'7G--rO   )FNFNNNNr   ry   s   @rN   r   r   Z  s    5 48:>;?(,.||. ))D0.  %0047	.
 !& 1 1D 8. . +,. 
u||	.rO   r   c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )Data2VecTextIntermediatec                    t         |           t        j                  |j                  |j
                        | _        t        |j                  t              rt        |j                     | _        y |j                  | _        y r   )r2   r3   r4   r   r7   intermediate_sizer   r   
hidden_actstrr	   intermediate_act_fnrJ   s     rN   r3   z!Data2VecTextIntermediate.__init__x  s]    YYv1163K3KL
f''-'-f.?.?'@D$'-'8'8D$rO   r   rS   c                 J    | j                  |      }| j                  |      }|S r   )r   r   )rK   r   s     rN   rd   z Data2VecTextIntermediate.forward  s&    

=100?rO   r   ry   s   @rN   r   r   w  s#    9U\\ ell rO   r   c                   n     e Zd Z fdZdej
                  dej
                  dej
                  fdZ xZS )Data2VecTextOutputc                 (   t         |           t        j                  |j                  |j
                        | _        t        j                  |j
                  |j                        | _        t        j                  |j                        | _        y r   )r2   r3   r4   r   r   r7   r   r<   r=   r>   r?   r@   rJ   s     rN   r3   zData2VecTextOutput.__init__  s`    YYv779K9KL
f&8&8f>S>STzz&"<"<=rO   r   r   rS   c                 r    | j                  |      }| j                  |      }| j                  ||z         }|S r   r   r   s      rN   rd   zData2VecTextOutput.forward  r   rO   r   ry   s   @rN   r   r     r   rO   r   c                        e Zd Zd fd	Z	 	 	 	 ddej
                  dej                  dz  dej                  dz  dej                  dz  dedz  dee	   d	ej
                  fd
Z
d Z xZS )Data2VecTextLayerNc                    t         |           |j                  | _        d| _        t	        ||j
                  |      | _        |j
                  | _        |j                  | _        | j                  r.| j
                  st        |  d      t	        |d|d      | _	        t        |      | _        t        |      | _        y )Nr$   r   z> should be used as a decoder model if cross attention is addedFT)r   r   r   )r2   r3   chunk_size_feed_forwardseq_len_dimr   r   	attentionadd_cross_attentionr   crossattentionr   intermediater   r   )rK   rL   r   rM   s      rN   r3   zData2VecTextLayer.__init__  s    '-'E'E$.vARAR^gh ++#)#=#= ##?? D6)g!hii"7##'	#D 5V<(0rO   r   r~   r   r   r   r   rS   c                      | j                   ||fd|i|\  }}|}	| j                  r:|8t        | d      st        d|  d       | j                  |d ||fd|i|\  }
}|
}	t        | j                  | j                  | j                  |	      }|S )Nr   r   z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`)	r   r   r[   r   r   r   feed_forward_chunkr   r   )rK   r   r~   r   r   r   r   self_attention_output_r   cross_attention_outputlayer_outputs               rN   rd   zData2VecTextLayer.forward  s     $24>>$
 ,$
 	$
 q 1??4@4!12 =dV DD D 
 )<(;(;%%&	)
 !0) )%"A  60##T%A%A4CSCSUe
 rO   c                 L    | j                  |      }| j                  ||      }|S r   )r   r   )rK   r   intermediate_outputr   s       rN   r   z$Data2VecTextLayer.feed_forward_chunk  s,    "//0@A{{#68HIrO   r   r   )rp   rq   rr   r3   rB   rv   ru   r   r   r   rd   r   rx   ry   s   @rN   r   r     s    1, 48:>;?(,%||% ))D0%  %0047	%
 !& 1 1D 8% % +,% 
%NrO   r   c                   N     e Zd ZeZdZdZddgZdZdZ	dZ
dZeeedZ fdZ xZS )Data2VecTextPreTrainedModeldata2vec_textTData2VecTextForTextEmbeddingsr   )r   
attentionscross_attentionsc                 6   t         |   |       t        |t              ryt	        j
                  |j                  t        j                  |j                  j                  d         j                  d             t	        j                  |j                         y y )Nr.   r-   )r2   _init_weightsr   r'   initcopy_r,   rB   rC   r]   rE   zeros_r0   )rK   rz   rM   s     rN   r   z)Data2VecTextPreTrainedModel._init_weights  sm    f%f45JJv**ELL9L9L9R9RSU9V,W,^,^_f,ghKK--. 6rO   )rp   rq   rr   r%   config_classbase_model_prefixsupports_gradient_checkpointing_no_split_modules_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendr   r   r   _can_record_outputsr   rx   ry   s   @rN   r   r     sR    %L'&*#8:MNN"&*/6/ /rO   r   c                        e Zd Z fdZ	 	 	 	 	 ddej
                  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  d	e	e
   d
eej
                     ez  fdZ xZS )Data2VecTextEncoderc           	          t         |           || _        t        j                  t        |j                        D cg c]  }t        ||       c}      | _        y c c}w )N)r   )	r2   r3   rL   r4   
ModuleListrangenum_hidden_layersr   layer)rK   rL   irM   s      rN   r3   zData2VecTextEncoder.__init__  sH    ]]TYZ`ZrZrTs#tq$5f$J#tu
#ts   ANr   r~   r   r   r   	use_cacher   rS   c                     t        | j                        D ]  \  }}	 |	|||f||d|} t        ||r|      S d       S )N)r   r   )last_hidden_stater   )	enumerater  r   )
rK   r   r~   r   r   r   r  r   r  layer_modules
             rN   rd   zData2VecTextEncoder.forward  sn      )4 	OA|(% (> / M	 9+/8O
 	
>B
 	
rO   )NNNNN)rp   rq   rr   r3   rB   rv   ru   r   boolr   r   r   r   rd   rx   ry   s   @rN   r	  r	    s    v 48:>;?(,!%
||
 ))D0
  %0047	

 !& 1 1D 8
 
 $;
 +,
 
u||	H	H
rO   r	  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )Data2VecTextPoolerc                     t         |           t        j                  |j                  |j                        | _        t        j                         | _        y r   )r2   r3   r4   r   r7   r   Tanh
activationrJ   s     rN   r3   zData2VecTextPooler.__init__  s9    YYv1163E3EF
'')rO   r   rS   c                 \    |d d df   }| j                  |      }| j                  |      }|S Nr   )r   r  )rK   r   first_token_tensorpooled_outputs       rN   rd   zData2VecTextPooler.forward  s6     +1a40

#566rO   r   ry   s   @rN   r  r    s#    $
U\\ ell rO   r  c                       e Zd ZddgZd fd	Zd Zd Zeee		 	 	 	 	 	 	 	 	 dde
j                  dz  de
j                  dz  d	e
j                  dz  d
e
j                  dz  de
j                  dz  de
j                  dz  de
j                  dz  dedz  dedz  dee   dee
j                     ez  fd                     Zd Z xZS )Data2VecTextModelr'   r   c                     t         |   |       || _        d| _        t	        |      | _        t        |      | _        |rt        |      nd| _	        | j                          y)zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        FN)r2   r3   rL   gradient_checkpointingr'   rc   r	  encoderr  pooler	post_init)rK   rL   add_pooling_layerrM   s      rN   r3   zData2VecTextModel.__init__  sV    
 	 &+#08*624E(04 	rO   c                 .    | j                   j                  S r   rc   r9   rK   s    rN   get_input_embeddingsz&Data2VecTextModel.get_input_embeddings0  s    ...rO   c                 &    || j                   _        y r   r(  )rK   r}   s     rN   set_input_embeddingsz&Data2VecTextModel.set_input_embeddings3  s    */'rO   NrP   r~   r0   r,   rQ   r   r   r   r  r   rS   c
           
         |d u |d uz  rt        d      | j                  j                  r|	|	n| j                  j                  }	nd}	|	rd|b|| j                  j                  r4t        t        | j                        t        | j                              nt        | j                        }||j                         nd}| j                  |||||      }| j                  |||||      \  }} | j                  |f|||||	|d|
}|j                  }| j                  | j                  |      nd }t        |||j                        S )	Nz:You must specify exactly one of input_ids or inputs_embedsF)rL   r   )rP   r,   r0   rQ   rR   )r~   r   embedding_outputr   r   )r~   r   r   r   r  r,   )r  pooler_outputr   )r   rL   r   r  is_encoder_decoderr   r   get_seq_lengthrc   _create_attention_masksr#  r  r$  r   r   )rK   rP   r~   r0   r,   rQ   r   r   r   r  r   rR   r.  encoder_outputssequence_outputr  s                   rN   rd   zData2VecTextModel.forward6  s|     -t";<YZZ;;!!%.%:	@U@UII0 )48V8V $L$DlZ^ZeZeFfg!5  FUE`!?!?!Afg??%)'#9 + 
 261M1M)#9-"7+ 2N 2
.. '$,,	
)"7#9+%	
 	
 *;;8<8OO4UY;-'+;;
 	
rO   c                     | j                   j                  rt        | j                   |||      }nt        | j                   ||      }|t        | j                   |||      }||fS )N)rL   rQ   r~   r   )rL   rQ   r~   )rL   rQ   r~   r   )rL   r   r   r   )rK   r~   r   r.  r   r   s         rN   r2  z)Data2VecTextModel._create_attention_masksz  su     ;;!!/{{.- /	N 7{{.-N "-%>{{.5&;	&" 555rO   )T)	NNNNNNNNN)rp   rq   rr   r  r3   r*  r,  r"   r#   r   rB   rv   r   r  r   r   r   r   rd   r2  rx   ry   s   @rN   r   r     s-   13FG"/0   *..2.2,0-1596:(,!%?
<<$&?
 t+?
 t+	?

 llT)?
 ||d*?
  %||d2?
 !&t 3?
 ?
 $;?
 +,?
 
u||	K	K?
    ?
B6rO   r   c                   (     e Zd ZdZ fdZd Z xZS )Data2VecTextLMHeadz/Data2VecText Head for masked language modeling.c                    t         |           t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _
        t        j                  t        j                  |j                              | _        y r   )r2   r3   r4   r   r7   r   r<   r=   
layer_normr6   decoder	ParameterrB   rF   biasrJ   s     rN   r3   zData2VecTextLMHead.__init__  s    YYv1163E3EF
,,v'9'9v?T?TUyy!3!3V5F5FGLLV->->!?@	rO   c                     | j                  |      }t        |      }| j                  |      }| j                  |      }|S r   )r   r
   r9  r:  rK   featuresr   xs       rN   rd   zData2VecTextLMHead.forward  s;    JJx GOOA LLOrO   rp   rq   rr   rs   r3   rd   rx   ry   s   @rN   r7  r7    s    9ArO   r7  c                   (     e Zd ZdZ fdZd Z xZS )Data2VecTextClassificationHeadz-Head for sentence-level classification tasks.c                 Z   t         |           t        j                  |j                  |j                        | _        |j                  |j                  n|j                  }t        j                  |      | _	        t        j                  |j                  |j                        | _        y r   )r2   r3   r4   r   r7   r   classifier_dropoutr?   r>   r@   
num_labelsout_projrK   rL   rE  rM   s      rN   r3   z'Data2VecTextClassificationHead.__init__  s    YYv1163E3EF
)/)B)B)NF%%TZTnTn 	 zz"45		&"4"4f6G6GHrO   c                     |d d dd d f   }| j                  |      }| j                  |      }t        j                  |      }| j                  |      }| j	                  |      }|S r  )r@   r   rB   tanhrG  r>  s       rN   rd   z&Data2VecTextClassificationHead.forward  sY    Q1WLLOJJqMJJqMLLOMM!rO   rA  ry   s   @rN   rC  rC    s    7IrO   rC  zX
    Data2VecText Model with a `language modeling` head on top for CLM fine-tuning.
    )custom_introc                       e Zd ZdddZ fdZd Zd Zee	 	 	 	 	 	 	 	 	 	 	 dde	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  deee	j                        dz  dedz  dee	j                  z  dee   deez  fd              Z xZS )Data2VecTextForCausalLM/data2vec_text.embeddings.word_embeddings.weightlm_head.biaszlm_head.decoder.weightzlm_head.decoder.biasc                     t         |   |       |j                  st        j	                  d       t        |d      | _        t        |      | _        | j                          y )NzTIf you want to use `Data2VecTextLMHeadModel` as a standalone, add `is_decoder=True.`Fr&  
r2   r3   r   loggerwarningr   r   r7  lm_headr%  rJ   s     rN   r3   z Data2VecTextForCausalLM.__init__  sM       NNqr.vO)&1 	rO   c                 .    | j                   j                  S r   rV  r:  r)  s    rN   get_output_embeddingsz-Data2VecTextForCausalLM.get_output_embeddings      ||###rO   c                 &    || j                   _        y r   rX  rK   new_embeddingss     rN   set_output_embeddingsz-Data2VecTextForCausalLM.set_output_embeddings      -rO   NrP   r~   r0   r,   rQ   r   r   labelsr   r  logits_to_keepr   rS   c                    |d}
 | j                   |f|||||||	|
dd	|}|j                  }t        |t              rt	        | d      n|}| j                  |dd|ddf         }d}|* | j                  d||| j                  j                  d|}t        |||j                  |j                  |j                  |j                        S )aA  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in
            `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are
            ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Data2VecTextForCausalLM, Data2VecTextConfig
        >>> import torch

        >>> tokenizer = AutoTokenizer.from_pretrained("facebook/data2vec-text-base")
        >>> config = Data2VecTextConfig.from_pretrained("facebook/data2vec-text-base")
        >>> config.is_decoder = True
        >>> model = Data2VecTextForCausalLM.from_pretrained("facebook/data2vec-text-base", config=config)

        >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
        >>> outputs = model(**inputs)

        >>> prediction_logits = outputs.logits
        ```NFT)	r~   r0   r,   rQ   r   r   r   r  return_dict)logitsr`  r6   )lossrd  r   r   r   r    )r   r  r   rk   slicerV  loss_functionrL   r6   r   r   r   r   r   )rK   rP   r~   r0   r,   rQ   r   r   r`  r   r  ra  r   outputsr   slice_indicesrd  re  s                     rN   rd   zData2VecTextForCausalLM.forward  s    N I@R@R@RA
))%'"7#9+A
 A
  118B>SV8W~ot4]kmA}a,?@A%4%%pVFt{{OeOepiopD0#33!//))$55
 	
rO   )NNNNNNNNNNr   )rp   rq   rr   _tied_weights_keysr3   rY  r^  r!   r   rB   rt   ru   r   r  rk   rv   r   r   r   rd   rx   ry   s   @rN   rM  rM    sw    #T .

$.  .237260426:>;?*.BF!%-.F
##d*F
 ))D0F
 ((4/	F

 &&-F
 ((4/F
  %0047F
 !& 1 1D 8F
   4'F
 uU%6%6784?F
 $;F
 ell*F
 +,F
 
2	2F
  F
rO   rM  c                   f    e Zd ZdddZ fdZd Zd Zee	 	 	 	 	 	 	 	 dde	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  dee   deez  fd              Z xZS )Data2VecTextForMaskedLMrN  rO  rP  c                     t         |   |       |j                  rt        j	                  d       t        |d      | _        t        |      | _        | j                          y )NzsIf you want to use `Data2VecTextForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.FrR  rS  rJ   s     rN   r3   z Data2VecTextForMaskedLM.__init__6  sS     NN1
 /vO)&1 	rO   c                 .    | j                   j                  S r   rX  r)  s    rN   rY  z-Data2VecTextForMaskedLM.get_output_embeddingsE  rZ  rO   c                 &    || j                   _        y r   rX  r\  s     rN   r^  z-Data2VecTextForMaskedLM.set_output_embeddingsH  r_  rO   NrP   r~   r0   r,   rQ   r   r   r`  r   rS   c	                 t    | j                   |f||||||dd|	}
|
d   }| j                  |      }d}|at               }|j                  |j                        } ||j                  d| j                  j                        |j                  d            }t        |||
j                  |
j                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        T)r~   r0   r,   rQ   r   r   rc  r   Nr.   re  rd  r   r   )r   rV  r   r\   rX   r   rL   r6   r   r   r   )rK   rP   r~   r0   r,   rQ   r   r   r`  r   ri  r4  prediction_scoresmasked_lm_lossloss_fcts                  rN   rd   zData2VecTextForMaskedLM.forwardK  s    ( %$$$

))%'"7#9

 

 "!* LL9')HYY0778F%&7&<&<RAWAW&XZ`ZeZefhZijN$!//))	
 	
rO   )NNNNNNNN)rp   rq   rr   rk  r3   rY  r^  r!   r   rB   rt   ru   r   r   r   r   rd   rx   ry   s   @rN   rm  rm  /  s    #T .
$.  .237260426:>;?*.,
##d*,
 ))D0,
 ((4/	,

 &&-,
 ((4/,
  %0047,
 !& 1 1D 8,
   4',
 +,,
 
	,
  ,
rO   rm  z
    Data2VecText Model transformer with a sequence classification/regression head on top (a linear layer on top of the
    pooled output) e.g. for GLUE tasks.
    c                       e Zd Z fdZee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
   d
eez  fd              Z xZS )%Data2VecTextForSequenceClassificationc                     t         |   |       |j                  | _        || _        t	        |d      | _        t        |      | _        | j                          y NFrR  )	r2   r3   rF  rL   r   r   rC  
classifierr%  rJ   s     rN   r3   z.Data2VecTextForSequenceClassification.__init__  sK      ++.vO8@ 	rO   NrP   r~   r0   r,   rQ   r`  r   rS   c           	          | j                   |f||||dd|}|d   }	| j                  |	      }
d}||j                  |
j                        }| j                  j
                  | j                  dk(  rd| j                  _        nl| j                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                  _        nd| j                  _        | j                  j
                  dk(  rIt               }| j                  dk(  r& ||
j                         |j                               }n ||
|      }n| j                  j
                  dk(  r=t               } ||
j                  d	| j                        |j                  d	            }n,| j                  j
                  dk(  rt               } ||
|      }t!        ||
|j"                  |j$                  
      S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        Tr~   r0   r,   rQ   rc  r   Nr$   
regressionsingle_label_classificationmulti_label_classificationr.   rr  )r   rz  r\   rX   rL   problem_typerF  r1   rB   rH   rk   r   squeezer   r   r   r   r   r   rK   rP   r~   r0   r,   rQ   r`  r   ri  r4  rd  re  ru  s                rN   rd   z-Data2VecTextForSequenceClassification.forward  s   $ %$$$
))%'
 
 "!*1YYv}}-F{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#FF3D))-JJ+-B @&++b/R))-II,./'!//))	
 	
rO   NNNNNN)rp   rq   rr   r3   r!   r   rB   rt   ru   r   r   r   r   rd   rx   ry   s   @rN   rw  rw  |  s    	  .237260426*.:
##d*:
 ))D0:
 ((4/	:

 &&-:
 ((4/:
   4':
 +,:
 
)	):
  :
rO   rw  c                       e Zd Z fdZee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
   d
eez  fd              Z xZS )Data2VecTextForMultipleChoicec                     t         |   |       t        |      | _        t	        j
                  |j                        | _        t	        j                  |j                  d      | _
        | j                          y )Nr$   )r2   r3   r   r   r4   r>   r?   r@   r   r7   rz  r%  rJ   s     rN   r3   z&Data2VecTextForMultipleChoice.__init__  sW     .v6zz&"<"<=))F$6$6: 	rO   NrP   r0   r~   r`  r,   rQ   r   rS   c           	      "   ||j                   d   n|j                   d   }|!|j                  d|j                  d            nd}	|!|j                  d|j                  d            nd}
|!|j                  d|j                  d            nd}|!|j                  d|j                  d            nd}|1|j                  d|j                  d      |j                  d            nd} | j                  |	f|
|||dd|}|d   }| j	                  |      }| j                  |      }|j                  d|      }d}|.t               }|j                  |j                        } |||      }t        |||j                  |j                        S )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        Nr$   r.   T)r,   r0   r~   rQ   rc  rr  )r]   r   rG   r   r@   rz  r   r\   rX   r   r   r   )rK   rP   r0   r~   r`  r,   rQ   r   num_choicesflat_input_idsflat_position_idsflat_token_type_idsflat_attention_maskflat_inputs_embedsri  r  rd  reshaped_logitsre  ru  s                       rN   rd   z%Data2VecTextForMultipleChoice.forward  s   T -6,Aiooa(}GZGZ[\G]CLCXINN2,>?^bLXLdL--b,2C2CB2GHjnR`Rln11"n6I6I"6MNrvR`Rln11"n6I6I"6MNrv ( r=#5#5b#9=;M;Mb;QR 	 %$$$
*..,
 
  
]3/ ++b+6')HYY556FOV4D("!//))	
 	
rO   r  )rp   rq   rr   r3   r!   r   rB   rt   ru   r   r   r   r   rd   rx   ry   s   @rN   r  r    s      .22637*.0426O
##d*O
 ((4/O
 ))D0	O

   4'O
 &&-O
 ((4/O
 +,O
 
*	*O
  O
rO   r  c                       e Zd Z fdZee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
   d
eez  fd              Z xZS )"Data2VecTextForTokenClassificationc                 d   t         |   |       |j                  | _        t        |d      | _        |j
                  |j
                  n|j                  }t        j                  |      | _	        t        j                  |j                  |j                        | _        | j                          y ry  )r2   r3   rF  r   r   rE  r?   r4   r>   r@   r   r7   rz  r%  rH  s      rN   r3   z+Data2VecTextForTokenClassification.__init__/  s      ++.vO)/)B)B)NF%%TZTnTn 	 zz"45))F$6$68I8IJ 	rO   NrP   r~   r0   r,   rQ   r`  r   rS   c           	      ~    | j                   |f||||dd|}|d   }	| j                  |	      }	| j                  |	      }
d}|Wt               }|j	                  |
j
                        } ||
j                  d| j                        |j                  d            }t        ||
|j                  |j                        S )z
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Tr|  r   Nr.   rr  )r   r@   rz  r   r\   rX   r   rF  r   r   r   r  s                rN   rd   z*Data2VecTextForTokenClassification.forward=  s      %$$$
))%'
 
 "!*,,71')HYYv}}-FFKKDOO<fkk"oND$!//))	
 	
rO   r  )rp   rq   rr   r3   r!   r   rB   rt   ru   r   r   r   r   rd   rx   ry   s   @rN   r  r  -  s      .237260426*.)
##d*)
 ))D0)
 ((4/	)

 &&-)
 ((4/)
   4')
 +,)
 
&	&)
  )
rO   r  c                   0    e Zd Z fdZee	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
e	e
   deez  fd              Z xZS ) Data2VecTextForQuestionAnsweringc                     t         |   |       |j                  | _        t        |d      | _        t        j                  |j                  |j                        | _        | j                          y ry  )
r2   r3   rF  r   r   r4   r   r7   
qa_outputsr%  rJ   s     rN   r3   z)Data2VecTextForQuestionAnswering.__init__m  sV      ++.vO))F$6$68I8IJ 	rO   NrP   r~   r0   r,   rQ   start_positionsend_positionsr   rS   c           	          | j                   |f||||dd|}	|	d   }
| j                  |
      }|j                  dd      \  }}|j                  d      j	                         }|j                  d      j	                         }d }||t        |j                               dkD  r|j                  d      }t        |j                               dkD  r|j                  d      }|j                  d      }|j                  d|      }|j                  d|      }t        |      } |||      } |||      }||z   dz  }t        ||||	j                  |	j                  	      S )
NTr|  r   r$   r.   ri   )ignore_indexr   )re  start_logits
end_logitsr   r   )r   r  splitr  r   lenrG   clampr   r   r   r   )rK   rP   r~   r0   r,   rQ   r  r  r   ri  r4  rd  r  r  
total_lossignored_indexru  
start_lossend_losss                      rN   rd   z(Data2VecTextForQuestionAnswering.forwardw  s    %$$$
))%'
 
 "!*1#)<<r<#: j#++B/::<''+668

&=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9(--a0M-33A}EO)//=AM']CH!,@J
M:H$x/14J+%!!//))
 	
rO   )NNNNNNN)rp   rq   rr   r3   r!   r   rB   rt   ru   r   r   r   r   rd   rx   ry   s   @rN   r  r  k  s      .23726042637153
##d*3
 ))D03
 ((4/	3

 &&-3
 ((4/3
 ))D03
 ''$.3
 +,3
 
-	-3
  3
rO   r  )rM  rm  r  r  rw  r  r   r   )Nr   )Pcollections.abcr   rB   torch.nnr4   r   r   r    r   r   activationsr	   r
   cache_utilsr   r   r   
generationr   masking_utilsr   r   modeling_layersr   modeling_outputsr   r   r   r   r   r   r   r   modeling_utilsr   r   processing_utilsr   pytorch_utilsr   utilsr   r   r    utils.genericr!   r"   utils.output_capturingr#   configuration_data2vec_textr%   
get_loggerrp   rT  Moduler'   rv   floatr   r   r   r   r   r   r   r   r   r	  r  r   r7  rC  rM  rm  rw  r  r  r  __all__rf  rO   rN   <module>r     s  * %   A A & ' C C ) J 9	 	 	 G & 6 @ @ I 5 ; 
		H	%g8RYY g8` !%II%<<% 
% <<	%
 LL4'% T\% % '(%8@)		 @)FI) I)XRYY .BII .:ryy  >2 >B // / /,
")) 
@  |63 |6 |6~ ,RYY , 
`
9? `

`
F I
9 I
 I
X H
,G H
H
V \
$? \
 \
~ :
)D :
 :
z @
'B @
 @
F	rO   