Ë
    ÛÓ^jo¯  ã                   óJ  — d Z ddlZddlmZ ddlZddlmZ ddlmZmZm	Z	 ddl
mZ ddlmZmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZmZ ddlmZ ddlmZ ddlm Z  ddl!m"Z"m#Z#m$Z$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+  e%jX                  e-«      Z. G d„ dej^                  «      Z0 G d„ dej^                  «      Z1 G d„ dej^                  «      Z2 G d„ dej^                  «      Z3 G d„ dej^                  «      Z4 G d„ dej^                  «      Z5 G d „ d!ej^                  «      Z6 G d"„ d#ej^                  «      Z7 G d$„ d%e«      Z8e# G d&„ d'e«      «       Z9 G d(„ d)ej^                  «      Z: G d*„ d+ej^                  «      Z; G d,„ d-ej^                  «      Z<e# G d.„ d/e9«      «       Z= G d0„ d1ej^                  «      Z>e# G d2„ d3e9«      «       Z? G d4„ d5ej^                  «      Z@ e#d6¬7«       G d8„ d9e9«      «       ZAe# G d:„ d;e9«      «       ZBe# G d<„ d=e9«      «       ZCe# G d>„ d?e9«      «       ZDg d@¢ZEy)AzPyTorch ConvBERT model.é    N)ÚCallable)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FNÚget_activation)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)Ú"BaseModelOutputWithCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)ÚUnpack)Úapply_chunking_to_forward)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚConvBertConfigc                   óÄ   ‡ — e Zd ZdZˆ fd„Z	 	 	 	 d
dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  f
d	„Zˆ xZ	S )ÚConvBertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  |j                  ¬«      | _        t        j                  |j                  |j
                  «      | _	        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        | j#                  dt%        j&                  |j                  «      j)                  d«      d¬«       | j#                  dt%        j*                  | j,                  j/                  «       t$        j0                  ¬«      d¬«       y )	N)Úpadding_idx©ÚepsÚposition_ids©r   éÿÿÿÿF)Ú
persistentÚtoken_type_ids©Údtype)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚembedding_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferÚtorchÚarangeÚexpandÚzerosr%   ÚsizeÚlong©ÚselfÚconfigÚ	__class__s     €úy/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/convbert/modeling_convbert.pyr-   zConvBertEmbeddings.__init__7   s  ø€ Ü‰ÑÔÜ!Ÿ|™|¨F×,=Ñ,=¸v×?TÑ?TÐbh×buÑbuÔvˆÔÜ#%§<¡<°×0NÑ0NÐPV×PeÑPeÓ#fˆÔ Ü%'§\¡\°&×2HÑ2HÈ&×J_ÑJ_Ó%`ˆÔ"äŸ™ f×&;Ñ&;À×AVÑAVÔWˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒà×ÑØœEŸL™L¨×)GÑ)GÓH×OÑOÐPWÓXÐejð 	ô 	
ð 	×ÑØœeŸk™k¨$×*;Ñ*;×*@Ñ*@Ó*BÌ%Ï*É*ÔUÐbgð 	õ 	
ó    NÚ	input_idsr)   r%   Úinputs_embedsÚreturnc                 ó2  — |�|j                  «       }n|j                  «       d d }|d   }|€| j                  d d …d |…f   }|€st        | d«      r-| j                  d d …d |…f   }|j	                  |d   |«      }|}n:t        j                  |t
        j                  | j                  j                  ¬«      }|€| j                  |«      }| j                  |«      }	| j                  |«      }
||	z   |
z   }| j                  |«      }| j                  |«      }|S )Nr'   r   r)   r   ©r+   Údevice)rA   r%   Úhasattrr)   r?   r=   r@   rB   rN   r2   r4   r6   r7   r;   )rD   rI   r)   r%   rJ   Úinput_shapeÚ
seq_lengthÚbuffered_token_type_idsÚ buffered_token_type_ids_expandedr4   r6   Ú
embeddingss               rG   ÚforwardzConvBertEmbeddings.forwardG   s,  € ð Ð Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà  ‘^ˆ
àÐØ×,Ñ,ªQ°°°¨^Ñ<ˆLð
 Ð!Ü�tÐ-Ô.Ø*.×*=Ñ*=ºaÀÀ*À¸nÑ*MÐ'Ø3J×3QÑ3QÐR]Ð^_ÑR`ÐblÓ3mÐ0Ø!A‘ä!&§¡¨[ÄÇ
Á
ÐSW×SdÑSd×SkÑSkÔ!l�àÐ Ø ×0Ñ0°Ó;ˆMØ"×6Ñ6°|ÓDÐØ $× :Ñ :¸>Ó JÐà"Ð%8Ñ8Ð;PÑPˆ
Ø—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐrH   )NNNN)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__r-   r=   Ú
LongTensorÚFloatTensorrU   Ú__classcell__©rF   s   @rG   r    r    4   s€   ø„ ÙQô
ð$ .2Ø26Ø04Ø26ñ$à×#Ñ# dÑ*ð$ð ×(Ñ(¨4Ñ/ð$ð ×&Ñ&¨Ñ-ð	$ð
 ×(Ñ(¨4Ñ/ð$ð 
×	Ñ	÷$rH   r    c                   óZ   ‡ — e Zd ZdZˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚSeparableConv1DzSThis class implements separable convolution, i.e. a depthwise and a pointwise layerc                 óì  •— t         ‰| �  «        t        j                  |||||dz  d¬«      | _        t        j                  ||dd¬«      | _        t        j                  t        j                  |d«      «      | _	        | j                  j                  j                  j                  d|j                  ¬«       | j
                  j                  j                  j                  d|j                  ¬«       y )Né   F)Úkernel_sizeÚgroupsÚpaddingÚbiasr   )rb   re   ç        ©ÚmeanÚstd)r,   r-   r   ÚConv1dÚ	depthwiseÚ	pointwiseÚ	Parameterr=   r@   re   ÚweightÚdataÚnormal_Úinitializer_range)rD   rE   Úinput_filtersÚoutput_filtersrb   ÚkwargsrF   s         €rG   r-   zSeparableConv1D.__init__q   s»   ø€ Ü‰ÑÔÜŸ™ØØØ#Ø Ø 1Ñ$Øô
ˆŒô Ÿ™ =°.ÈaÐV[Ô\ˆŒÜ—L‘L¤§¡¨^¸QÓ!?Ó@ˆŒ	à�‰×Ñ×"Ñ"×*Ñ*°¸×9QÑ9QÐ*ÔRØ�‰×Ñ×"Ñ"×*Ñ*°¸×9QÑ9QÐ*ÕRrH   Úhidden_statesrK   c                 óh   — | j                  |«      }| j                  |«      }|| j                  z  }|S ©N)rk   rl   re   )rD   ru   Úxs      rG   rU   zSeparableConv1D.forward�   s0   € Ø�N‰N˜=Ó)ˆØ�N‰N˜1ÓˆØ	ˆT�Y‰Y‰ˆØˆrH   ©	rV   rW   rX   rY   r-   r=   ÚTensorrU   r\   r]   s   @rG   r_   r_   n   s'   ø„ Ù]ôSð  U§\¡\ð °e·l±l÷ rH   r_   c                   óÀ   ‡ — e Zd Zˆ fd„Z	 	 d	dej
                  dej                  dz  dej
                  dz  dee   de	ej
                  ej
                  f   f
d„Z
ˆ xZS )
ÚConvBertSelfAttentionc                 ój  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|j                  |j                  z  }|dk  r|j                  | _        d| _        n|| _        |j                  | _        |j                  | _        |j                  | j                  z  dk7  rt        d«      ‚|j                  | j                  z  dz  | _        | j                  | j                  z  | _	        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        ||j                  | j                  | j                  «      | _        t        j                  | j                  | j                  | j                  z  «      | _        t        j                  |j                  | j                  «      | _        t        j&                  | j                  dgt)        | j                  dz
  dz  «      dg¬	«      | _        t        j,                  |j.                  «      | _        y )
Nr   r0   zThe hidden size (z6) is not a multiple of the number of attention heads (ú)r   z6hidden_size should be divisible by num_attention_headsra   )rb   rd   )r,   r-   Úhidden_sizeÚnum_attention_headsrO   Ú
ValueErrorÚ
head_ratioÚconv_kernel_sizeÚattention_head_sizeÚall_head_sizer   ÚLinearÚqueryÚkeyÚvaluer_   Úkey_conv_attn_layerÚconv_kernel_layerÚconv_out_layerÚUnfoldÚintÚunfoldr9   Úattention_probs_dropout_probr;   )rD   rE   Únew_num_attention_headsrF   s      €rG   r-   zConvBertSelfAttention.__init__‰   s>  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð
 #)×"<Ñ"<À×@QÑ@QÑ"QÐØ" QÒ&Ø$×8Ñ8ˆDŒOØ'(ˆDÕ$à'>ˆDÔ$Ø$×/Ñ/ˆDŒOà &× 7Ñ 7ˆÔØ×Ñ × 8Ñ 8Ñ8¸AÒ=ÜÐUÓVÐVà$*×$6Ñ$6¸$×:RÑ:RÑ$RÐWXÑ#XˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä#2Ø�F×&Ñ&¨×(:Ñ(:¸D×<QÑ<Qó$
ˆÔ ô "$§¡¨4×+=Ñ+=¸t×?WÑ?WÐZ^×ZoÑZoÑ?oÓ!pˆÔÜ Ÿi™i¨×(:Ñ(:¸D×<NÑ<NÓOˆÔä—i‘iØ×.Ñ.°Ð2¼SÀ$×BWÑBWÐZ[ÑB[Ð_`ÑA`Ó=aÐcdÐ<eô
ˆŒô —z‘z &×"EÑ"EÓFˆ�rH   Nru   Úattention_maskÚencoder_hidden_statesrt   rK   c                 ó   — |j                   d d }g |¢d‘| j                  ‘­}|�#| j                  |«      }| j                  |«      }n"| j                  |«      }| j                  |«      }| j	                  |j                  dd«      «      }	|	j                  dd«      }	| j                  |«      }
|
j                  |«      j                  dd«      }|j                  |«      j                  dd«      }|j                  |«      j                  dd«      }t        j                  |	|
«      }| j                  |«      }t        j                  |d| j                  dg«      }t        j                  |d¬«      }| j                  |«      }t        j                  ||d   d| j                  g«      }|j                  dd«      j!                  «       j#                  d«      }t$        j&                  j)                  || j                  dgd| j                  dz
  dz  dgd¬«      }|j                  dd«      j                  |d   d| j                  | j                  «      }t        j                  |d| j                  | j                  g«      }t        j*                  ||«      }t        j                  |d| j                  g«      }t        j*                  ||j                  dd«      «      }|t-        j.                  | j                  «      z  }|�||z   }t$        j&                  j                  |d¬«      }| j1                  |«      }t        j*                  ||«      }|j3                  dddd«      j!                  «       }t        j                  ||d   d| j4                  | j                  g«      }t        j6                  ||gd«      }|j9                  «       d d | j4                  | j                  z  dz  fz   } |j                  |Ž }||fS )	Nr'   r   ra   ©Údimr   )rb   Údilationrd   Ústrideéþÿÿÿr   )Úshaper„   rˆ   r‰   rŠ   Ú	transposer‡   Úviewr=   Úmultiplyr‹   Úreshaperƒ   ÚsoftmaxrŒ   r…   Ú
contiguousÚ	unsqueezer   Ú
functionalr�   ÚmatmulÚmathÚsqrtr;   Úpermuter€   ÚcatrA   )rD   ru   r’   r“   rt   rP   Úhidden_shapeÚmixed_key_layerÚmixed_value_layerÚmixed_key_conv_attn_layerÚmixed_query_layerÚquery_layerÚ	key_layerÚvalue_layerÚconv_attn_layerr‹   rŒ   Úattention_scoresÚattention_probsÚcontext_layerÚconv_outÚnew_context_layer_shapes                         rG   rU   zConvBertSelfAttention.forward°   sÏ  € ð $×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆð !Ð,Ø"Ÿh™hÐ'<Ó=ˆOØ $§
¡
Ð+@Ó AÑà"Ÿh™h }Ó5ˆOØ $§
¡
¨=Ó 9Ðà$(×$<Ñ$<¸]×=TÑ=TÐUVÐXYÓ=ZÓ$[Ð!Ø$=×$GÑ$GÈÈ1Ó$MÐ!à ŸJ™J }Ó5ÐØ'×,Ñ,¨\Ó:×DÑDÀQÈÓJˆà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆ	Ø'×,Ñ,¨\Ó:×DÑDÀQÈÓJˆäŸ.™.Ð)BÐDUÓVˆà ×2Ñ2°?ÓCÐÜ!ŸM™MÐ*;¸bÀ$×BWÑBWÐYZÐ=[Ó\ÐÜ!ŸM™MÐ*;ÀÔCÐà×,Ñ,¨]Ó;ˆÜŸ™ ~¸ÀA¹ÈÈD×L^ÑL^Ð7_Ó`ˆØ'×1Ñ1°!°QÓ7×BÑBÓD×NÑNÈrÓRˆÜŸ™×-Ñ-ØØ×.Ñ.°Ð2ØØ×+Ñ+¨aÑ/°AÑ5°qÐ9Øð .ó 
ˆð (×1Ñ1°!°QÓ7×?Ñ?Ø˜‰N˜B × 2Ñ 2°D×4IÑ4Ió
ˆô Ÿ™ ~¸¸D×<TÑ<TÐVZ×VkÑVkÐ7lÓmˆÜŸ™ nÐ6GÓHˆÜŸ™ ~¸¸D×<NÑ<NÐ7OÓPˆô !Ÿ<™<¨°Y×5HÑ5HÈÈRÓ5PÓQÐØ+¬d¯i©i¸×8PÑ8PÓ.QÑQÐØÐ%à/°.Ñ@Ðô Ÿ-™-×/Ñ/Ð0@ÀbÐ/ÓIˆð Ÿ,™, Ó7ˆäŸ™ _°kÓBˆØ%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆä—=‘=Ø˜[¨™^¨R°×1IÑ1IÈ4×KcÑKcÐdó
ˆô Ÿ	™	 =°(Ð";¸QÓ?ˆð #0×"4Ñ"4Ó"6°s¸Ð";Ø×$Ñ$ t×'?Ñ'?Ñ?À!ÑCð?
ñ #
Ðð +˜×*Ñ*Ð,CÐDˆà˜oÐ-Ð-rH   ©NN)rV   rW   rX   r-   r=   rz   r[   r   r   ÚtuplerU   r\   r]   s   @rG   r|   r|   ˆ   s}   ø„ ô%GðT 48Ø59ñ	O.à—|‘|ðO.ð ×)Ñ)¨DÑ0ðO.ð  %Ÿ|™|¨dÑ2ð	O.ð
 Ð+Ñ,ðO.ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷O.rH   r|   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚConvBertSelfOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _
        y ©Nr#   )r,   r-   r   r†   r   Údenser7   r8   r9   r:   r;   rC   s     €rG   r-   zConvBertSelfOutput.__init__  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�rH   ru   Úinput_tensorrK   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S rw   ©r¼   r;   r7   ©rD   ru   r½   s      rG   rU   zConvBertSelfOutput.forward	  ó7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°|Ñ'CÓDˆØÐrH   ©rV   rW   rX   r-   r=   rz   rU   r\   r]   s   @rG   r¹   r¹     s1   ø„ ô>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ rH   r¹   c                   ó¢   ‡ — e Zd Zˆ fd„Z	 	 d	dej
                  dej                  dz  dej
                  dz  dee   dej
                  f
d„Z	ˆ xZ
S )
ÚConvBertAttentionc                 ób   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        y rw   )r,   r-   r|   rD   r¹   ÚoutputrC   s     €rG   r-   zConvBertAttention.__init__  s&   ø€ Ü‰ÑÔÜ)¨&Ó1ˆŒ	Ü(¨Ó0ˆ�rH   Nru   r’   r“   rt   rK   c                 ó\   —  | j                   ||fd|i|¤Ž\  }}| j                  ||«      }|S )Nr“   )rD   rÆ   )rD   ru   r’   r“   rt   r³   Ú_Úattention_outputs           rG   rU   zConvBertAttention.forward  sL   € ð %˜4Ÿ9™9ØØñ
ð #8ð
ð ñ	
Ñˆ�qð  Ÿ;™; }°mÓDÐØÐrH   r¶   )rV   rW   rX   r-   r=   rz   r[   r   r   rU   r\   r]   s   @rG   rÄ   rÄ     sg   ø„ ô1ð 48Ø59ñ	 à—|‘|ð ð ×)Ñ)¨DÑ0ð ð  %Ÿ|™|¨dÑ2ð	 ð
 Ð+Ñ,ð ð 
�‰÷ rH   rÄ   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚGroupedLinearLayerc                 ó¸  •— t         ‰| �  «        || _        || _        || _        | j                  | j                  z  | _        | j                  | j                  z  | _        t        j                  t        j                  | j                  | j
                  | j                  «      «      | _        t        j                  t        j                  |«      «      | _        y rw   )r,   r-   Ú
input_sizeÚoutput_sizeÚ
num_groupsÚgroup_in_dimÚgroup_out_dimr   rm   r=   Úemptyrn   re   )rD   rÍ   rÎ   rÏ   rF   s       €rG   r-   zGroupedLinearLayer.__init__(  s—   ø€ Ü‰ÑÔØ$ˆŒØ&ˆÔØ$ˆŒØ ŸO™O¨t¯©Ñ>ˆÔØ!×-Ñ-°·±Ñ@ˆÔÜ—l‘l¤5§;¡;¨t¯©À×@QÑ@QÐSW×SeÑSeÓ#fÓgˆŒÜ—L‘L¤§¡¨[Ó!9Ó:ˆ�	rH   ru   rK   c                 óˆ  — t        |j                  «       «      d   }t        j                  |d| j                  | j
                  g«      }|j                  ddd«      }t        j                  || j                  «      }|j                  ddd«      }t        j                  ||d| j                  g«      }|| j                  z   }|S )Nr   r'   r   ra   )ÚlistrA   r=   rž   rÏ   rÐ   r¦   r£   rn   rÎ   re   )rD   ru   Ú
batch_sizerx   s       rG   rU   zGroupedLinearLayer.forward2  s¢   € Ü˜-×,Ñ,Ó.Ó/°Ñ2ˆ
Ü�M‰M˜-¨"¨d¯o©o¸t×?PÑ?PÐ)QÓRˆØ�I‰I�a˜˜AÓˆÜ�L‰L˜˜DŸK™KÓ(ˆØ�I‰I�a˜˜AÓˆÜ�M‰M˜!˜j¨"¨d×.>Ñ.>Ð?Ó@ˆØ�—	‘	‰MˆØˆrH   rÂ   r]   s   @rG   rË   rË   '  s#   ø„ ô;ð U§\¡\ð °e·l±l÷ rH   rË   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚConvBertIntermediatec                 óŠ  •— t         ‰| �  «        |j                  dk(  r0t        j                  |j
                  |j                  «      | _        n1t        |j
                  |j                  |j                  ¬«      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y )Nr   ©rÍ   rÎ   rÏ   )r,   r-   rÏ   r   r†   r   Úintermediate_sizer¼   rË   Ú
isinstanceÚ
hidden_actÚstrr
   Úintermediate_act_fnrC   s     €rG   r-   zConvBertIntermediate.__init__>  s“   ø€ Ü‰ÑÔØ×Ñ Ò!ÜŸ™ 6×#5Ñ#5°v×7OÑ7OÓPˆD�Jä+Ø!×-Ñ-¸6×;SÑ;SÐ`f×`qÑ`qôˆDŒJô �f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$rH   ru   rK   c                 óJ   — | j                  |«      }| j                  |«      }|S rw   )r¼   rÞ   ©rD   ru   s     rG   rU   zConvBertIntermediate.forwardK  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐrH   rÂ   r]   s   @rG   r×   r×   =  s#   ø„ ô9ð U§\¡\ð °e·l±l÷ rH   r×   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚConvBertOutputc                 óª  •— t         ‰| �  «        |j                  dk(  r0t        j                  |j
                  |j                  «      | _        n1t        |j
                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  ¬«      | _	        t        j                  |j                  «      | _        y )Nr   rÙ   r#   )r,   r-   rÏ   r   r†   rÚ   r   r¼   rË   r7   r8   r9   r:   r;   rC   s     €rG   r-   zConvBertOutput.__init__R  s–   ø€ Ü‰ÑÔØ×Ñ Ò!ÜŸ™ 6×#;Ñ#;¸V×=OÑ=OÓPˆD�Jä+Ø!×3Ñ3À×ASÑASÐ`f×`qÑ`qôˆDŒJô Ÿ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�rH   ru   r½   rK   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S rw   r¿   rÀ   s      rG   rU   zConvBertOutput.forward]  rÁ   rH   rÂ   r]   s   @rG   râ   râ   Q  s1   ø„ ô	>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ rH   râ   c                   óÈ   ‡ — e Zd Zˆ fd„Z	 	 	 ddej
                  dej                  dz  dej
                  dz  dej
                  dz  dee   dej
                  fd	„Z	d
„ Z
ˆ xZS )ÚConvBertLayerc                 ób  •— t         ‰| �  «        |j                  | _        d| _        t	        |«      | _        |j                  | _        |j                  | _        | j                  r*| j                  st        | › d�«      ‚t	        |«      | _	        t        |«      | _        t        |«      | _        y )Nr   z> should be used as a decoder model if cross attention is added)r,   r-   Úchunk_size_feed_forwardÚseq_len_dimrÄ   Ú	attentionÚ
is_decoderÚadd_cross_attentionÚ	TypeErrorÚcrossattentionr×   Úintermediaterâ   rÆ   rC   s     €rG   r-   zConvBertLayer.__init__e  s”   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ*¨6Ó2ˆŒØ ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ Ø×#Ò#Ø—?’?Ü 4 &Ð(fÐ gÓhÐhÜ"3°FÓ";ˆDÔÜ0°Ó8ˆÔÜ$ VÓ,ˆ�rH   Nru   r’   r“   Úencoder_attention_maskrt   rK   c                 ó  —  | j                   ||fi |¤Ž}| j                  r3|�1t        | d«      st        d| › d�«      ‚ | j                  ||fd|i|¤Ž}t        | j                  | j                  | j                  |«      }|S )Nrî   z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r“   )	rê   rë   rO   ÚAttributeErrorrî   r   Úfeed_forward_chunkrè   ré   )rD   ru   r’   r“   rð   rt   rÉ   Úlayer_outputs           rG   rU   zConvBertLayer.forwards  s¿   € ð *˜4Ÿ>™>ØØñ
ð ñ
Ðð �?Š?Ð4Ð@Ü˜4Ð!1Ô2Ü$Ø=¸d¸Vð DDð Dóð ð  3˜t×2Ñ2Ø Ø&ñ ð '<ð ð ñ	 Ðô 1Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUeó
ˆð ÐrH   c                 óL   — | j                  |«      }| j                  ||«      }|S rw   )rï   rÆ   )rD   rÉ   Úintermediate_outputrô   s       rG   ró   z ConvBertLayer.feed_forward_chunk“  s,   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6Ð8HÓIˆØÐrH   ©NNN)rV   rW   rX   r-   r=   rz   r[   r   r   rU   ró   r\   r]   s   @rG   ræ   ræ   d  sƒ   ø„ ô-ð" 48Ø59Ø6:ñà—|‘|ðð ×)Ñ)¨DÑ0ðð  %Ÿ|™|¨dÑ2ð	ð
 !&§¡¨tÑ 3ðð Ð+Ñ,ðð 
�‰óö@rH   ræ   c                   ód   ‡ — e Zd ZU eed<   dZdZeedœZ	 e
j                  «       ˆ fd„«       Zˆ xZS )ÚConvBertPreTrainedModelrE   ÚconvbertT)ru   Ú
attentionsc                 ób  •— t         ‰| �  |«       t        |t        «      r t	        j
                  |j                  «       yt        |t        «      rVt	        j                  |j                  d| j                  j                  ¬«       t	        j
                  |j                  «       yt        |t        «      ryt	        j                  |j                  t        j                   |j                  j"                  d   «      j%                  d«      «       t	        j
                  |j&                  «       yy)zInitialize the weightsrf   rg   r'   r&   N)r,   Ú_init_weightsrÛ   r_   ÚinitÚzeros_re   rË   rp   rn   rE   rq   r    Úcopy_r%   r=   r>   rš   r?   r)   )rD   ÚmodulerF   s     €rG   rý   z%ConvBertPreTrainedModel._init_weights£  sÂ   ø€ ô 	‰Ñ˜fÔ%Ü�fœoÔ.Ü�K‰K˜Ÿ™Õ$Ü˜Ô 2Ô3Ü�L‰L˜Ÿ™¨S°d·k±k×6SÑ6SÕTÜ�K‰K˜Ÿ™Õ$Ü˜Ô 2Ô3Ü�J‰J�v×*Ñ*¬E¯L©L¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÔhÜ�K‰K˜×-Ñ-Õ.ð 4rH   )rV   rW   rX   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingræ   r|   Ú_can_record_outputsr=   Úno_gradrý   r\   r]   s   @rG   rù   rù   ™  s?   ø… àÓØ"ÐØ&*Ð#à&Ø+ñÐð
 €U‡]�]ƒ_ó
/ó ô
/rH   rù   c                   ó¤   ‡ — e Zd Zˆ fd„Z	 	 	 d	dej
                  dej                  dz  dej
                  dz  dej
                  dz  def
d„Zˆ xZ	S )
ÚConvBertEncoderc                 óÐ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _	        y c c}w )NF)
r,   r-   rE   r   Ú
ModuleListÚrangeÚnum_hidden_layersræ   ÚlayerÚgradient_checkpointing)rD   rE   rÈ   rF   s      €rG   r-   zConvBertEncoder.__init__²  sN   ø€ Ü‰ÑÔØˆŒÜ—]‘]Ä5È×IaÑIaÓCbÖ#c¸a¤M°&Õ$9Ò#cÓdˆŒ
Ø&+ˆÕ#ùò $ds   ½A#Nru   r’   r“   rð   rK   c                 óV   — | j                   D ]  } |||f||dœ|¤Ž}Œ t        |¬«      S )N)r“   rð   )Úlast_hidden_state)r  r   )rD   ru   r’   r“   rð   rt   Úlayer_modules          rG   rU   zConvBertEncoder.forward¸  sP   € ð !ŸJ™Jò 	ˆLÙ(ØØðð '<Ø'=ñ	ð
 ñ‰Mð	ô 2Ø+ô
ð 	
rH   r÷   )
rV   rW   rX   r-   r=   rz   r[   r   rU   r\   r]   s   @rG   r  r  ±  si   ø„ ô,ð 48Ø59Ø6:ñ
à—|‘|ð
ð ×)Ñ)¨DÑ0ð
ð  %Ÿ|™|¨dÑ2ð	
ð
 !&§¡¨tÑ 3ð
ð 
,÷
rH   r  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚConvBertPredictionHeadTransformc                 óh  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        |j                  t        «      rt        |j                     | _
        n|j                  | _
        t        j                  |j                  |j                  ¬«      | _        y r»   )r,   r-   r   r†   r   r¼   rÛ   rÜ   rÝ   r
   Útransform_act_fnr7   r8   rC   s     €rG   r-   z(ConvBertPredictionHeadTransform.__init__Ï  s{   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü�f×'Ñ'¬Ô-Ü$*¨6×+<Ñ+<Ñ$=ˆDÕ!à$*×$5Ñ$5ˆDÔ!ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆ�rH   ru   rK   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rw   )r¼   r  r7   rà   s     rG   rU   z'ConvBertPredictionHeadTransform.forwardØ  s4   € ØŸ
™
 =Ó1ˆØ×-Ñ-¨mÓ<ˆØŸ™ }Ó5ˆØÐrH   rÂ   r]   s   @rG   r  r  Î  s$   ø„ ôUð U§\¡\ð °e·l±l÷ rH   r  c                   ó‚   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej                  dej                  dz  dej                  fd„Z	ˆ xZ
S )
ÚConvBertSequenceSummaryaÐ  
    Compute a single vector summary of a sequence hidden states.

    Args:
        config ([`ConvBertConfig`]):
            The config used by the model. Relevant arguments in the config class of the model are (refer to the actual
            config class of your model for the default values it uses):

            - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are:

                - `"last"` -- Take the last token hidden state (like XLNet)
                - `"first"` -- Take the first token hidden state (like Bert)
                - `"mean"` -- Take the mean of all tokens hidden states
                - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2)
                - `"attn"` -- Not implemented now, use multi-head attention

            - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction.
            - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes
              (otherwise to `config.hidden_size`).
            - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output,
              another string or `None` will add no activation.
            - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation.
            - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation.
    rE   c                 óf  •— t         ‰| �  «        t        |dd«      | _        | j                  dk(  rt        ‚t        j                  «       | _        t        |d«      rq|j                  ret        |d«      r(|j                  r|j                  dkD  r|j                  }n|j                  }t        j                  |j                  |«      | _        t        |dd «      }|rt        |«      nt        j                  «       | _        t        j                  «       | _        t        |d«      r3|j"                  dkD  r$t        j$                  |j"                  «      | _        t        j                  «       | _        t        |d	«      r5|j(                  dkD  r%t        j$                  |j(                  «      | _        y y y )
NÚsummary_typeÚlastÚattnÚsummary_use_projÚsummary_proj_to_labelsr   Úsummary_activationÚsummary_first_dropoutÚsummary_last_dropout)r,   r-   Úgetattrr  ÚNotImplementedErrorr   ÚIdentityÚsummaryrO   r  r  Ú
num_labelsr   r†   r   Ú
activationÚfirst_dropoutr   r9   Úlast_dropoutr!  )rD   rE   Únum_classesÚactivation_stringrF   s       €rG   r-   z ConvBertSequenceSummary.__init__ú  sU  ø€ Ü‰ÑÔä# F¨N¸FÓCˆÔØ×Ñ Ò&ô &Ð%ä—{‘{“}ˆŒÜ�6Ð-Ô.°6×3JÒ3JÜ�vÐ7Ô8¸V×=ZÒ=ZÐ_e×_pÑ_pÐstÒ_tØ$×/Ñ/‘à$×0Ñ0�ÜŸ9™9 V×%7Ñ%7¸ÓEˆDŒLä# FÐ,@À$ÓGÐÙIZ¤NÐ3DÔ$EÔ`b×`kÑ`kÓ`mˆŒäŸ[™[›]ˆÔÜ�6Ð2Ô3¸×8TÑ8TÐWXÒ8XÜ!#§¡¨F×,HÑ,HÓ!IˆDÔäŸK™K›MˆÔÜ�6Ð1Ô2°v×7RÑ7RÐUVÒ7VÜ "§
¡
¨6×+FÑ+FÓ GˆDÕð 8WÐ2rH   Nru   Ú	cls_indexrK   c                 óü  — | j                   dk(  r|dd…df   }�n| j                   dk(  r|dd…df   }�n| j                   dk(  r|j                  d¬«      }ná| j                   d	k(  r½|€At        j                  |d
dd…dd…f   |j                  d   dz
  t        j
                  ¬«      }nX|j                  d«      j                  d«      }|j                  d|j                  «       dz
  z  |j                  d«      fz   «      }|j                  d|«      j                  d«      }n| j                   dk(  rt        ‚| j                  «      }| j                  |«      }| j                  |«      }| j!                  |«      }|S )ak  
        Compute a single vector summary of a sequence hidden states.

        Args:
            hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`):
                The hidden states of the last layer.
            cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*):
                Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token.

        Returns:
            `torch.FloatTensor`: The summary of the sequence hidden states.
        r  Nr'   Úfirstr   rh   r   r•   r,  .r™   r*   )r'   r  )r  rh   r=   Ú	full_likerš   rB   r¡   r?   r–   rA   ÚgatherÚsqueezer#  r(  r%  r'  r)  )rD   ru   r,  rÆ   s       rG   rU   zConvBertSequenceSummary.forward  sn  € ð ×Ñ Ò&Ø"¢1 b 5Ñ)ŠFØ×Ñ 'Ò)Ø"¢1 a 4Ñ(ŠFØ×Ñ &Ò(Ø"×'Ñ'¨AÐ'Ó.‰FØ×Ñ +Ò-ØÐ Ü!ŸO™OØ! # r¨ rª1 *Ñ-Ø!×'Ñ'¨Ñ+¨aÑ/ÜŸ*™*ô‘	ð &×/Ñ/°Ó3×=Ñ=¸bÓA�	Ø%×,Ñ,¨U°i·m±m³oÈÑ6IÑ-JÈm×N`ÑN`ÐacÓNdÐMfÑ-fÓg�	à"×)Ñ)¨"¨iÓ8×@Ñ@ÀÓD‰FØ×Ñ &Ò(Ü%Ð%à×#Ñ# FÓ+ˆØ—‘˜fÓ%ˆØ—‘ Ó(ˆØ×"Ñ" 6Ó*ˆàˆrH   rw   )rV   rW   rX   rY   r   r-   r=   r[   rZ   rU   r\   r]   s   @rG   r  r  à  sQ   ø„ ñð2H˜~õ Hð< VZñ)Ø"×.Ñ.ð)Ø;@×;KÑ;KÈdÑ;Rð)à	×	Ñ	÷)rH   r  c                   ó   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zeee	 	 	 	 	 dde	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  d	e	j                  dz  d
ee   defd„«       «       «       Zˆ xZS )ÚConvBertModelc                 ó"  •— t         ‰| �  |«       t        |«      | _        |j                  |j
                  k7  r/t        j                  |j                  |j
                  «      | _        t        |«      | _
        || _        | j                  «        y rw   )r,   r-   r    rT   r0   r   r   r†   Úembeddings_projectr  ÚencoderrE   Ú	post_initrC   s     €rG   r-   zConvBertModel.__init__E  sl   ø€ Ü‰Ñ˜Ô Ü,¨VÓ4ˆŒà× Ñ  F×$6Ñ$6Ò6Ü&(§i¡i°×0EÑ0EÀv×GYÑGYÓ&ZˆDÔ#ä& vÓ.ˆŒØˆŒà�‰ÕrH   c                 ó.   — | j                   j                  S rw   ©rT   r2   ©rD   s    rG   Úget_input_embeddingsz"ConvBertModel.get_input_embeddingsQ  s   € Ø�‰×.Ñ.Ð.rH   c                 ó&   — || j                   _        y rw   r9  )rD   r‰   s     rG   Úset_input_embeddingsz"ConvBertModel.set_input_embeddingsT  s   € Ø*/ˆ�‰Õ'rH   NrI   r’   r)   r%   rJ   rt   rK   c                 óÂ  — |�|�t        d«      ‚|�#| j                  ||«       |j                  «       }n!|�|j                  «       d d }nt        d«      ‚|\  }}	|�|j                  n|j                  }
|€t	        j
                  ||
¬«      }|€pt        | j                  d«      r4| j                  j                  d d …d |	…f   }|j                  ||	«      }|}n&t	        j                  |t        j                  |
¬«      }| j                  ||||¬«      }t        | d«      r| j                  |«      }t        | j                  ||¬	«      } | j                  |fd
|i|¤Ž}|S )NzDYou cannot specify both input_ids and inputs_embeds at the same timer'   z5You have to specify either input_ids or inputs_embeds)rN   r)   rM   )rI   r%   r)   rJ   r5  )rE   rJ   r’   r’   )r�   Ú%warn_if_padding_and_no_attention_maskrA   rN   r=   ÚonesrO   rT   r)   r?   r@   rB   r5  r   rE   r6  )rD   rI   r’   r)   r%   rJ   rt   rP   rÕ   rQ   rN   rR   rS   ru   Úencoder_outputss                  rG   rU   zConvBertModel.forwardW  s€  € ð Ð  ]Ð%>ÜÐcÓdÐdØÐ"Ø×6Ñ6°yÀ.ÔQØ#Ÿ.™.Ó*‰KØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà!,Ñˆ
�JØ%.Ð%:�×!Ò!À×@TÑ@TˆàÐ!Ü"ŸZ™Z¨¸FÔCˆNØÐ!Ü�t—‘Ð(8Ô9Ø*.¯/©/×*HÑ*HÊÈKÈZÈKÈÑ*XÐ'Ø3J×3QÑ3QÐR\Ð^hÓ3iÐ0Ø!A‘ä!&§¡¨[ÄÇ
Á
ÐSYÔ!Z�àŸ™Ø¨lÈ>Ðivð (ó 
ˆô �4Ð-Ô.Ø ×3Ñ3°MÓBˆMä2Ø—;‘;Ø'Ø)ô
ˆð ?K¸d¿l¹lØñ?
à)ð?
ð ñ?
ˆð ÐrH   )NNNNN)rV   rW   rX   r-   r;  r=  r   r   r   r=   rZ   r[   r   r   r   rU   r\   r]   s   @rG   r3  r3  C  sÈ   ø„ ô
ò/ò0ð  ØØð .2Ø37Ø26Ø04Ø26ñ3à×#Ñ# dÑ*ð3ð ×)Ñ)¨DÑ0ð3ð ×(Ñ(¨4Ñ/ð	3ð
 ×&Ñ&¨Ñ-ð3ð ×(Ñ(¨4Ñ/ð3ð Ð+Ñ,ð3ð 
,ò3ó ó ó  ô3rH   r3  c                   óZ   ‡ — e Zd ZdZˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚConvBertGeneratorPredictionszAPrediction module for the generator, made up of two dense layers.c                 ó   •— t         ‰| �  «        t        d«      | _        t	        j
                  |j                  |j                  ¬«      | _        t	        j                  |j                  |j                  «      | _
        y )NÚgelur#   )r,   r-   r   r'  r   r7   r0   r8   r†   r   r¼   rC   s     €rG   r-   z%ConvBertGeneratorPredictions.__init__“  sV   ø€ Ü‰ÑÔä(¨Ó0ˆŒÜŸ™ f×&;Ñ&;À×AVÑAVÔWˆŒÜ—Y‘Y˜v×1Ñ1°6×3HÑ3HÓIˆ�
rH   Úgenerator_hidden_statesrK   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rw   )r¼   r'  r7   )rD   rF  ru   s      rG   rU   z$ConvBertGeneratorPredictions.forwardš  s3   € ØŸ
™
Ð#:Ó;ˆØŸ™¨Ó6ˆØŸ™ }Ó5ˆàÐrH   )	rV   rW   rX   rY   r-   r=   r[   rU   r\   r]   s   @rG   rC  rC  �  s+   ø„ ÙKôJð¨u×/@Ñ/@ð ÀU×EVÑEV÷ rH   rC  c                   ó$  ‡ — e Zd ZddiZˆ fd„Zd„ Zd„ Zee	 	 	 	 	 	 dde	j                  dz  de	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  dee   deez  fd„«       «       Zˆ xZS )ÚConvBertForMaskedLMzgenerator_lm_head.weightz*convbert.embeddings.word_embeddings.weightc                 óâ   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        t        j                  |j                  |j                  «      | _
        | j                  «        y rw   )r,   r-   r3  rú   rC  Úgenerator_predictionsr   r†   r0   r/   Úgenerator_lm_headr7  rC   s     €rG   r-   zConvBertForMaskedLM.__init__¦  sR   ø€ Ü‰Ñ˜Ô ä% fÓ-ˆŒÜ%AÀ&Ó%IˆÔ"ä!#§¡¨6×+@Ñ+@À&×BSÑBSÓ!TˆÔà�‰ÕrH   c                 ó   — | j                   S rw   ©rL  r:  s    rG   Úget_output_embeddingsz)ConvBertForMaskedLM.get_output_embeddings°  s   € Ø×%Ñ%Ð%rH   c                 ó   — || _         y rw   rN  )rD   r2   s     rG   Úset_output_embeddingsz)ConvBertForMaskedLM.set_output_embeddings³  s
   € Ø!0ˆÕrH   NrI   r’   r)   r%   rJ   Úlabelsrt   rK   c                 ón  —  | j                   |f||||dœ|¤Ž}|d   }	| j                  |	«      }
| j                  |
«      }
d}|�Pt        j                  «       } ||
j                  d| j                  j                  «      |j                  d«      «      }t        ||
|j                  |j                  ¬«      S )a¢  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        ©r’   r)   r%   rJ   r   Nr'   ©ÚlossÚlogitsru   rû   )rú   rK  rL  r   r   rœ   rE   r/   r   ru   rû   )rD   rI   r’   r)   r%   rJ   rR  rt   rF  Úgenerator_sequence_outputÚprediction_scoresrV  Úloss_fcts                rG   rU   zConvBertForMaskedLM.forward¶  sØ   € ð$ GTÀdÇmÁmØðG
à)Ø)Ø%Ø'ñG
ð ñG
Ðð %<¸AÑ$>Ð!à ×6Ñ6Ð7PÓQÐØ ×2Ñ2Ð3DÓEÐàˆàÐÜ×*Ñ*Ó,ˆHÙÐ-×2Ñ2°2°t·{±{×7MÑ7MÓNÐPV×P[ÑP[Ð\^ÓP_Ó`ˆDäØØ$Ø1×?Ñ?Ø.×9Ñ9ô	
ð 	
rH   ©NNNNNN)rV   rW   rX   Ú_tied_weights_keysr-   rO  rQ  r   r   r=   rZ   r[   r   r   r·   r   rU   r\   r]   s   @rG   rI  rI  ¢  sé   ø„ à4Ð6bÐcÐôò&ò1ð Øð .2Ø37Ø26Ø04Ø26Ø*.ñ(
à×#Ñ# dÑ*ð(
ð ×)Ñ)¨DÑ0ð(
ð ×(Ñ(¨4Ñ/ð	(
ð
 ×&Ñ&¨Ñ-ð(
ð ×(Ñ(¨4Ñ/ð(
ð × Ñ  4Ñ'ð(
ð Ð+Ñ,ð(
ð 
�Ñ	ò(
ó ó ô(
rH   rI  c                   óZ   ‡ — e Zd ZdZˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚConvBertClassificationHeadz-Head for sentence-level classification tasks.c                 óh  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        |j                  �|j                  n|j                  }t        j                  |«      | _	        t        j                  |j                  |j                  «      | _        || _        y rw   )r,   r-   r   r†   r   r¼   Úclassifier_dropoutr:   r9   r;   r&  Úout_projrE   ©rD   rE   r`  rF   s      €rG   r-   z#ConvBertClassificationHead.__init__æ  s†   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
à)/×)BÑ)BÐ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z‘zÐ"4Ó5ˆŒÜŸ	™	 &×"4Ñ"4°f×6GÑ6GÓHˆŒàˆ�rH   ru   rK   c                 óê   — |d d …dd d …f   }| j                  |«      }| j                  |«      }t        | j                  j                     |«      }| j                  |«      }| j                  |«      }|S )Nr   )r;   r¼   r
   rE   rÜ   ra  )rD   ru   rt   rx   s       rG   rU   z"ConvBertClassificationHead.forwardñ  se   € Øš!˜Q¢˜'Ñ"ˆØ�L‰L˜‹OˆØ�J‰J�q‹MˆÜ�4—;‘;×)Ñ)Ñ*¨1Ó-ˆØ�L‰L˜‹OˆØ�M‰M˜!ÓˆØˆrH   ry   r]   s   @rG   r^  r^  ã  s&   ø„ Ù7ô	ð U§\¡\ð ÀÇÁ÷ rH   r^  z 
    ConvBERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the
    pooled output) e.g. for GLUE tasks.
    )Úcustom_introc                   ó  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
   d
eez  fd„«       «       Zˆ xZS )Ú!ConvBertForSequenceClassificationc                 ó´   •— t         ‰| �  |«       |j                  | _        || _        t	        |«      | _        t        |«      | _        | j                  «        y rw   )	r,   r-   r&  rE   r3  rú   r^  Ú
classifierr7  rC   s     €rG   r-   z*ConvBertForSequenceClassification.__init__  sH   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØˆŒÜ% fÓ-ˆŒÜ4°VÓ<ˆŒð 	�‰ÕrH   NrI   r’   r)   r%   rJ   rR  rt   rK   c                 ó¼  —  | j                   |f||||dœ|¤Ž}|d   }	| j                  |	«      }
d}|��‡| j                  j                  €�| j                  dk(  rd| j                  _        nl| j                  dkD  rL|j
                  t        j                  k(  s|j
                  t        j                  k(  rd| j                  _        nd| j                  _        | j                  j                  dk(  rIt        «       }| j                  dk(  r& ||
j                  «       |j                  «       «      }nŒ ||
|«      }n‚| j                  j                  dk(  r=t        «       } ||
j                  d| j                  «      |j                  d«      «      }n,| j                  j                  dk(  rt        «       } ||
|«      }t        ||
|j                  |j                   ¬	«      S )
a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        rT  r   Nr   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr'   rU  )rú   rh  rE   Úproblem_typer&  r+   r=   rB   rŽ   r   r1  r   rœ   r   r   ru   rû   ©rD   rI   r’   r)   r%   rJ   rR  rt   ÚoutputsÚsequence_outputrW  rV  rZ  s                rG   rU   z)ConvBertForSequenceClassification.forward  s™  € ð$ 7D°d·m±mØð7
à)Ø)Ø%Ø'ñ7
ð ñ7
ˆð " !™*ˆØ—‘ Ó1ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rH   r[  )rV   rW   rX   r-   r   r   r=   rZ   r[   r   r   r·   r   rU   r\   r]   s   @rG   rf  rf  û  sÔ   ø„ ôð Øð .2Ø37Ø26Ø04Ø26Ø*.ñ8
à×#Ñ# dÑ*ð8
ð ×)Ñ)¨DÑ0ð8
ð ×(Ñ(¨4Ñ/ð	8
ð
 ×&Ñ&¨Ñ-ð8
ð ×(Ñ(¨4Ñ/ð8
ð × Ñ  4Ñ'ð8
ð Ð+Ñ,ð8
ð 
Ð)Ñ	)ò8
ó ó ô8
rH   rf  c                   ó  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
   d
eez  fd„«       «       Zˆ xZS )ÚConvBertForMultipleChoicec                 óÎ   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        t        j                  |j                  d«      | _	        | j                  «        y )Nr   )r,   r-   r3  rú   r  Úsequence_summaryr   r†   r   rh  r7  rC   s     €rG   r-   z"ConvBertForMultipleChoice.__init__K  sM   ø€ Ü‰Ñ˜Ô ä% fÓ-ˆŒÜ 7¸Ó ?ˆÔÜŸ)™) F×$6Ñ$6¸Ó:ˆŒð 	�‰ÕrH   NrI   r’   r)   r%   rJ   rR  rt   rK   c                 óê  — |�|j                   d   n|j                   d   }|�!|j                  d|j                  d«      «      nd}|�!|j                  d|j                  d«      «      nd}|�!|j                  d|j                  d«      «      nd}|�!|j                  d|j                  d«      «      nd}|�1|j                  d|j                  d«      |j                  d«      «      nd} | j                  |f||||dœ|¤Ž}	|	d   }
| j	                  |
«      }| j                  |«      }|j                  d|«      }d}|�t        «       } |||«      }t        |||	j                  |	j                  ¬«      S )a\  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:


            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   r'   r™   rT  r   rU  )
rš   rœ   rA   rú   rt  rh  r   r   ru   rû   )rD   rI   r’   r)   r%   rJ   rR  rt   Únum_choicesro  rp  Úpooled_outputrW  Úreshaped_logitsrV  rZ  s                   rG   rU   z!ConvBertForMultipleChoice.forwardU  sž  € ðV -6Ð,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆà>GÐ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÐG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ð(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð 7D°d·m±mØð7
à)Ø)Ø%Ø'ñ7
ð ñ7
ˆð " !™*ˆà×-Ñ-¨oÓ>ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÐÜ'Ó)ˆHÙ˜O¨VÓ4ˆDä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
rH   r[  )rV   rW   rX   r-   r   r   r=   rZ   r[   r   r   r·   r   rU   r\   r]   s   @rG   rr  rr  I  sÞ   ø„ ôð Øð .2Ø37Ø26Ø04Ø26Ø*.ñN
à×#Ñ# dÑ*ðN
ð ×)Ñ)¨DÑ0ðN
ð ×(Ñ(¨4Ñ/ð	N
ð
 ×&Ñ&¨Ñ-ðN
ð ×(Ñ(¨4Ñ/ðN
ð × Ñ  4Ñ'ðN
ð Ð+Ñ,ðN
ð 
Ð*Ñ	*òN
ó ó ôN
rH   rr  c                   ó  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	e	e
   d
eez  fd„«       «       Zˆ xZS )ÚConvBertForTokenClassificationc                 ó`  •— t         ‰| �  |«       |j                  | _        t        |«      | _        |j
                  �|j
                  n|j                  }t        j                  |«      | _	        t        j                  |j                  |j                  «      | _        | j                  «        y rw   )r,   r-   r&  r3  rú   r`  r:   r   r9   r;   r†   r   rh  r7  rb  s      €rG   r-   z'ConvBertForTokenClassification.__init__ª  sˆ   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä% fÓ-ˆŒà)/×)BÑ)BÐ)NˆF×%Ò%ÐTZ×TnÑTnð 	ô —z‘zÐ"4Ó5ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÕrH   NrI   r’   r)   r%   rJ   rR  rt   rK   c                 óF  —  | j                   |f||||dœ|¤Ž}|d   }	| j                  |	«      }	| j                  |	«      }
d}|�<t        «       } ||
j	                  d| j
                  «      |j	                  d«      «      }t        ||
|j                  |j                  ¬«      S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        rT  r   Nr'   rU  )	rú   r;   rh  r   rœ   r&  r   ru   rû   rn  s                rG   rU   z&ConvBertForTokenClassification.forward¸  sµ   € ð  7D°d·m±mØð7
à)Ø)Ø%Ø'ñ7
ð ñ7
ˆð " !™*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÐÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rH   r[  )rV   rW   rX   r-   r   r   r=   rZ   r[   r   r   r·   r   rU   r\   r]   s   @rG   rz  rz  ¨  sÔ   ø„ ôð Øð .2Ø37Ø26Ø04Ø26Ø*.ñ&
à×#Ñ# dÑ*ð&
ð ×)Ñ)¨DÑ0ð&
ð ×(Ñ(¨4Ñ/ð	&
ð
 ×&Ñ&¨Ñ-ð&
ð ×(Ñ(¨4Ñ/ð&
ð × Ñ  4Ñ'ð&
ð Ð+Ñ,ð&
ð 
Ð&Ñ	&ò&
ó ó ô&
rH   rz  c                   ó*  ‡ — e Zd Zˆ fd„Zee	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
e	e
   defd„«       «       Zˆ xZS )ÚConvBertForQuestionAnsweringc                 óä   •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y rw   )
r,   r-   r&  r3  rú   r   r†   r   Ú
qa_outputsr7  rC   s     €rG   r-   z%ConvBertForQuestionAnswering.__init__å  sS   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ% fÓ-ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÕrH   NrI   r’   r)   r%   rJ   Ústart_positionsÚend_positionsrt   rK   c                 óÄ  —  | j                   |f||||dœ|¤Ž}	|	d   }
| j                  |
«      }|j                  dd¬«      \  }}|j                  d«      j	                  «       }|j                  d«      j	                  «       }d }|�·|�µt        |j                  «       «      dkD  r|j                  d«      }t        |j                  «       «      dkD  r|j                  d«      }|j                  d«      }|j                  d|«      }|j                  d|«      }t        |¬«      } |||«      } |||«      }||z   dz  }t        ||||	j                  |	j                  ¬«      S )	NrT  r   r   r'   r•   )Úignore_indexra   )rV  Ústart_logitsÚ
end_logitsru   rû   )rú   r€  Úsplitr1  r    ÚlenrA   Úclampr   r   ru   rû   )rD   rI   r’   r)   r%   rJ   r�  r‚  rt   ro  rp  rW  r…  r†  Ú
total_lossÚignored_indexrZ  Ú
start_lossÚend_losss                      rG   rU   z$ConvBertForQuestionAnswering.forwardï  s~  € ð 7D°d·m±mØð7
à)Ø)Ø%Ø'ñ7
ð ñ7
ˆð " !™*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
rH   )NNNNNNN)rV   rW   rX   r-   r   r   r=   rZ   r[   r   r   r   rU   r\   r]   s   @rG   r~  r~  ã  sæ   ø„ ôð Øð .2Ø37Ø26Ø04Ø26Ø37Ø15ñ2
à×#Ñ# dÑ*ð2
ð ×)Ñ)¨DÑ0ð2
ð ×(Ñ(¨4Ñ/ð	2
ð
 ×&Ñ&¨Ñ-ð2
ð ×(Ñ(¨4Ñ/ð2
ð ×)Ñ)¨DÑ0ð2
ð ×'Ñ'¨$Ñ.ð2
ð Ð+Ñ,ð2
ð 
&ò2
ó ó ô2
rH   r~  )rI  rr  r~  rf  rz  ræ   r3  rù   )FrY   r¤   Úcollections.abcr   r=   r   Útorch.nnr   r   r   Ú r	   rþ   Úactivationsr
   r   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   Úprocessing_utilsr   Úpytorch_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úconfiguration_convbertr   Ú
get_loggerrV   ÚloggerÚModuler    r_   r|   r¹   rÄ   rË   r×   râ   ræ   rù   r  r  r  r3  rC  rI  r^  rf  rr  rz  r~  Ú__all__© rH   rG   ú<module>r¡     s.  ðñ ã Ý $ã Ý ß AÑ Aå &ß 1Ý 6Ý 9÷÷ õ .Ý &Ý 6÷ó õ 8Ý 5Ý 2ð 
ˆ×	Ñ	˜HÓ	%€ô7˜Ÿ™ô 7ôt�b—i‘iô ô4w.˜BŸI™Iô w.ôt˜Ÿ™ô ô ˜Ÿ	™	ô  ô.˜Ÿ™ô ô,˜2Ÿ9™9ô ô(�R—Y‘Yô ô&2Ð.ô 2ðj ô/˜oó /ó ð/ô.
�b—i‘iô 
ô: b§i¡iô ô$`˜bŸi™iô `ðF ôIÐ+ó Ió ðIôX 2§9¡9ô ð$ ô=
Ð1ó =
ó ð=
ô@ §¡ô ñ0 ðôôE
Ð(?ó E
óðE
ðP ô[
Ð 7ó [
ó ð[
ð| ô7
Ð%<ó 7
ó ð7
ðt ô?
Ð#:ó ?
ó ð?
òD	�rH   