Ë
    ÛÓ^j]½  ã                   ó¸  — d Z ddlZddlZddlmZ ddlmZmZmZ ddlm	Z
 ddlmZ ddlmZ dd	lmZmZmZmZmZmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$  ejJ                  e&«      Z' G d„ dejP                  «      Z) G d„ dejP                  «      Z* G d„ dejP                  «      Z+ G d„ dejP                  «      Z, G d„ dejP                  «      Z- G d„ dejP                  «      Z. G d„ dejP                  «      Z/ G d„ dejP                  «      Z0 G d„ d ejP                  «      Z1e G d!„ d"e«      «       Z2e G d#„ d$e2«      «       Z3e G d%„ d&e2«      «       Z4 G d'„ d(ejP                  «      Z5 ed)¬*«       G d+„ d,e2«      «       Z6e G d-„ d.e2«      «       Z7e G d/„ d0e2«      «       Z8 G d1„ d2ejP                  «      Z9e G d3„ d4e2«      «       Z:d7d5„Z;g d6¢Z<y)8zPyTorch I-BERT model.é    N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)Úgelu)Úcreate_bidirectional_mask)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Úauto_docstringÚloggingé   )ÚIBertConfig)ÚIntGELUÚIntLayerNormÚ
IntSoftmaxÚQuantActÚQuantEmbeddingÚQuantLinearc                   ó2   ‡ — e Zd ZdZˆ fd„Z	 dd„Zd„ Zˆ xZS )ÚIBertEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    c                 óz  •— t         ‰| �  «        |j                  | _        d| _        d| _        d| _        d| _        d| _        t        |j                  |j                  |j                  | j                  | j                  ¬«      | _        t        |j                  |j                  | j                  | j                  ¬«      | _        | j                  dt!        j"                  |j$                  «      j'                  d«      d	¬
«       |j                  | _        t        |j$                  |j                  | j(                  | j                  | j                  ¬«      | _        t-        | j                  | j                  ¬«      | _        t-        | j                  | j                  ¬«      | _        t3        |j                  |j4                  | j                  | j                  |j6                  ¬«      | _        t-        | j
                  | j                  ¬«      | _        t=        j>                  |j@                  «      | _!        y )Né   é   é   é    )Úpadding_idxÚ
weight_bitÚ
quant_mode)r%   r&   Úposition_ids©r   éÿÿÿÿF)Ú
persistent©r&   ©ÚepsÚ
output_bitr&   Úforce_dequant)"ÚsuperÚ__init__r&   Úembedding_bitÚembedding_act_bitÚact_bitÚln_input_bitÚln_output_bitr   Ú
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚregister_bufferÚtorchÚarangeÚmax_position_embeddingsÚexpandr$   Úposition_embeddingsr   Úembeddings_act1Úembeddings_act2r   Úlayer_norm_epsr/   Ú	LayerNormÚoutput_activationr   ÚDropoutÚhidden_dropout_probÚdropout©ÚselfÚconfigÚ	__class__s     €ús/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/ibert/modeling_ibert.pyr1   zIBertEmbeddings.__init__3   sÅ  ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØˆÔØ!#ˆÔØˆŒØˆÔØˆÔä-Ø×ÑØ×ÑØ×+Ñ+Ø×)Ñ)Ø—‘ô 
ˆÔô &4Ø×"Ñ" F×$6Ñ$6À4×CUÑCUÐbf×bqÑbqô&
ˆÔ"ð
 	×ÑØœEŸL™L¨×)GÑ)GÓH×OÑOÐPWÓXÐejð 	ô 	
ð
 "×.Ñ.ˆÔÜ#1Ø×*Ñ*Ø×ÑØ×(Ñ(Ø×)Ñ)Ø—‘ô$
ˆÔ ô  (¨×(>Ñ(>È4Ï?É?Ô[ˆÔÜ'¨×(>Ñ(>È4Ï?É?Ô[ˆÔä%Ø×ÑØ×%Ñ%Ø×)Ñ)Ø—‘Ø ×.Ñ.ô
ˆŒô "*¨$¯,©,À4Ç?Á?Ô!SˆÔÜ—z‘z &×"<Ñ"<Ó=ˆ�ó    c                 ó²  — |€D|�1t        || j                  |«      j                  |j                  «      }n| j	                  |«      }|�|j                  «       }n|j                  «       d d }|€:t        j                  |t        j                  | j                  j                  ¬«      }|€| j                  |«      \  }}nd }| j                  |«      \  }}	| j                  ||||	¬«      \  }
}| j                  |«      \  }}| j                  |
|||¬«      \  }
}| j                  |
|«      \  }
}| j                  |
«      }
| j!                  |
|«      \  }
}|
|fS )Nr)   ©ÚdtypeÚdevice©ÚidentityÚidentity_scaling_factor)Ú"create_position_ids_from_input_idsr$   ÚtorT   Ú&create_position_ids_from_inputs_embedsÚsizer>   ÚzerosÚlongr'   r:   r<   rC   rB   rF   rJ   rG   )rL   Ú	input_idsÚtoken_type_idsr'   Úinputs_embedsÚpast_key_values_lengthÚinput_shapeÚinputs_embeds_scaling_factorr<   Ú$token_type_embeddings_scaling_factorÚ
embeddingsÚembeddings_scaling_factorrB   Ú"position_embeddings_scaling_factors                 rO   ÚforwardzIBertEmbeddings.forwardd   sŽ  € ð ÐØÐ$äAØ˜t×/Ñ/Ð1Gó ç‘"�Y×%Ñ%Ó&ñ ð  $×JÑJÈ=ÓY�àÐ Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKàÐ!Ü"Ÿ[™[¨¼E¿J¹JÈt×O`ÑO`×OgÑOgÔhˆNàÐ Ø:>×:NÑ:NÈyÓ:YÑ7ˆMÑ7à+/Ð(ØFJ×F`ÑF`ÐaoÓFpÑCÐÐCà04×0DÑ0DØØ(Ø*Ø$Hð	 1Eó 1
Ñ-ˆ
Ð-ð CG×BZÑBZÐ[gÓBhÑ?ÐÐ?Ø04×0DÑ0DØØ%Ø(Ø$Fð	 1Eó 1
Ñ-ˆ
Ð-ð 15·±¸zÐKdÓ0eÑ-ˆ
Ð-Ø—\‘\ *Ó-ˆ
Ø04×0FÑ0FÀzÐSlÓ0mÑ-ˆ
Ð-ØÐ4Ð4Ð4rP   c                 ó  — |j                  «       dd }|d   }t        j                  | j                  dz   || j                  z   dz   t        j                  |j
                  ¬«      }|j                  d«      j                  |«      S )z×
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr)   r   rR   r   )r[   r>   r?   r$   r]   rT   Ú	unsqueezerA   )rL   r`   rb   Úsequence_lengthr'   s        rO   rZ   z6IBertEmbeddings.create_position_ids_from_inputs_embeds’   s€   € ð $×(Ñ(Ó*¨3¨BÐ/ˆØ% a™.ˆä—|‘|Ø×Ñ˜qÑ  /°D×4DÑ4DÑ"DÀqÑ"HÔPU×PZÑPZÐcp×cwÑcwô
ˆð ×%Ñ% aÓ(×/Ñ/°Ó<Ð<rP   )NNNNr   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r1   rh   rZ   Ú__classcell__©rN   s   @rO   r   r   .   s    ø„ ñô/>ðd rsó,5ö\=rP   r   c                   ó*   ‡ — e Zd Zˆ fd„Z	 	 dd„Zˆ xZS )ÚIBertSelfAttentionc           	      óF  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|j                  | _        d| _        d| _        d| _	        |j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _        t        |j                  | j                  d| j                  | j                  | j                  d¬	«      | _        t        |j                  | j                  d| j                  | j                  | j                  d¬	«      | _        t        |j                  | j                  d| j                  | j                  | j                  d¬	«      | _        t#        | j                  | j                  ¬
«      | _        t#        | j                  | j                  ¬
«      | _        t#        | j                  | j                  ¬
«      | _        t#        | j                  | j                  ¬
«      | _        t-        j.                  |j0                  «      | _        t5        | j                  | j                  |j6                  ¬«      | _        y )Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)r    r#   T©Úbiasr%   Úbias_bitr&   Úper_channelr+   ©r&   r/   )r0   r1   r8   Únum_attention_headsÚhasattrÚ
ValueErrorr&   r%   ry   r4   ÚintÚattention_head_sizeÚall_head_sizer   ÚqueryÚkeyÚvaluer   Úquery_activationÚkey_activationÚvalue_activationrG   r   rH   Úattention_probs_dropout_probrJ   r   r/   ÚsoftmaxrK   s     €rO   r1   zIBertSelfAttention.__init__¥   s  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð !×+Ñ+ˆŒØˆŒØˆŒØˆŒà#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔô !Ø×ÑØ×ÑØØ—‘Ø—]‘]Ø—‘Øô
ˆŒ
ô Ø×ÑØ×ÑØØ—‘Ø—]‘]Ø—‘Øô
ˆŒô !Ø×ÑØ×ÑØØ—‘Ø—]‘]Ø—‘Øô
ˆŒ
ô !)¨¯©À$Ç/Á/Ô RˆÔÜ& t§|¡|ÀÇÁÔPˆÔÜ (¨¯©À$Ç/Á/Ô RˆÔÜ!)¨$¯,©,À4Ç?Á?Ô!SˆÔä—z‘z &×"EÑ"EÓFˆŒä! $§,¡,¸4¿?¹?ÐZ`×ZnÑZnÔoˆ�rP   c                 ó^  — | j                  ||«      \  }}| j                  ||«      \  }}| j                  ||«      \  }	}
| j                  ||«      \  }}| j	                  ||«      \  }}| j                  |	|
«      \  }}|j                  d d }g |¢d‘| j                  ‘­}|j                  |«      j                  dd«      }|j                  |«      j                  dd«      }|j                  |«      j                  dd«      }t        j                  ||j                  dd«      «      }t        j                  | j                  «      }||z  }| j                  r	||z  |z  }nd }|�||z   }| j                  ||«      \  }}| j!                  |«      }t        j                  ||«      }|�||z  }nd }|j#                  dddd«      j%                  «       }|j'                  «       d d | j(                  fz   } |j                  |Ž }| j+                  ||«      \  }}|r||fn|f}|r||fn|f}||fS )Nr)   r   é   éþÿÿÿr   r   )r‚   rƒ   r„   r…   r†   r‡   Úshaper€   ÚviewÚ	transposer>   ÚmatmulÚmathÚsqrtr&   r‰   rJ   ÚpermuteÚ
contiguousr[   r�   rG   )rL   Úhidden_statesÚhidden_states_scaling_factorÚattention_maskÚoutput_attentionsÚmixed_query_layerÚ mixed_query_layer_scaling_factorÚmixed_key_layerÚmixed_key_layer_scaling_factorÚmixed_value_layerÚ mixed_value_layer_scaling_factorÚquery_layerÚquery_layer_scaling_factorÚ	key_layerÚkey_layer_scaling_factorÚvalue_layerÚvalue_layer_scaling_factorrb   Úhidden_shapeÚattention_scoresÚscaleÚattention_scores_scaling_factorÚattention_probsÚattention_probs_scaling_factorÚcontext_layerÚcontext_layer_scaling_factorÚnew_context_layer_shapeÚoutputsÚoutput_scaling_factors                                rO   rh   zIBertSelfAttention.forwardÜ   s˜  € ð ?C¿j¹jÈÐXtÓ>uÑ;ÐÐ;Ø:>¿(¹(À=ÐRnÓ:oÑ7ˆÐ7Ø>B¿j¹jÈÐXtÓ>uÑ;ÐÐ;ð 37×2GÑ2GØÐ?ó3
Ñ/ˆÐ/ð /3×.AÑ.AÀ/ÐSqÓ.rÑ+ˆ	Ð+Ø26×2GÑ2GØÐ?ó3
Ñ/ˆÐ/ð
 $×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆØ!×&Ñ& |Ó4×>Ñ>¸qÀ!ÓDˆØ—N‘N <Ó0×:Ñ:¸1¸aÓ@ˆ	Ø!×&Ñ& |Ó4×>Ñ>¸qÀ!ÓDˆô !Ÿ<™<¨°Y×5HÑ5HÈÈRÓ5PÓQÐÜ—	‘	˜$×2Ñ2Ó3ˆØ+¨eÑ3ÐØ�?Š?Ø.HÐKcÑ.cÐfkÑ.kÑ+à.2Ð+àÐ%à/°.Ñ@Ðð ;?¿,¹,ØÐ=ó;
Ñ7ˆÐ7ð Ÿ,™, Ó7ˆäŸ™ _°kÓBˆØ)Ð5Ø+IÐLfÑ+fÑ(à+/Ð(à%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ*˜×*Ñ*Ð,CÐDˆð 7;×6LÑ6LØÐ7ó7
Ñ3ˆÐ3ñ 7H�= /Ñ2ÈmÐM]ˆñ !ð *Ð+IÑJà.Ð0ð 	ð Ð-Ð-Ð-rP   ©NF©rl   rm   rn   r1   rh   rp   rq   s   @rO   rs   rs   ¤   s   ø„ ô5pðv Ø÷H.rP   rs   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚIBertSelfOutputc           	      óŠ  •— t         ‰| �  «        |j                  | _        d| _        d| _        d| _        d| _        d| _        t        |j                  |j                  d| j                  | j
                  | j                  d¬«      | _
        t        | j                  | j                  ¬«      | _        t        |j                  |j                  | j                  | j                  |j                  ¬«      | _        t        | j                  | j                  ¬«      | _        t%        j&                  |j(                  «      | _        y ©Nr    r#   r"   Trw   r+   r,   )r0   r1   r&   r4   r%   ry   r5   r6   r   r8   Údenser   Úln_input_actr   rE   r/   rF   rG   r   rH   rI   rJ   rK   s     €rO   r1   zIBertSelfOutput.__init__(  sú   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØˆŒØˆŒØˆŒØˆÔØˆÔä Ø×ÑØ×ÑØØ—‘Ø—]‘]Ø—‘Øô
ˆŒ
ô % T×%6Ñ%6À4Ç?Á?ÔSˆÔÜ%Ø×ÑØ×%Ñ%Ø×)Ñ)Ø—‘Ø ×.Ñ.ô
ˆŒô "*¨$¯,©,À4Ç?Á?Ô!SˆÔÜ—z‘z &×"<Ñ"<Ó=ˆ�rP   c                 óÚ   — | j                  ||«      \  }}| j                  |«      }| j                  ||||¬«      \  }}| j                  ||«      \  }}| j	                  ||«      \  }}||fS ©NrU   ©r¶   rJ   r·   rF   rG   ©rL   r•   r–   Úinput_tensorÚinput_tensor_scaling_factors        rO   rh   zIBertSelfOutput.forwardE  ó’   € Ø6:·j±jÀÐPlÓ6mÑ3ˆÐ3ØŸ™ ]Ó3ˆØ6:×6GÑ6GØØ(Ø!Ø$?ð	 7Hó 7
Ñ3ˆÐ3ð 7;·n±nÀ]ÐTpÓ6qÑ3ˆÐ3à6:×6LÑ6LØÐ7ó7
Ñ3ˆÐ3ð Ð:Ð:Ð:rP   r±   rq   s   @rO   r³   r³   '  ó   ø„ ô>ö:;rP   r³   c                   ó*   ‡ — e Zd Zˆ fd„Z	 	 dd„Zˆ xZS )ÚIBertAttentionc                 ó„   •— t         ‰| �  «        |j                  | _        t        |«      | _        t        |«      | _        y ©N)r0   r1   r&   rs   rL   r³   ÚoutputrK   s     €rO   r1   zIBertAttention.__init__W  s3   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒÜ& vÓ.ˆŒ	Ü% fÓ-ˆ�rP   c                 ó–   — | j                  ||||«      \  }}| j                  |d   |d   ||«      \  }}|f|dd  z   }	|f|dd  z   }
|	|
fS )Nr   r   )rL   rÄ   )rL   r•   r–   r—   r˜   Úself_outputsÚself_outputs_scaling_factorÚattention_outputÚattention_output_scaling_factorr®   Úoutputs_scaling_factors              rO   rh   zIBertAttention.forward]  sŠ   € ð 59·I±IØØ(ØØó	5
Ñ1ˆÐ1ð =A¿K¹KØ˜‰OÐ8¸Ñ;¸]ÐLhó=
Ñ9ÐÐ9ð $Ð%¨°Q°RÐ(8Ñ8ˆØ"AÐ!CÐFaÐbcÐbdÐFeÑ!eÐØÐ.Ð.Ð.rP   r°   r±   rq   s   @rO   rÁ   rÁ   V  s   ø„ ô.ð Ø÷/rP   rÁ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚIBertIntermediatec           	      óÌ  •— t         ‰| �  «        |j                  | _        d| _        d| _        d| _        t        |j                  |j                  d| j                  | j
                  | j                  d¬«      | _	        |j                  dk7  rt        d«      ‚t        | j                  |j                  ¬«      | _        t        | j                  | j                  ¬«      | _        y )	Nr    r#   Trw   r	   z3I-BERT only supports 'gelu' for `config.hidden_act`r{   r+   )r0   r1   r&   r4   r%   ry   r   r8   Úintermediate_sizer¶   Ú
hidden_actr~   r   r/   Úintermediate_act_fnr   rG   rK   s     €rO   r1   zIBertIntermediate.__init__s  s¶   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØˆŒØˆŒØˆŒÜ Ø×ÑØ×$Ñ$ØØ—‘Ø—]‘]Ø—‘Øô
ˆŒ
ð ×Ñ Ò&ÜÐRÓSÐSÜ#*°d·o±oÐU[×UiÑUiÔ#jˆÔ Ü!)¨$¯,©,À4Ç?Á?Ô!SˆÕrP   c                 óˆ   — | j                  ||«      \  }}| j                  ||«      \  }}| j                  ||«      \  }}||fS rÃ   )r¶   rÐ   rG   )rL   r•   r–   s      rO   rh   zIBertIntermediate.forward‡  sa   € Ø6:·j±jÀÐPlÓ6mÑ3ˆÐ3Ø6:×6NÑ6NØÐ7ó7
Ñ3ˆÐ3ð
 7;×6LÑ6LØÐ7ó7
Ñ3ˆÐ3ð Ð:Ð:Ð:rP   r±   rq   s   @rO   rÌ   rÌ   r  s   ø„ ôTö(
;rP   rÌ   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚIBertOutputc           	      óŠ  •— t         ‰| �  «        |j                  | _        d| _        d| _        d| _        d| _        d| _        t        |j                  |j                  d| j                  | j
                  | j                  d¬«      | _        t        | j                  | j                  ¬«      | _        t        |j                  |j                  | j                  | j                  |j                   ¬«      | _        t        | j                  | j                  ¬«      | _        t'        j(                  |j*                  «      | _        y rµ   )r0   r1   r&   r4   r%   ry   r5   r6   r   rÎ   r8   r¶   r   r·   r   rE   r/   rF   rG   r   rH   rI   rJ   rK   s     €rO   r1   zIBertOutput.__init__•  sú   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØˆŒØˆŒØˆŒØˆÔØˆÔä Ø×$Ñ$Ø×ÑØØ—‘Ø—]‘]Ø—‘Øô
ˆŒ
ô % T×%6Ñ%6À4Ç?Á?ÔSˆÔÜ%Ø×ÑØ×%Ñ%Ø×)Ñ)Ø—‘Ø ×.Ñ.ô
ˆŒô "*¨$¯,©,À4Ç?Á?Ô!SˆÔÜ—z‘z &×"<Ñ"<Ó=ˆ�rP   c                 óÚ   — | j                  ||«      \  }}| j                  |«      }| j                  ||||¬«      \  }}| j                  ||«      \  }}| j	                  ||«      \  }}||fS r¹   rº   r»   s        rO   rh   zIBertOutput.forward²  r¾   rP   r±   rq   s   @rO   rÓ   rÓ   ”  r¿   rP   rÓ   c                   ó0   ‡ — e Zd Zˆ fd„Z	 	 dd„Zd„ Zˆ xZS )Ú
IBertLayerc                 óX  •— t         ‰| �  «        |j                  | _        d| _        d| _        t        |«      | _        t        |«      | _        t        |«      | _
        t        | j                  | j                  ¬«      | _        t        | j                  | j                  ¬«      | _        y )Nr    r   r+   )r0   r1   r&   r4   Úseq_len_dimrÁ   Ú	attentionrÌ   ÚintermediaterÓ   rÄ   r   Úpre_intermediate_actÚpre_output_actrK   s     €rO   r1   zIBertLayer.__init__Ä  s}   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØˆŒàˆÔÜ'¨Ó/ˆŒÜ-¨fÓ5ˆÔÜ! &Ó)ˆŒä$,¨T¯\©\ÀdÇoÁoÔ$VˆÔ!Ü& t§|¡|ÀÇÁÔPˆÕrP   c                 óŠ   — | j                  ||||¬«      \  }}|d   }|d   }|dd  }	| j                  ||«      \  }
}|
f|	z   }	|	S )N)r˜   r   r   )rÚ   Úfeed_forward_chunk)rL   r•   r–   r—   r˜   Úself_attention_outputsÚ%self_attention_outputs_scaling_factorrÈ   rÉ   r®   Úlayer_outputÚlayer_output_scaling_factors               rO   rh   zIBertLayer.forwardÑ  sƒ   € ð IMÏÉØØ(ØØ/ð	 IWó I
ÑEÐÐ Eð 2°!Ñ4ÐØ*OÐPQÑ*RÐ'à(¨¨Ð,ˆà48×4KÑ4KØÐ=ó5
Ñ1ˆÐ1ð  �/ GÑ+ˆàˆrP   c                 ó¶   — | j                  ||«      \  }}| j                  ||«      \  }}| j                  ||«      \  }}| j                  ||||«      \  }}||fS rÃ   )rÜ   rÛ   rÝ   rÄ   )rL   rÈ   rÉ   Úintermediate_outputÚ"intermediate_output_scaling_factorrâ   rã   s          rO   rß   zIBertLayer.feed_forward_chunkê  s“   € Ø<@×<UÑ<UØÐ=ó=
Ñ9ÐÐ9ð CG×BSÑBSØÐ=óC
Ñ?ÐÐ?ð CG×BUÑBUØÐ!CóC
Ñ?ÐÐ?ð 59·K±KØÐ!CÐEUÐWvó5
Ñ1ˆÐ1ð Ð8Ð8Ð8rP   r°   )rl   rm   rn   r1   rh   rß   rp   rq   s   @rO   r×   r×   Ã  s   ø„ ôQð" Øóö29rP   r×   c                   ó.   ‡ — e Zd Zˆ fd„Z	 	 	 	 dd„Zˆ xZS )ÚIBertEncoderc                 óä   •— t         ‰| �  «        || _        |j                  | _        t	        j
                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _	        y c c}w rÃ   )
r0   r1   rM   r&   r   Ú
ModuleListÚrangeÚnum_hidden_layersr×   Úlayer)rL   rM   Ú_rN   s      €rO   r1   zIBertEncoder.__init__ü  sP   ø€ Ü‰ÑÔØˆŒØ ×+Ñ+ˆŒÜ—]‘]ÄÀf×F^ÑF^Ó@_Ö#`¸1¤J¨vÕ$6Ò#`Óaˆ�
ùÒ#`s   ÁA-c                 óþ   — |rdnd }|rdnd }d }	t        | j                  «      D ])  \  }
}|r||fz   } |||||«      }|d   }|sŒ!||d   fz   }Œ+ |r||fz   }|st        d„ ||||	fD «       «      S t        ||||	¬«      S )N© r   r   c              3   ó$   K  — | ]  }|�|–— Œ
 y ­wrÃ   rð   )Ú.0Úvs     rO   ú	<genexpr>z'IBertEncoder.forward.<locals>.<genexpr>"  s   è ø€ ò 	àð �=ô ñ	ùs   ‚)Úlast_hidden_stater•   Ú
attentionsÚcross_attentions)Ú	enumeraterí   Útupler   )rL   r•   r–   r—   r˜   Úoutput_hidden_statesÚreturn_dictÚall_hidden_statesÚall_self_attentionsÚall_cross_attentionsÚiÚlayer_moduleÚlayer_outputss                rO   rh   zIBertEncoder.forward  sà   € ñ #7™B¸DÐÙ$5™b¸4ÐØ#Ðä(¨¯©Ó4ò 	P‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!á(ØØ,ØØ!ó	ˆMð *¨!Ñ,ˆMÚ Ø&9¸]È1Ñ=MÐ<OÑ&OÑ#ð	Pñ  Ø 1°]Ð4DÑ DÐáÜñ 	ð "Ø%Ø'Ø(ð	ô	ó 	ð 	ô 9Ø+Ø+Ø*Ø1ô	
ð 	
rP   )NFFTr±   rq   s   @rO   rè   rè   û  s   ø„ ôbð ØØ"Ø÷/
rP   rè   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚIBertPoolerc                 óÔ   •— t         ‰| �  «        |j                  | _        t        j                  |j
                  |j
                  «      | _        t        j                  «       | _        y rÃ   )	r0   r1   r&   r   ÚLinearr8   r¶   ÚTanhÚ
activationrK   s     €rO   r1   zIBertPooler.__init__5  sF   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'™'›)ˆ�rP   c                 ó\   — |d d …df   }| j                  |«      }| j                  |«      }|S ©Nr   )r¶   r  )rL   r•   Úfirst_token_tensorÚpooled_outputs       rO   rh   zIBertPooler.forward;  s6   € ð +ª1¨a¨4Ñ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐrP   r±   rq   s   @rO   r  r  4  s   ø„ ô$örP   r  c                   ó^   ‡ — e Zd ZU eed<   dZ ej                  «       ˆ fd„«       Zdd„Z	ˆ xZ
S )ÚIBertPreTrainedModelrM   Úibertc                 óL  •— t         ‰| �  |«       t        |t        «      rÚt	        j
                  |j                  d| j                  j                  ¬«       |j                  �t	        j                  |j                  «       t        |dd«      �>t	        j                  |j                  «       t	        j                  |j                  «       t        |dd«      � t	        j                  |j                  «       yyt        |t        «      rÒt	        j
                  |j                  d| j                  j                  ¬«       |j                   �Ct        |j                  dd«      s,t	        j                  |j                  |j                      «       t        |dd«      �?t	        j                  |j"                  «       t	        j                  |j                  «       yyt        |t$        «      r.t        |d	d«      � t	        j                  |j&                  «       yyt        |t(        «      r t	        j                  |j                  «       yt        |t*        «      rZt	        j,                  |j.                  t1        j2                  |j.                  j4                  d
   «      j7                  d«      «       yt        |t8        «      r`t	        j:                  |j<                  d«       t	        j:                  |j>                  d«       t	        j                  |j@                  «       yy)zInitialize the weightsg        )ÚmeanÚstdNÚweight_integerÚbias_integerÚ_is_hf_initializedFÚweight_scaling_factorÚshiftr)   r(   gñhãˆµøä¾gñhãˆµøä>)!r0   Ú_init_weightsÚ
isinstancer   ÚinitÚnormal_ÚweightrM   Úinitializer_rangerx   Úzeros_Úgetattrr  Úfc_scaling_factorr  r   r$   r  r   r  ÚIBertLMHeadr   Úcopy_r'   r>   r?   r�   rA   r   Ú	constant_Úx_minÚx_maxÚact_scaling_factor)rL   ÚmodulerN   s     €rO   r  z"IBertPreTrainedModel._init_weightsI  s  ø€ ô 	‰Ñ˜fÔ%Ü�fœkÔ*Ü�L‰L˜Ÿ™¨S°d·k±k×6SÑ6SÕTØ�{‰{Ð&Ü—‘˜FŸK™KÔ(Ü�vÐ/°Ó6ÐBÜ—‘˜F×1Ñ1Ô2Ü—‘˜F×4Ñ4Ô5Ü�v˜~¨tÓ4Ð@Ü—‘˜F×/Ñ/Õ0ð Aä˜¤Ô/Ü�L‰L˜Ÿ™¨S°d·k±k×6SÑ6SÕTà×!Ñ!Ð-´g¸f¿m¹mÐMaÐchÔ6iÜ—‘˜FŸM™M¨&×*<Ñ*<Ñ=Ô>Ü�vÐ6¸Ó=ÐIÜ—‘˜F×8Ñ8Ô9Ü—‘˜F×1Ñ1Õ2ð Jô ˜¤Ô-Ü�v˜w¨Ó-Ð9Ü—‘˜FŸL™LÕ)ð :ä˜¤Ô,Ü�K‰K˜Ÿ™Õ$Ü˜¤Ô0Ü�J‰J�v×*Ñ*¬E¯L©L¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÕhÜ˜¤Ô)Ü�N‰N˜6Ÿ<™<¨Ô/Ü�N‰N˜6Ÿ<™<¨Ô.Ü�K‰K˜×1Ñ1Õ2ð *rP   c                 ó   — t        d«      ‚)Nz6`resize_token_embeddings` is not supported for I-BERT.)ÚNotImplementedError)rL   Únew_num_tokenss     rO   Úresize_token_embeddingsz,IBertPreTrainedModel.resize_token_embeddingsj  s   € Ü!Ð"ZÓ[Ð[rP   rÃ   )rl   rm   rn   r   Ú__annotations__Úbase_model_prefixr>   Úno_gradr  r*  rp   rq   s   @rO   r  r  D  s0   ø… àÓØÐà€U‡]�]ƒ_ó3ó ð3÷@\rP   r  c                   ó,  ‡ — e Zd ZdZdˆ fd„	Zd„ Zd„ Ze	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dedz  dedz  dedz  deeej                     z  fd„«       Zˆ xZS )Ú
IBertModela®  

    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in [Attention is
    all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

    c                 óÜ   •— t         ‰| �  |«       || _        |j                  | _        t	        |«      | _        t        |«      | _        |rt        |«      nd| _	        | j                  «        y)zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        N)r0   r1   rM   r&   r   re   rè   Úencoderr  ÚpoolerÚ	post_init)rL   rM   Úadd_pooling_layerrN   s      €rO   r1   zIBertModel.__init__y  sZ   ø€ ô
 	‰Ñ˜Ô ØˆŒØ ×+Ñ+ˆŒä)¨&Ó1ˆŒÜ# FÓ+ˆŒá->”k &Ô)ÀDˆŒð 	�‰ÕrP   c                 ó.   — | j                   j                  S rÃ   ©re   r:   ©rL   s    rO   Úget_input_embeddingszIBertModel.get_input_embeddingsŠ  s   € Ø�‰×.Ñ.Ð.rP   c                 ó&   — || j                   _        y rÃ   r6  )rL   r„   s     rO   Úset_input_embeddingszIBertModel.set_input_embeddings�  s   € Ø*/ˆ�‰Õ'rP   Nr^   r—   r_   r'   r`   r˜   rú   rû   Úreturnc	                 óV  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|�|�t	        d«      ‚|�#| j                  ||«       |j                  «       }
n!|�|j                  «       d d }
nt	        d«      ‚|
\  }}|�|j                  n|j                  }|€t        j                  ||f|¬«      }|€&t        j                  |
t        j                  |¬«      }| j                  ||||¬«      \  }}t        | j                   ||¬«      }| j                  ||||||¬«      }|d	   }| j                  �| j                  |«      nd }|s
||f|d
d  z   S t!        |||j"                  |j$                  |j&                  ¬«      S )NzDYou cannot specify both input_ids and inputs_embeds at the same timer)   z5You have to specify either input_ids or inputs_embeds)rT   rR   )r^   r'   r_   r`   )rM   r`   r—   )r—   r˜   rú   rû   r   r   )rõ   Úpooler_outputr•   rö   r÷   )rM   r˜   rú   rû   r~   Ú%warn_if_padding_and_no_attention_maskr[   rT   r>   Úonesr\   r]   re   r
   r1  r2  r   r•   rö   r÷   )rL   r^   r—   r_   r'   r`   r˜   rú   rû   Úkwargsrb   Ú
batch_sizeÚ
seq_lengthrT   Úembedding_outputÚembedding_output_scaling_factorÚencoder_outputsÚsequence_outputr  s                      rO   rh   zIBertModel.forward�  sà  € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆàÐ  ]Ð%>ÜÐcÓdÐdØÐ"Ø×6Ñ6°yÀ.ÔQØ#Ÿ.™.Ó*‰KØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà!,Ñˆ
�JØ%.Ð%:�×!Ò!À×@TÑ@TˆàÐ!Ü"ŸZ™Z¨*°jÐ)AÈ6ÔRˆNØÐ!Ü"Ÿ[™[¨¼E¿J¹JÈvÔVˆNà<@¿O¹OØØ%Ø)Ø'ð	 =Ló =
Ñ9ÐÐ9ô 3Ø—;‘;Ø*Ø)ô
ˆð Ÿ,™,ØØ+Ø)Ø/Ø!5Ø#ð 'ó 
ˆð *¨!Ñ,ˆØ8<¿¹Ð8O˜Ÿ™ OÔ4ÐUYˆáØ# ]Ð3°oÀaÀbÐ6IÑIÐIä;Ø-Ø'Ø)×7Ñ7Ø&×1Ñ1Ø,×=Ñ=ô
ð 	
rP   )T)NNNNNNNN)rl   rm   rn   ro   r1   r8  r:  r   r>   Ú
LongTensorÚFloatTensorÚboolr   rù   rh   rp   rq   s   @rO   r/  r/  n  sû   ø„ ñõò"/ò0ð ð .2Ø37Ø26Ø04Ø26Ø)-Ø,0Ø#'ñE
à×#Ñ# dÑ*ðE
ð ×)Ñ)¨DÑ0ðE
ð ×(Ñ(¨4Ñ/ð	E
ð
 ×&Ñ&¨Ñ-ðE
ð ×(Ñ(¨4Ñ/ðE
ð   $™;ðE
ð # T™kðE
ð ˜D‘[ðE
ð 
6¸¸e×>OÑ>OÑ8PÑ	PòE
ó ôE
rP   r/  c                   óP  ‡ — e Zd ZdddœZˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 	 	 ddej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  deeej                     z  fd„«       Zˆ xZS )ÚIBertForMaskedLMz(ibert.embeddings.word_embeddings.weight$zlm_head.bias)zlm_head.decoder.weightzlm_head.decoder.biasc                 óˆ   •— t         ‰| �  |«       t        |d¬«      | _        t	        |«      | _        | j                  «        y ©NF)r4  )r0   r1   r/  r  r   Úlm_headr3  rK   s     €rO   r1   zIBertForMaskedLM.__init__à  s6   ø€ Ü‰Ñ˜Ô ä ¸%Ô@ˆŒ
Ü" 6Ó*ˆŒð 	�‰ÕrP   c                 ó.   — | j                   j                  S rÃ   )rN  Údecoderr7  s    rO   Úget_output_embeddingsz&IBertForMaskedLM.get_output_embeddingsé  s   € Ø�|‰|×#Ñ#Ð#rP   c                 ó\   — || j                   _        |j                  | j                   _        y rÃ   )rN  rP  rx   )rL   Únew_embeddingss     rO   Úset_output_embeddingsz&IBertForMaskedLM.set_output_embeddingsì  s    € Ø-ˆ�‰ÔØ*×/Ñ/ˆ�‰ÕrP   Nr^   r—   r_   r'   r`   Úlabelsr˜   rú   rû   r;  c
           
      ó˜  — |	�|	n| j                   j                  }	| j                  ||||||||	¬«      }|d   }| j                  |«      }d}|�Ft	        «       } ||j                  d| j                   j                  «      |j                  d«      «      }|	s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )a¢  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        N©r—   r_   r'   r`   r˜   rú   rû   r   r)   r‹   ©ÚlossÚlogitsr•   rö   )
rM   rû   r  rN  r   rŽ   r7   r   r•   rö   )rL   r^   r—   r_   r'   r`   rU  r˜   rú   rû   r@  r®   rF  Úprediction_scoresÚmasked_lm_lossÚloss_fctrÄ   s                    rO   rh   zIBertForMaskedLM.forwardð  s÷   € ð( &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—*‘*ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ó 	
ˆð " !™*ˆØ ŸL™L¨Ó9ÐàˆØÐÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNáØ'Ð)¨G°A°B¨KÑ7ˆFØ3AÐ3M�^Ð%¨Ñ.ÐYÐSYÐYäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
rP   ©	NNNNNNNNN)rl   rm   rn   Ú_tied_weights_keysr1   rQ  rT  r   r>   rG  rH  rI  r   rù   rh   rp   rq   s   @rO   rK  rK  Ù  s  ø„ ð #MØ .ñÐô
ò$ò0ð ð .2Ø37Ø26Ø04Ø26Ø*.Ø)-Ø,0Ø#'ñ0
à×#Ñ# dÑ*ð0
ð ×)Ñ)¨DÑ0ð0
ð ×(Ñ(¨4Ñ/ð	0
ð
 ×&Ñ&¨Ñ-ð0
ð ×(Ñ(¨4Ñ/ð0
ð × Ñ  4Ñ'ð0
ð   $™;ð0
ð # T™kð0
ð ˜D‘[ð0
ð 
˜% × 1Ñ 1Ñ2Ñ	2ò0
ó ô0
rP   rK  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )r   z)I-BERT Head for masked language modeling.c                 ó¬  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  |j                  «      | _
        t        j                  t        j                  |j                  «      «      | _        y )N)r-   )r0   r1   r   r  r8   r¶   rF   rE   Ú
layer_normr7   rP  Ú	Parameterr>   r\   rx   rK   s     €rO   r1   zIBertLMHead.__init__'  s‡   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒä—y‘y ×!3Ñ!3°V×5FÑ5FÓGˆŒÜ—L‘L¤§¡¨V×->Ñ->Ó!?Ó@ˆ�	rP   c                 ó‚   — | j                  |«      }t        |«      }| j                  |«      }| j                  |«      }|S rÃ   )r¶   r	   rb  rP  )rL   Úfeaturesr@  Úxs       rO   rh   zIBertLMHead.forward/  s;   € Ø�J‰J�xÓ ˆÜ�‹GˆØ�O‰O˜AÓˆð �L‰L˜‹OˆàˆrP   ©rl   rm   rn   ro   r1   rh   rp   rq   s   @rO   r   r   $  s   ø„ Ù3ôAörP   r   zž
    I-BERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    )Úcustom_introc                   ó:  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dedz  de	e
ej                     z  fd„«       Zˆ xZS )ÚIBertForSequenceClassificationc                 óª   •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        t        |«      | _        | j                  «        y rM  )r0   r1   Ú
num_labelsr/  r  ÚIBertClassificationHeadÚ
classifierr3  rK   s     €rO   r1   z'IBertForSequenceClassification.__init__A  sC   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä ¸%Ô@ˆŒ
Ü1°&Ó9ˆŒð 	�‰ÕrP   Nr^   r—   r_   r'   r`   rU  r˜   rú   rû   r;  c
           
      ó  — |	�|	n| j                   j                  }	| j                  ||||||||	¬«      }|d   }| j                  |«      }d}|��‡| j                   j                  €�| j
                  dk(  rd| j                   _        nl| j
                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j                  dk(  rIt        «       }| j
                  dk(  r& ||j                  «       |j                  «       «      }nŒ |||«      }n‚| j                   j                  dk(  r=t        «       } ||j                  d| j
                  «      |j                  d«      «      }n,| j                   j                  dk(  rt        «       } |||«      }|	s|f|d	d z   }|�|f|z   S |S t        |||j                   |j"                  ¬
«      S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NrW  r   r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr)   r‹   rX  )rM   rû   r  rn  Úproblem_typerl  rS   r>   r]   r   r   Úsqueezer   rŽ   r   r   r•   rö   ©rL   r^   r—   r_   r'   r`   rU  r˜   rú   rû   r@  r®   rF  rZ  rY  r]  rÄ   s                    rO   rh   z&IBertForSequenceClassification.forwardK  s×  € ð( &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—*‘*ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ó 	
ˆð " !™*ˆØ—‘ Ó1ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ÙØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rP   r^  )rl   rm   rn   r1   r   r>   rG  rH  rI  r   rù   rh   rp   rq   s   @rO   rj  rj  :  s  ø„ ôð ð .2Ø37Ø26Ø04Ø26Ø*.Ø)-Ø,0Ø#'ñA
à×#Ñ# dÑ*ðA
ð ×)Ñ)¨DÑ0ðA
ð ×(Ñ(¨4Ñ/ð	A
ð
 ×&Ñ&¨Ñ-ðA
ð ×(Ñ(¨4Ñ/ðA
ð × Ñ  4Ñ'ðA
ð   $™;ðA
ð # T™kðA
ð ˜D‘[ðA
ð 
" E¨%×*;Ñ*;Ñ$<Ñ	<òA
ó ôA
rP   rj  c                   ó:  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dedz  de	e
ej                     z  fd„«       Zˆ xZS )ÚIBertForMultipleChoicec                 óö   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  «      | _        t	        j                  |j                  d«      | _
        | j                  «        y )Nr   )r0   r1   r/  r  r   rH   rI   rJ   r  r8   rn  r3  rK   s     €rO   r1   zIBertForMultipleChoice.__init__’  sV   ø€ Ü‰Ñ˜Ô ä Ó'ˆŒ
Ü—z‘z &×"<Ñ"<Ó=ˆŒÜŸ)™) F×$6Ñ$6¸Ó:ˆŒð 	�‰ÕrP   Nr^   r_   r—   rU  r'   r`   r˜   rú   rû   r;  c
           
      óJ  — |	�|	n| j                   j                  }	|�|j                  d   n|j                  d   }|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�1|j                  d|j	                  d«      |j	                  d«      «      nd}| j                  ||||||||	¬«      }|d   }| j                  |«      }| j                  |«      }|j                  d|«      }d}|�t        «       } |||«      }|	s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        Nr   r)   rŒ   )r'   r_   r—   r`   r˜   rú   rû   r‹   rX  )rM   rû   r�   rŽ   r[   r  rJ   rn  r   r   r•   rö   )rL   r^   r_   r—   rU  r'   r`   r˜   rú   rû   r@  Únum_choicesÚflat_input_idsÚflat_position_idsÚflat_token_type_idsÚflat_attention_maskÚflat_inputs_embedsr®   r  rZ  Úreshaped_logitsrY  r]  rÄ   s                           rO   rh   zIBertForMultipleChoice.forwardœ  sÞ  € ðX &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆØ,5Ð,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆàCLÐCX˜Ÿ™¨¨I¯N©N¸2Ó,>Ô?Ð^bˆØLXÐLd˜L×-Ñ-¨b°,×2CÑ2CÀBÓ2GÔHÐjnÐØR`ÐRl˜n×1Ñ1°"°n×6IÑ6IÈ"Ó6MÔNÐrvÐØR`ÐRl˜n×1Ñ1°"°n×6IÑ6IÈ"Ó6MÔNÐrvÐð Ð(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —*‘*ØØ*Ø.Ø.Ø,Ø/Ø!5Ø#ð ó 	
ˆð   ™
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÐÜ'Ó)ˆHÙ˜O¨VÓ4ˆDáØ%Ð'¨'°!°"¨+Ñ5ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
rP   r^  )rl   rm   rn   r1   r   r>   rG  rH  rI  r   rù   rh   rp   rq   s   @rO   rw  rw  �  s  ø„ ôð ð .2Ø26Ø37Ø*.Ø04Ø26Ø)-Ø,0Ø#'ñV
à×#Ñ# dÑ*ðV
ð ×(Ñ(¨4Ñ/ðV
ð ×)Ñ)¨DÑ0ð	V
ð
 × Ñ  4Ñ'ðV
ð ×&Ñ&¨Ñ-ðV
ð ×(Ñ(¨4Ñ/ðV
ð   $™;ðV
ð # T™kðV
ð ˜D‘[ðV
ð 
# U¨5×+<Ñ+<Ñ%=Ñ	=òV
ó ôV
rP   rw  c                   ó:  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dedz  de	e
ej                     z  fd„«       Zˆ xZS )ÚIBertForTokenClassificationc                 ó0  •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j                  «        y rM  )r0   r1   rl  r/  r  r   rH   rI   rJ   r  r8   rn  r3  rK   s     €rO   r1   z$IBertForTokenClassification.__init__ø  sk   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä ¸%Ô@ˆŒ
Ü—z‘z &×"<Ñ"<Ó=ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÕrP   Nr^   r—   r_   r'   r`   rU  r˜   rú   rû   r;  c
           
      ó¦  — |	�|	n| j                   j                  }	| j                  ||||||||	¬«      }|d   }| j                  |«      }| j	                  |«      }d}|�<t        «       } ||j                  d| j                  «      |j                  d«      «      }|	s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        NrW  r   r)   r‹   rX  )rM   rû   r  rJ   rn  r   rŽ   rl  r   r•   rö   ru  s                    rO   rh   z#IBertForTokenClassification.forward  só   € ð$ &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—*‘*ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ó 	
ˆð " !™*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÐÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDáØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
rP   r^  )rl   rm   rn   r1   r   r>   rG  rH  rI  r   rù   rh   rp   rq   s   @rO   r‚  r‚  ö  sù   ø„ ô	ð ð .2Ø37Ø26Ø04Ø26Ø*.Ø)-Ø,0Ø#'ñ1
à×#Ñ# dÑ*ð1
ð ×)Ñ)¨DÑ0ð1
ð ×(Ñ(¨4Ñ/ð	1
ð
 ×&Ñ&¨Ñ-ð1
ð ×(Ñ(¨4Ñ/ð1
ð × Ñ  4Ñ'ð1
ð   $™;ð1
ð # T™kð1
ð ˜D‘[ð1
ð 
  u×'8Ñ'8Ñ!9Ñ	9ò1
ó ô1
rP   r‚  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )rm  z-Head for sentence-level classification tasks.c                 ó&  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _
        y rÃ   )r0   r1   r   r  r8   r¶   rH   rI   rJ   rl  Úout_projrK   s     €rO   r1   z IBertClassificationHead.__init__;  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü—z‘z &×"<Ñ"<Ó=ˆŒÜŸ	™	 &×"4Ñ"4°f×6GÑ6GÓHˆ�rP   c                 óÐ   — |d d …dd d …f   }| j                  |«      }| j                  |«      }t        j                  |«      }| j                  |«      }| j	                  |«      }|S r	  )rJ   r¶   r>   Útanhr‡  )rL   re  r@  r•   s       rO   rh   zIBertClassificationHead.forwardA  s^   € Ø ¢ A¢q Ñ)ˆØŸ™ ]Ó3ˆØŸ
™
 =Ó1ˆÜŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ mÓ4ˆØÐrP   rg  rq   s   @rO   rm  rm  8  s   ø„ Ù7ôIörP   rm  c                   óZ  ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
edz  dedz  dedz  de	e
ej                     z  fd„«       Zˆ xZS )ÚIBertForQuestionAnsweringc                 óè   •— t         ‰| �  |«       |j                  | _        t        |d¬«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y rM  )
r0   r1   rl  r/  r  r   r  r8   Ú
qa_outputsr3  rK   s     €rO   r1   z"IBertForQuestionAnswering.__init__M  sU   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä ¸%Ô@ˆŒ
ÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÕrP   Nr^   r—   r_   r'   r`   Ústart_positionsÚend_positionsr˜   rú   rû   r;  c           
      ó&  — |
�|
n| j                   j                  }
| j                  |||||||	|
¬«      }|d   }| j                  |«      }|j	                  dd¬«      \  }}|j                  d«      j                  «       }|j                  d«      j                  «       }d }|�·|�µt        |j                  «       «      dkD  r|j                  d«      }t        |j                  «       «      dkD  r|j                  d«      }|j                  d«      }|j                  d|«      }|j                  d|«      }t        |¬«      } |||«      } |||«      }||z   dz  }|
s||f|dd  z   }|�|f|z   S |S t        ||||j                  |j                  ¬«      S )	NrW  r   r   r)   ©Údim)Úignore_indexr‹   )rY  Ústart_logitsÚ
end_logitsr•   rö   )rM   rû   r  r�  Úsplitrt  r”   Úlenr[   Úclampr   r   r•   rö   )rL   r^   r—   r_   r'   r`   rŽ  r�  r˜   rú   rû   r@  r®   rF  rZ  r”  r•  Ú
total_lossÚignored_indexr]  Ú
start_lossÚend_lossrÄ   s                          rO   rh   z!IBertForQuestionAnswering.forwardW  s¿  € ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—*‘*ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ó 	
ˆð " !™*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJáØ" JÐ/°'¸!¸"°+Ñ=ˆFØ/9Ð/E�Z�M FÑ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
rP   )
NNNNNNNNNN)rl   rm   rn   r1   r   r>   rG  rH  rI  r   rù   rh   rp   rq   s   @rO   r‹  r‹  K  s  ø„ ôð ð .2Ø37Ø26Ø04Ø26Ø37Ø15Ø)-Ø,0Ø#'ñ=
à×#Ñ# dÑ*ð=
ð ×)Ñ)¨DÑ0ð=
ð ×(Ñ(¨4Ñ/ð	=
ð
 ×&Ñ&¨Ñ-ð=
ð ×(Ñ(¨4Ñ/ð=
ð ×)Ñ)¨DÑ0ð=
ð ×'Ñ'¨$Ñ.ð=
ð   $™;ð=
ð # T™kð=
ð ˜D‘[ð=
ð 
&¨¨e×.?Ñ.?Ñ(@Ñ	@ò=
ó ô=
rP   r‹  c                 ó¾   — | j                  |«      j                  «       }t        j                  |d¬«      j	                  |«      |z   |z  }|j                  «       |z   S )aM  
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's *utils.make_positions*.

    Args:
    input_ids (`torch.LongTensor`):
           Indices of input sequence tokens in the vocabulary.

    Returns: torch.Tensor
    r   r‘  )Úner   r>   ÚcumsumÚtype_asr]   )r^   r$   ra   ÚmaskÚincremental_indicess        rO   rX   rX   ˜  sW   € ð �<‰<˜Ó$×(Ñ(Ó*€DÜ Ÿ<™<¨°!Ô4×<Ñ<¸TÓBÐE[Ñ[Ð_cÑcÐØ×#Ñ#Ó%¨Ñ3Ð3rP   )rK  rw  r‹  rj  r‚  r/  r  )r   )=ro   r‘   r>   r   Útorch.nnr   r   r   Ú r   r  Úactivationsr	   Úmasking_utilsr
   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_ibertr   Úquant_modulesr   r   r   r   r   r   Ú
get_loggerrl   ÚloggerÚModuler   rs   r³   rÁ   rÌ   rÓ   r×   rè   r  r  r/  rK  r   rj  rw  r‚  rm  r‹  rX   Ú__all__rð   rP   rO   ú<module>r°     s  ðñ" ã ã Ý ß AÑ Aå &Ý Ý 6÷÷ ñ õ .ß ,Ý ,ß c× cð 
ˆ×	Ñ	˜HÓ	%€ôs=�b—i‘iô s=ôl@.˜Ÿ™ô @.ôF,;�b—i‘iô ,;ô^/�R—Y‘Yô /ô8;˜Ÿ	™	ô ;ôD,;�"—)‘)ô ,;ô^59�—‘ô 59ôp6
�2—9‘9ô 6
ôr�"—)‘)ô ð  ô&\˜?ó &\ó ð&\ðR ôg
Ð%ó g
ó ðg
ðT ôG
Ð+ó G
ó ðG
ôT�"—)‘)ô ñ, ðôôM
Ð%9ó M
óðM
ð` ôb
Ð1ó b
ó ðb
ðJ ô>
Ð"6ó >
ó ð>
ôB˜bŸi™iô ð& ôI
Ð 4ó I
ó ðI
óX4ò"�rP   