
    ^j]                        d Z ddlZddlZddlmZ ddlmZmZmZ ddlm	Z
 ddlmZ ddlmZ dd	lmZmZmZmZmZmZmZ dd
lmZ ddlmZmZ ddlmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$  ejJ                  e&      Z' G d dejP                        Z) G d dejP                        Z* G d dejP                        Z+ G d dejP                        Z, G d dejP                        Z- G d dejP                        Z. G d dejP                        Z/ G d dejP                        Z0 G d d ejP                        Z1e G d! d"e             Z2e G d# d$e2             Z3e G d% d&e2             Z4 G d' d(ejP                        Z5 ed)*       G d+ d,e2             Z6e G d- d.e2             Z7e G d/ d0e2             Z8 G d1 d2ejP                        Z9e G d3 d4e2             Z:d7d5Z;g d6Z<y)8zPyTorch I-BERT model.    N)nn)BCEWithLogitsLossCrossEntropyLossMSELoss   )initialization)gelu)create_bidirectional_mask))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)auto_docstringlogging   )IBertConfig)IntGELUIntLayerNorm
IntSoftmaxQuantActQuantEmbeddingQuantLinearc                   2     e Zd ZdZ fdZ	 ddZd Z xZS )IBertEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    c                 z   t         |           |j                  | _        d| _        d| _        d| _        d| _        d| _        t        |j                  |j                  |j                  | j                  | j                        | _        t        |j                  |j                  | j                  | j                        | _        | j                  dt!        j"                  |j$                        j'                  d      d	
       |j                  | _        t        |j$                  |j                  | j(                  | j                  | j                        | _        t-        | j                  | j                        | _        t-        | j                  | j                        | _        t3        |j                  |j4                  | j                  | j                  |j6                        | _        t-        | j
                  | j                        | _        t=        j>                  |j@                        | _!        y )N             )padding_idx
weight_bit
quant_mode)r%   r&   position_idsr   F)
persistentr&   eps
output_bitr&   force_dequant)"super__init__r&   embedding_bitembedding_act_bitact_bitln_input_bitln_output_bitr   
vocab_sizehidden_sizepad_token_idword_embeddingstype_vocab_sizetoken_type_embeddingsregister_buffertorcharangemax_position_embeddingsexpandr$   position_embeddingsr   embeddings_act1embeddings_act2r   layer_norm_epsr/   	LayerNormoutput_activationr   Dropouthidden_dropout_probdropoutselfconfig	__class__s     s/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/ibert/modeling_ibert.pyr1   zIBertEmbeddings.__init__3   s    ++!#-++)) 
 &4""F$6$64CUCUbfbqbq&
"
 	ELL)G)GHOOPWXej 	 	

 "..#1**(())$
   ((>(>4??['(>(>4??[%%%)) ..
 "*$,,4??!Szz&"<"<=    c                    |D|1t        || j                  |      j                  |j                        }n| j	                  |      }||j                         }n|j                         d d }|:t        j                  |t        j                  | j                  j                        }|| j                  |      \  }}nd }| j                  |      \  }}	| j                  ||||	      \  }
}| j                  |      \  }}| j                  |
|||      \  }
}| j                  |
|      \  }
}| j                  |
      }
| j!                  |
|      \  }
}|
|fS )Nr)   dtypedeviceidentityidentity_scaling_factor)"create_position_ids_from_input_idsr$   torT   &create_position_ids_from_inputs_embedssizer>   zeroslongr'   r:   r<   rC   rB   rF   rJ   rG   )rL   	input_idstoken_type_idsr'   inputs_embedspast_key_values_lengthinput_shapeinputs_embeds_scaling_factorr<   $token_type_embeddings_scaling_factor
embeddingsembeddings_scaling_factorrB   "position_embeddings_scaling_factors                 rO   forwardzIBertEmbeddings.forwardd   s    $At//1G "Y%%&   $JJ=Y #..*K',,.s3K!"[[EJJtO`O`OgOghN :>:N:Ny:Y7M7+/(FJF`F`aoFpCC040D0D(*$H	 1E 1
-
- CGBZBZ[gBh??040D0D%($F	 1E 1
-
- 15zKd0e-
-\\*-
040F0FzSl0m-
-444rP   c                    |j                         dd }|d   }t        j                  | j                  dz   || j                  z   dz   t        j                  |j
                        }|j                  d      j                  |      S )z
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: torch.Tensor

        Returns: torch.Tensor
        Nr)   r   rR   r   )r[   r>   r?   r$   r]   rT   	unsqueezerA   )rL   r`   rb   sequence_lengthr'   s        rO   rZ   z6IBertEmbeddings.create_position_ids_from_inputs_embeds   s     $((*3B/%a.||q /D4D4D"Dq"HPUPZPZcpcwcw
 %%a(//<<rP   )NNNNr   )__name__
__module____qualname____doc__r1   rh   rZ   __classcell__rN   s   @rO   r   r   .   s     />d rs,5\=rP   r   c                   *     e Zd Z fdZ	 	 ddZ xZS )IBertSelfAttentionc           	      F   t         |           |j                  |j                  z  dk7  r2t	        |d      s&t        d|j                   d|j                   d      |j                  | _        d| _        d| _        d| _	        |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _        t        |j                  | j                  d| j                  | j                  | j                  d	      | _        t        |j                  | j                  d| j                  | j                  | j                  d	      | _        t        |j                  | j                  d| j                  | j                  | j                  d	      | _        t#        | j                  | j                  
      | _        t#        | j                  | j                  
      | _        t#        | j                  | j                  
      | _        t#        | j                  | j                  
      | _        t-        j.                  |j0                        | _        t5        | j                  | j                  |j6                        | _        y )Nr   embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()r    r#   Tbiasr%   bias_bitr&   per_channelr+   r&   r/   )r0   r1   r8   num_attention_headshasattr
ValueErrorr&   r%   ry   r4   intattention_head_sizeall_head_sizer   querykeyvaluer   query_activationkey_activationvalue_activationrG   r   rH   attention_probs_dropout_probrJ   r   r/   softmaxrK   s     rO   r1   zIBertSelfAttention.__init__   s    : ::a?PVXhHi#F$6$6#7 8 445Q8  !++#)#=#= #&v'9'9F<V<V'V#W !558P8PP !]]

 ]]
 !]]

 !)$// R&t||P ($// R!)$,,4??!Szz&"E"EF!$,,4??Z`ZnZnorP   c                 ^   | j                  ||      \  }}| j                  ||      \  }}| j                  ||      \  }	}
| j                  ||      \  }}| j	                  ||      \  }}| j                  |	|
      \  }}|j                  d d }g |d| j                  }|j                  |      j                  dd      }|j                  |      j                  dd      }|j                  |      j                  dd      }t        j                  ||j                  dd            }t        j                  | j                        }||z  }| j                  r	||z  |z  }nd }|||z   }| j                  ||      \  }}| j!                  |      }t        j                  ||      }|||z  }nd }|j#                  dddd      j%                         }|j'                         d d | j(                  fz   } |j                  | }| j+                  ||      \  }}|r||fn|f}|r||fn|f}||fS )Nr)   r      r   r   )r   r   r   r   r   r   shaper   view	transposer>   matmulmathsqrtr&   r   rJ   permute
contiguousr[   r   rG   )rL   hidden_stateshidden_states_scaling_factorattention_maskoutput_attentionsmixed_query_layer mixed_query_layer_scaling_factormixed_key_layermixed_key_layer_scaling_factormixed_value_layer mixed_value_layer_scaling_factorquery_layerquery_layer_scaling_factor	key_layerkey_layer_scaling_factorvalue_layervalue_layer_scaling_factorrb   hidden_shapeattention_scoresscaleattention_scores_scaling_factorattention_probsattention_probs_scaling_factorcontext_layercontext_layer_scaling_factornew_context_layer_shapeoutputsoutput_scaling_factors                                rO   rh   zIBertSelfAttention.forward   s    ?CjjXt>u;;:>((=Rn:o77>BjjXt>u;; 372G2G?3
// /3.A.A/Sq.r+	+262G2G?3
//
 $))#2.CCbC$*B*BC!&&|4>>q!DNN<0::1a@	!&&|4>>q!D !<<Y5H5HR5PQ		$223+e3??.HKc.cfk.k+.2+%/.@ ;?,,=;
77 ,,7_kB)5+ILf+f(+/(%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S***,CD 7;6L6L77
33 7H=/2mM] ! *+IJ.0 	 ---rP   NFrl   rm   rn   r1   rh   rp   rq   s   @rO   rs   rs      s    5pv H.rP   rs   c                   $     e Zd Z fdZd Z xZS )IBertSelfOutputc           	         t         |           |j                  | _        d| _        d| _        d| _        d| _        d| _        t        |j                  |j                  d| j                  | j
                  | j                  d      | _
        t        | j                  | j                        | _        t        |j                  |j                  | j                  | j                  |j                        | _        t        | j                  | j                        | _        t%        j&                  |j(                        | _        y Nr    r#   r"   Trw   r+   r,   )r0   r1   r&   r4   r%   ry   r5   r6   r   r8   denser   ln_input_actr   rE   r/   rF   rG   r   rH   rI   rJ   rK   s     rO   r1   zIBertSelfOutput.__init__(  s     ++ ]]

 %T%6%64??S%%%)) ..
 "*$,,4??!Szz&"<"<=rP   c                     | j                  ||      \  }}| j                  |      }| j                  ||||      \  }}| j                  ||      \  }}| j	                  ||      \  }}||fS NrU   r   rJ   r   rF   rG   rL   r   r   input_tensorinput_tensor_scaling_factors        rO   rh   zIBertSelfOutput.forwardE      6:jjPl6m33]36:6G6G(!$?	 7H 7
33 7;nn]Tp6q336:6L6L77
33 :::rP   r   rq   s   @rO   r   r   '      >:;rP   r   c                   *     e Zd Z fdZ	 	 ddZ xZS )IBertAttentionc                     t         |           |j                  | _        t        |      | _        t        |      | _        y N)r0   r1   r&   rs   rL   r   outputrK   s     rO   r1   zIBertAttention.__init__W  s3     ++&v.	%f-rP   c                     | j                  ||||      \  }}| j                  |d   |d   ||      \  }}|f|dd  z   }	|f|dd  z   }
|	|
fS )Nr   r   )rL   r   )rL   r   r   r   r   self_outputsself_outputs_scaling_factorattention_outputattention_output_scaling_factorr   outputs_scaling_factors              rO   rh   zIBertAttention.forward]  s     59II(	5
11 =AKKO8;]Lh=
99 $%QR(88"A!CFabcbdFe!e...rP   r   r   rq   s   @rO   r   r   V  s    . /rP   r   c                   $     e Zd Z fdZd Z xZS )IBertIntermediatec           	         t         |           |j                  | _        d| _        d| _        d| _        t        |j                  |j                  d| j                  | j
                  | j                  d      | _	        |j                  dk7  rt        d      t        | j                  |j                        | _        t        | j                  | j                        | _        y )	Nr    r#   Trw   r	   z3I-BERT only supports 'gelu' for `config.hidden_act`r{   r+   )r0   r1   r&   r4   r%   ry   r   r8   intermediate_sizer   
hidden_actr~   r   r/   intermediate_act_fnr   rG   rK   s     rO   r1   zIBertIntermediate.__init__s  s     ++ $$]]

 &RSS#*dooU[UiUi#j !)$,,4??!SrP   c                     | j                  ||      \  }}| j                  ||      \  }}| j                  ||      \  }}||fS r   )r   r   rG   )rL   r   r   s      rO   rh   zIBertIntermediate.forward  sa    6:jjPl6m336:6N6N77
33
 7;6L6L77
33 :::rP   r   rq   s   @rO   r   r   r  s    T(
;rP   r   c                   $     e Zd Z fdZd Z xZS )IBertOutputc           	         t         |           |j                  | _        d| _        d| _        d| _        d| _        d| _        t        |j                  |j                  d| j                  | j
                  | j                  d      | _        t        | j                  | j                        | _        t        |j                  |j                  | j                  | j                  |j                         | _        t        | j                  | j                        | _        t'        j(                  |j*                        | _        y r   )r0   r1   r&   r4   r%   ry   r5   r6   r   r   r8   r   r   r   r   rE   r/   rF   rG   r   rH   rI   rJ   rK   s     rO   r1   zIBertOutput.__init__  s     ++ $$]]

 %T%6%64??S%%%)) ..
 "*$,,4??!Szz&"<"<=rP   c                     | j                  ||      \  }}| j                  |      }| j                  ||||      \  }}| j                  ||      \  }}| j	                  ||      \  }}||fS r   r   r   s        rO   rh   zIBertOutput.forward  r   rP   r   rq   s   @rO   r   r     r   rP   r   c                   0     e Zd Z fdZ	 	 ddZd Z xZS )
IBertLayerc                 X   t         |           |j                  | _        d| _        d| _        t        |      | _        t        |      | _        t        |      | _
        t        | j                  | j                        | _        t        | j                  | j                        | _        y )Nr    r   r+   )r0   r1   r&   r4   seq_len_dimr   	attentionr   intermediater   r   r   pre_intermediate_actpre_output_actrK   s     rO   r1   zIBertLayer.__init__  s}     ++'/-f5!&)$,T\\doo$V!&t||PrP   c                     | j                  ||||      \  }}|d   }|d   }|dd  }	| j                  ||      \  }
}|
f|	z   }	|	S )N)r   r   r   )r   feed_forward_chunk)rL   r   r   r   r   self_attention_outputs%self_attention_outputs_scaling_factorr   r   r   layer_outputlayer_output_scaling_factors               rO   rh   zIBertLayer.forward  s     IM(/	 IW I
E E 2!4*OPQ*R'(,484K4K=5
11  /G+rP   c                     | j                  ||      \  }}| j                  ||      \  }}| j                  ||      \  }}| j                  ||||      \  }}||fS r   )r   r   r   r   )rL   r   r   intermediate_output"intermediate_output_scaling_factorr   r   s          rO   r   zIBertLayer.feed_forward_chunk  s    <@<U<U==
99 CGBSBS=C
?? CGBUBU!CC
?? 59KK!CEUWv5
11 888rP   r   )rl   rm   rn   r1   rh   r   rp   rq   s   @rO   r   r     s    Q" 29rP   r   c                   .     e Zd Z fdZ	 	 	 	 ddZ xZS )IBertEncoderc                     t         |           || _        |j                  | _        t	        j
                  t        |j                        D cg c]  }t        |       c}      | _	        y c c}w r   )
r0   r1   rM   r&   r   
ModuleListrangenum_hidden_layersr   layer)rL   rM   _rN   s      rO   r1   zIBertEncoder.__init__  sP     ++]]fF^F^@_#`1Jv$6#`a
#`s   A-c                     |rdnd }|rdnd }d }	t        | j                        D ])  \  }
}|r||fz   } |||||      }|d   }|s!||d   fz   }+ |r||fz   }|st        d ||||	fD              S t        ||||	      S )N r   r   c              3   $   K   | ]  }|| 
 y wr   r   ).0vs     rO   	<genexpr>z'IBertEncoder.forward.<locals>.<genexpr>"  s      	 = 	s   )last_hidden_stater   
attentionscross_attentions)	enumerater   tupler   )rL   r   r   r   r   output_hidden_statesreturn_dictall_hidden_statesall_self_attentionsall_cross_attentionsilayer_modulelayer_outputss                rO   rh   zIBertEncoder.forward  s     #7BD$5b4#(4 	POA|#$58H$H!(,!	M *!,M &9]1=M<O&O#	P   1]4D D 	 "%'(		 	 	 9++*1	
 	
rP   )NFFTr   rq   s   @rO   r   r     s    b "/
rP   r   c                   $     e Zd Z fdZd Z xZS )IBertPoolerc                     t         |           |j                  | _        t        j                  |j
                  |j
                        | _        t        j                         | _        y r   )	r0   r1   r&   r   Linearr8   r   Tanh
activationrK   s     rO   r1   zIBertPooler.__init__5  sF     ++YYv1163E3EF
'')rP   c                 \    |d d df   }| j                  |      }| j                  |      }|S Nr   )r   r  )rL   r   first_token_tensorpooled_outputs       rO   rh   zIBertPooler.forward;  s6     +1a40

#566rP   r   rq   s   @rO   r  r  4  s    $rP   r  c                   ^     e Zd ZU eed<   dZ ej                          fd       ZddZ	 xZ
S )IBertPreTrainedModelrM   ibertc                 L   t         |   |       t        |t              rt	        j
                  |j                  d| j                  j                         |j                  t	        j                  |j                         t        |dd      >t	        j                  |j                         t	        j                  |j                         t        |dd       t	        j                  |j                         yyt        |t              rt	        j
                  |j                  d| j                  j                         |j                   Ct        |j                  dd      s,t	        j                  |j                  |j                             t        |dd      ?t	        j                  |j"                         t	        j                  |j                         yyt        |t$              r.t        |d	d       t	        j                  |j&                         yyt        |t(              r t	        j                  |j                         yt        |t*              rZt	        j,                  |j.                  t1        j2                  |j.                  j4                  d
         j7                  d             yt        |t8              r`t	        j:                  |j<                  d       t	        j:                  |j>                  d       t	        j                  |j@                         yy)zInitialize the weightsg        )meanstdNweight_integerbias_integer_is_hf_initializedFweight_scaling_factorshiftr)   r(   gh㈵gh㈵>)!r0   _init_weights
isinstancer   initnormal_weightrM   initializer_rangerx   zeros_getattrr  fc_scaling_factorr  r   r$   r  r   r  IBertLMHeadr   copy_r'   r>   r?   r   rA   r   	constant_x_minx_maxact_scaling_factor)rL   modulerN   s     rO   r  z"IBertPreTrainedModel._init_weightsI  s    	f%fk*LLSdkk6S6ST{{&FKK(v/6BF112F445v~t4@F//0 A/LLSdkk6S6ST!!-gfmmMach6iFMM&*<*<=>v6=IF889F112 J -vw-9FLL) :,KK$0JJv**ELL9L9L9R9RSU9V,W,^,^_f,gh)NN6<</NN6<<.KK112 *rP   c                     t        d      )Nz6`resize_token_embeddings` is not supported for I-BERT.)NotImplementedError)rL   new_num_tokenss     rO   resize_token_embeddingsz,IBertPreTrainedModel.resize_token_embeddingsj  s    !"Z[[rP   r   )rl   rm   rn   r   __annotations__base_model_prefixr>   no_gradr  r*  rp   rq   s   @rO   r  r  D  s0    U]]_3 3@\rP   r  c                   ,    e Zd ZdZd fd	Zd Zd Ze	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
ej                  dz  dedz  dedz  dedz  deeej                     z  fd       Z xZS )
IBertModela  

    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in [Attention is
    all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

    c                     t         |   |       || _        |j                  | _        t	        |      | _        t        |      | _        |rt        |      nd| _	        | j                          y)zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        N)r0   r1   rM   r&   r   re   r   encoderr  pooler	post_init)rL   rM   add_pooling_layerrN   s      rO   r1   zIBertModel.__init__y  sZ    
 	  ++)&1#F+->k&)D 	rP   c                 .    | j                   j                  S r   re   r:   rL   s    rO   get_input_embeddingszIBertModel.get_input_embeddings  s    ...rP   c                 &    || j                   _        y r   r6  )rL   r   s     rO   set_input_embeddingszIBertModel.set_input_embeddings  s    */'rP   Nr^   r   r_   r'   r`   r   r   r   returnc	                 V   ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }||t	        d      |#| j                  ||       |j                         }
n!||j                         d d }
nt	        d      |
\  }}||j                  n|j                  }|t        j                  ||f|      }|&t        j                  |
t        j                  |      }| j                  ||||      \  }}t        | j                   ||      }| j                  ||||||      }|d	   }| j                  | j                  |      nd }|s
||f|d
d  z   S t!        |||j"                  |j$                  |j&                        S )NzDYou cannot specify both input_ids and inputs_embeds at the same timer)   z5You have to specify either input_ids or inputs_embeds)rT   rR   )r^   r'   r_   r`   )rM   r`   r   )r   r   r   r   r   r   )r   pooler_outputr   r   r   )rM   r   r   r   r~   %warn_if_padding_and_no_attention_maskr[   rT   r>   onesr\   r]   re   r
   r1  r2  r   r   r   r   )rL   r^   r   r_   r'   r`   r   r   r   kwargsrb   
batch_size
seq_lengthrT   embedding_outputembedding_output_scaling_factorencoder_outputssequence_outputr  s                      rO   rh   zIBertModel.forward  s    2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++BYBY ]%>cdd"66y.Q#..*K&',,.s3KTUU!,
J%.%:!!@T@T!"ZZ*j)A6RN!"[[EJJvVN<@OO%)'	 =L =
99 3;;*)
 ,,+)/!5# ' 
 *!,8<8OO4UY#]3oab6III;-')77&11,==
 	
rP   )T)NNNNNNNN)rl   rm   rn   ro   r1   r8  r:  r   r>   
LongTensorFloatTensorboolr   r   rh   rp   rq   s   @rO   r/  r/  n  s    "/0  .237260426)-,0#'E
##d*E
 ))D0E
 ((4/	E

 &&-E
 ((4/E
  $;E
 #TkE
 D[E
 
6e>O>O8P	PE
 E
rP   r/  c                   P    e Zd ZdddZ fdZd Zd Ze	 	 	 	 	 	 	 	 	 ddej                  dz  d	ej                  dz  d
ej                  dz  dej                  dz  dej                  dz  dej                  dz  dedz  dedz  dedz  deeej                     z  fd       Z xZS )IBertForMaskedLMz(ibert.embeddings.word_embeddings.weight$zlm_head.bias)zlm_head.decoder.weightzlm_head.decoder.biasc                     t         |   |       t        |d      | _        t	        |      | _        | j                          y NF)r4  )r0   r1   r/  r  r   lm_headr3  rK   s     rO   r1   zIBertForMaskedLM.__init__  s6     %@
"6* 	rP   c                 .    | j                   j                  S r   )rN  decoderr7  s    rO   get_output_embeddingsz&IBertForMaskedLM.get_output_embeddings  s    ||###rP   c                 \    || j                   _        |j                  | j                   _        y r   )rN  rP  rx   )rL   new_embeddingss     rO   set_output_embeddingsz&IBertForMaskedLM.set_output_embeddings  s     -*//rP   Nr^   r   r_   r'   r`   labelsr   r   r   r;  c
           
         |	|	n| j                   j                  }	| j                  ||||||||	      }|d   }| j                  |      }d}|Ft	               } ||j                  d| j                   j                        |j                  d            }|	s|f|dd z   }||f|z   S |S t        |||j                  |j                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        Nr   r_   r'   r`   r   r   r   r   r)   r   losslogitsr   r   )
rM   r   r  rN  r   r   r7   r   r   r   )rL   r^   r   r_   r'   r`   rU  r   r   r   r@  r   rF  prediction_scoresmasked_lm_lossloss_fctr   s                    rO   rh   zIBertForMaskedLM.forward  s    ( &1%<k$++BYBY**))%'/!5#  	
 "!* LL9')H%&7&<&<RAWAW&XZ`ZeZefhZijN')GABK7F3A3M^%.YSYY$!//))	
 	
rP   	NNNNNNNNN)rl   rm   rn   _tied_weights_keysr1   rQ  rT  r   r>   rG  rH  rI  r   r   rh   rp   rq   s   @rO   rK  rK    s    #M .
$0  .237260426*.)-,0#'0
##d*0
 ))D00
 ((4/	0

 &&-0
 ((4/0
   4'0
  $;0
 #Tk0
 D[0
 
% 1 12	20
 0
rP   rK  c                   (     e Zd ZdZ fdZd Z xZS )r   z)I-BERT Head for masked language modeling.c                    t         |           t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _        t        j                  |j                  |j                        | _
        t        j                  t        j                  |j                              | _        y )N)r-   )r0   r1   r   r  r8   r   rF   rE   
layer_normr7   rP  	Parameterr>   r\   rx   rK   s     rO   r1   zIBertLMHead.__init__'  s    YYv1163E3EF
,,v'9'9v?T?TUyy!3!3V5F5FGLLV->->!?@	rP   c                     | j                  |      }t        |      }| j                  |      }| j                  |      }|S r   )r   r	   rb  rP  )rL   featuresr@  xs       rO   rh   zIBertLMHead.forward/  s;    JJx GOOA LLOrP   rl   rm   rn   ro   r1   rh   rp   rq   s   @rO   r   r   $  s    3ArP   r   z
    I-BERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    )custom_introc                   :    e Zd Z fdZe	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dedz  de	e
ej                     z  fd       Z xZS )IBertForSequenceClassificationc                     t         |   |       |j                  | _        t        |d      | _        t        |      | _        | j                          y rM  )r0   r1   
num_labelsr/  r  IBertClassificationHead
classifierr3  rK   s     rO   r1   z'IBertForSequenceClassification.__init__A  sC      ++%@
1&9 	rP   Nr^   r   r_   r'   r`   rU  r   r   r   r;  c
           
         |	|	n| j                   j                  }	| j                  ||||||||	      }|d   }| j                  |      }d}|| j                   j                  | j
                  dk(  rd| j                   _        nl| j
                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j                  dk(  rIt               }| j
                  dk(  r& ||j                         |j                               }n |||      }n| j                   j                  dk(  r=t               } ||j                  d| j
                        |j                  d            }n,| j                   j                  dk(  rt               } |||      }|	s|f|d	d z   }||f|z   S |S t        |||j                   |j"                  
      S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NrW  r   r   
regressionsingle_label_classificationmulti_label_classificationr)   r   rX  )rM   r   r  rn  problem_typerl  rS   r>   r]   r   r   squeezer   r   r   r   r   r   rL   r^   r   r_   r'   r`   rU  r   r   r   r@  r   rF  rZ  rY  r]  r   s                    rO   rh   z&IBertForSequenceClassification.forwardK  s   ( &1%<k$++BYBY**))%'/!5#  	
 "!*1{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#FF3D))-JJ+-B @&++b/R))-II,./Y,F)-)9TGf$EvE'!//))	
 	
rP   r^  )rl   rm   rn   r1   r   r>   rG  rH  rI  r   r   rh   rp   rq   s   @rO   rj  rj  :  s     .237260426*.)-,0#'A
##d*A
 ))D0A
 ((4/	A

 &&-A
 ((4/A
   4'A
  $;A
 #TkA
 D[A
 
"E%*;*;$<	<A
 A
rP   rj  c                   :    e Zd Z fdZe	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dedz  de	e
ej                     z  fd       Z xZS )IBertForMultipleChoicec                     t         |   |       t        |      | _        t	        j
                  |j                        | _        t	        j                  |j                  d      | _
        | j                          y )Nr   )r0   r1   r/  r  r   rH   rI   rJ   r  r8   rn  r3  rK   s     rO   r1   zIBertForMultipleChoice.__init__  sV     '
zz&"<"<=))F$6$6: 	rP   Nr^   r_   r   rU  r'   r`   r   r   r   r;  c
           
      J   |	|	n| j                   j                  }	||j                  d   n|j                  d   }|!|j                  d|j	                  d            nd}|!|j                  d|j	                  d            nd}|!|j                  d|j	                  d            nd}|!|j                  d|j	                  d            nd}|1|j                  d|j	                  d      |j	                  d            nd}| j                  ||||||||	      }|d   }| j                  |      }| j                  |      }|j                  d|      }d}|t               } |||      }|	s|f|dd z   }||f|z   S |S t        |||j                  |j                        S )a[  
        input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        Nr   r)   r   )r'   r_   r   r`   r   r   r   r   rX  )rM   r   r   r   r[   r  rJ   rn  r   r   r   r   )rL   r^   r_   r   rU  r'   r`   r   r   r   r@  num_choicesflat_input_idsflat_position_idsflat_token_type_idsflat_attention_maskflat_inputs_embedsr   r  rZ  reshaped_logitsrY  r]  r   s                           rO   rh   zIBertForMultipleChoice.forward  s   X &1%<k$++BYBY,5,Aiooa(}GZGZ[\G]CLCXINN2,>?^bLXLdL--b,2C2CB2GHjnR`Rln11"n6I6I"6MNrvR`Rln11"n6I6I"6MNrv ( r=#5#5b#9=;M;Mb;QR 	 ***..,/!5#  	
  
]3/ ++b+6')HOV4D%''!"+5F)-)9TGf$EvE("!//))	
 	
rP   r^  )rl   rm   rn   r1   r   r>   rG  rH  rI  r   r   rh   rp   rq   s   @rO   rw  rw    s     .22637*.0426)-,0#'V
##d*V
 ((4/V
 ))D0	V

   4'V
 &&-V
 ((4/V
  $;V
 #TkV
 D[V
 
#U5+<+<%=	=V
 V
rP   rw  c                   :    e Zd Z fdZe	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	edz  d
edz  dedz  de	e
ej                     z  fd       Z xZS )IBertForTokenClassificationc                 0   t         |   |       |j                  | _        t        |d      | _        t        j                  |j                        | _        t        j                  |j                  |j                        | _        | j                          y rM  )r0   r1   rl  r/  r  r   rH   rI   rJ   r  r8   rn  r3  rK   s     rO   r1   z$IBertForTokenClassification.__init__  sk      ++%@
zz&"<"<=))F$6$68I8IJ 	rP   Nr^   r   r_   r'   r`   rU  r   r   r   r;  c
           
         |	|	n| j                   j                  }	| j                  ||||||||	      }|d   }| j                  |      }| j	                  |      }d}|<t               } ||j                  d| j                        |j                  d            }|	s|f|dd z   }||f|z   S |S t        |||j                  |j                        S )z
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        NrW  r   r)   r   rX  )rM   r   r  rJ   rn  r   r   rl  r   r   r   ru  s                    rO   rh   z#IBertForTokenClassification.forward  s    $ &1%<k$++BYBY**))%'/!5#  	
 "!*,,71')HFKKDOO<fkk"oNDY,F)-)9TGf$EvE$!//))	
 	
rP   r^  )rl   rm   rn   r1   r   r>   rG  rH  rI  r   r   rh   rp   rq   s   @rO   r  r    s    	  .237260426*.)-,0#'1
##d*1
 ))D01
 ((4/	1

 &&-1
 ((4/1
   4'1
  $;1
 #Tk1
 D[1
 
u'8'8!9	91
 1
rP   r  c                   (     e Zd ZdZ fdZd Z xZS )rm  z-Head for sentence-level classification tasks.c                 &   t         |           t        j                  |j                  |j                        | _        t        j                  |j                        | _        t        j                  |j                  |j                        | _
        y r   )r0   r1   r   r  r8   r   rH   rI   rJ   rl  out_projrK   s     rO   r1   z IBertClassificationHead.__init__;  s`    YYv1163E3EF
zz&"<"<=		&"4"4f6G6GHrP   c                     |d d dd d f   }| j                  |      }| j                  |      }t        j                  |      }| j                  |      }| j	                  |      }|S r	  )rJ   r   r>   tanhr  )rL   re  r@  r   s       rO   rh   zIBertClassificationHead.forwardA  s^     Aq)]3

=1

=1]3m4rP   rg  rq   s   @rO   rm  rm  8  s    7IrP   rm  c                   Z    e Zd Z fdZe	 	 	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  dej                  dz  d	ej                  dz  d
edz  dedz  dedz  de	e
ej                     z  fd       Z xZS )IBertForQuestionAnsweringc                     t         |   |       |j                  | _        t        |d      | _        t        j                  |j                  |j                        | _        | j                          y rM  )
r0   r1   rl  r/  r  r   r  r8   
qa_outputsr3  rK   s     rO   r1   z"IBertForQuestionAnswering.__init__M  sU      ++%@
))F$6$68I8IJ 	rP   Nr^   r   r_   r'   r`   start_positionsend_positionsr   r   r   r;  c           
      &   |
|
n| j                   j                  }
| j                  |||||||	|
      }|d   }| j                  |      }|j	                  dd      \  }}|j                  d      j                         }|j                  d      j                         }d }||t        |j                               dkD  r|j                  d      }t        |j                               dkD  r|j                  d      }|j                  d      }|j                  d|      }|j                  d|      }t        |      } |||      } |||      }||z   dz  }|
s||f|dd  z   }||f|z   S |S t        ||||j                  |j                        S )	NrW  r   r   r)   dim)ignore_indexr   )rY  start_logits
end_logitsr   r   )rM   r   r  r  splitrt  r   lenr[   clampr   r   r   r   )rL   r^   r   r_   r'   r`   r  r  r   r   r   r@  r   rF  rZ  r  r  
total_lossignored_indexr]  
start_lossend_lossr   s                          rO   rh   z!IBertForQuestionAnswering.forwardW  s    &1%<k$++BYBY**))%'/!5#  	
 "!*1#)<<r<#: j#++B/::<''+668

&=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9(--a0M-33A}EO)//=AM']CH!,@J
M:H$x/14J"J/'!"+=F/9/EZMF*Q6Q+%!!//))
 	
rP   )
NNNNNNNNNN)rl   rm   rn   r1   r   r>   rG  rH  rI  r   r   rh   rp   rq   s   @rO   r  r  K  s     .2372604263715)-,0#'=
##d*=
 ))D0=
 ((4/	=

 &&-=
 ((4/=
 ))D0=
 ''$.=
  $;=
 #Tk=
 D[=
 
&e.?.?(@	@=
 =
rP   r  c                     | j                  |      j                         }t        j                  |d      j	                  |      |z   |z  }|j                         |z   S )aM  
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's *utils.make_positions*.

    Args:
    input_ids (`torch.LongTensor`):
           Indices of input sequence tokens in the vocabulary.

    Returns: torch.Tensor
    r   r  )ner   r>   cumsumtype_asr]   )r^   r$   ra   maskincremental_indicess        rO   rX   rX     sW     <<$((*D <<!4<<TBE[[_cc##%33rP   )rK  rw  r  rj  r  r/  r  )r   )=ro   r   r>   r   torch.nnr   r   r    r   r  activationsr	   masking_utilsr
   modeling_outputsr   r   r   r   r   r   r   modeling_utilsr   utilsr   r   configuration_ibertr   quant_modulesr   r   r   r   r   r   
get_loggerrl   loggerModuler   rs   r   r   r   r   r   r   r  r  r/  rK  r   rj  rw  r  rm  r  rX   __all__r   rP   rO   <module>r     s  "     A A &  6   . , , c c 
		H	%s=bii s=l@. @.F,;bii ,;^/RYY /8;		 ;D,;")) ,;^59 59p6
299 6
r"))   &\? &\ &\R g
% g
 g
T G
+ G
 G
T")) , M
%9 M
M
` b
1 b
 b
J >
"6 >
 >
Bbii & I
 4 I
 I
X4"rP   