^jddlmZddlZddlmZddlmZmZmZddlm Z ddl m Z m Z ddlmZmZmZddlmZdd lmZmZdd lmZdd lmZmZmZmZmZmZm Z m!Z!dd l"m#Z#m$Z$dd l%m&Z&ddl'm(Z(ddl)m*Z*m+Z+m,Z,ddl-m.Z.m/Z/ddl0m1Z1ddl2m3Z3e,jhe5Z6GddejnZ8 dJdejndejrdejrdejrdejrdzde:dzde:de&e*fdZ;Gdd ejnZ<Gd!d"ejnZ=Gd#d$ejnZ>Gd%d&ejnZ?Gd'd(ejnZ@Gd)d*ejnZAGd+d,eZBe+Gd-d.e$ZCGd/d0ejnZDGd1d2ejnZEe+d34Gd5d6eCZFe+d74Gd8d9eCeZGe+Gd:d;eCZHGd<d=ejnZIe+d>4Gd?d@eCZJe+GdAdBeCZKe+GdCdDeCZLGdEdFejnZMe+GdGdHeCZNgdIZOy)K)CallableN)BCEWithLogitsLossCrossEntropyLossMSELoss)initialization)ACT2FNgelu)Cache DynamicCacheEncoderDecoderCache)GenerationMixin)create_bidirectional_maskcreate_causal_mask)GradientCheckpointingLayer))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentions!CausalLMOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)apply_chunking_to_forward)TransformersKwargsauto_docstringlogging)can_return_tuplemerge_with_config_defaults)capture_outputs) RobertaConfigc eZdZdZfdZ d dej dzdej dzdej dzdejdzded ejf d Z e d Z e dd Z xZS)RobertaEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.cTt|tj|j|j |j |_tj|j|j |_ tj|j |j|_ tj|j|_|jdt!j"|j$j'dd|jdt!j(|j*j-t j.d|j |_tj|j$|j |j0|_y) N) padding_idxeps position_idsr$F) persistenttoken_type_ids)dtype)super__init__nn Embedding vocab_size hidden_size pad_token_idword_embeddingstype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_buffertorcharangemax_position_embeddingsexpandzerosr,sizelongr)position_embeddingsselfconfig __class__s w/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/roberta/modeling_roberta.pyr3zRobertaEmbeddings.__init__;s4 !||F,=,=v?Q?Q_e_r_rs%'\\&2H2H&J\J\%]"f&8&8f>S>STzz&"<"<=   ELL)G)GHOOPWXej    ekk$*;*;*@*@*B%**Ubg  "..#%<<  * *F,>,>DL\L\$  N input_idsr0r, inputs_embedspast_key_values_lengthreturnc|<||j||j|}n|j||j}||j}n|jdd}|\}}|t |drm|j j |jj|jdd} tj| d|} | j||}n:tj|tj|jj}||j|}|j!|} || z} |j#|} | | z} |j%| } |j'| } | S)Nr.r0rr$)dimindexr1device)"create_position_ids_from_input_idsr)&create_position_ids_from_inputs_embedsrGhasattrr0torXrEshaperBgatherrFrHr,r9r;rIr<r@) rKrPr0r,rQrR input_shape batch_size seq_lengthbuffered_token_type_idsr; embeddingsrIs rNforwardzRobertaEmbeddings.forwardOs  $#FFt//1G   $JJ=Z^ZjZjk  #..*K',,.s3K!, J  !t-.*.*=*=*@*@ATAT*U*\*\]i]o]opq]rtv*w'*/,,7NTU]i*j'!8!?!? J!W!&[ SWSdSdSkSk!l   00;M $ : :> J"%:: "66|D"55 ^^J/ \\*- rOc|jdd}|d}tj|dz||zdztj|j}|j dj |S)z We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids. Args: inputs_embeds: torch.Tensor Returns: torch.Tensor Nr.r$rWr)rGrBrCrHrX unsqueezerE)rQr)r_sequence_lengthr,s rNrZz8RobertaEmbeddings.create_position_ids_from_inputs_embedssp$((*3B/ %a.|| !O_{:Q>ejjYfYmYm %%a(// <L==((6??([L,,|U3K''1-88:K  $$rOc eZdZd fd Z d dej dej dzdedzdee de ej f dZ xZ S) RobertaSelfAttentionNc@t||j|jzdk7r2t |ds&t d|jd|jd||_|j|_t|j|jz |_|j|jz|_ |jdz|_ tj|j|j|_ tj|j|j|_tj|j|j|_tj |j"|_|j&|_||_||_yNrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()r)r2r3r7num_attention_headsr[ ValueErrorrLrkattention_head_size all_head_sizerr4Linearr{r|r}r>attention_probs_dropout_probr@ is_decoder is_causal layer_idxrKrLrrrMs rNr3zRobertaSelfAttention.__init__sP     : : :a ?PVXhHi#F$6$6#78 445Q8  #)#=#= #&v'9'9F|} t|tr |j} | j|| |j\}} tj|jjt } | |||| |f|j"sdn|j$j&|j(d|\} } | j*g|dj-} | | fS)Nr.r$rr@r)r]rr{viewrr|r} isinstancer self_attention_cacheupdaterr get_interfacerL_attn_implementationrrr@rrreshaper)rKrr~rrr_ hidden_shape query_layer key_layer value_layercurrent_past_key_valuesattention_interfacerrs rNrdzRobertaSelfAttention.forwards$))#2. CCbC$*B*BC 5djj/44lCMMaQRS 0DHH]+00,?II!QO 4djj/44lCMMaQRS  &&5 #/+>?*9*N*N'&=%C%CI{\`\j\j%k "I{(?(M(M KK , ,.E) %8      % $}}C$,,..LL %  % ! \*k));;;;FFH L((rOFN)NN rprqrrr3rBrvrur rrtuplerdrxrys@rNrrsg#648(, ')||')))D0') ') +, ') u||  ')rOrceZdZd fd Z d dej dej dzdej dzdedzdee de ej f d Z xZ S) RobertaCrossAttentionNct||j|jzdk7r2t |ds&t d|jd|jd||_|j|_t|j|jz |_|j|jz|_ |jdz|_ tj|j|j|_ tj|j|j|_tj|j|j|_tj |j"|_||_||_yr)r2r3r7rr[rrLrkrrrr4rr{r|r}r>rr@rrrs rNr3zRobertaCrossAttention.__init__sC     : : :a ?PVXhHi#F$6$6#78 445Q8  #)#=#= #&v'9'9FB**4>>:(?(M(M KK , ,.E) %8      % $}}C$,,..LL %  % ! \*k));;;;FFH L((rOr)NNN) rprqrrr3rBrvrur rrrrdrxrys@rNrrs#4;?376: 1)||1) %00471)))D0 1) -t3 1) +, 1) u|| 1)rOrcneZdZfdZdej dej dej fdZxZS)RobertaSelfOutputc(t|tj|j|j|_tj |j|j|_tj|j|_ yNr*) r2r3r4rr7denser<r=r>r?r@rJs rNr3zRobertaSelfOutput.__init__Ns` YYv1163E3EF f&8&8f>S>STzz&"<"<= rOr input_tensorrScr|j|}|j|}|j||z}|SNrr@r<rKrrs rNrdzRobertaSelfOutput.forwardT7 =1  ]3 }|'CD rOrprqrrr3rBrvrdrxrys@rNrrM1> U\\RWR^R^rOrceZdZd fd Z d dej dej dzdej dzdej dzdedzdee d e ej fd Z xZ S) RobertaAttentionNct|||_|rtnt}|||||_t ||_y)Nrr)r2r3is_cross_attentionrrrKroutput)rKrLrrrattention_classrMs rNr3zRobertaAttention.__init__\s= "43E/K_#Fi9U '/ rOrr~rencoder_attention_maskrrrSc |js|n|}|j|f|||d|\}}|j||}||fS)N)rr~r)rrKr) rKrr~rrrrattention_outputrs rNrdzRobertaAttention.forwardcsd04/F/FLb)2 * "7)+ *  * &, ;;'7G--rO)FNFNNNNrrys@rNrr[s048:>;?(, .||.))D0. %0047 . !& 1 1D 8 .  .+,. u|| .rOrcVeZdZfdZdej dej fdZxZS)RobertaIntermediatect|tj|j|j |_t|jtrt|j|_ y|j|_ yr) r2r3r4rr7intermediate_sizerr hidden_actstrr intermediate_act_fnrJs rNr3zRobertaIntermediate.__init__ys] YYv1163K3KL f'' -'-f.?.?'@D $'-'8'8D $rOrrScJ|j|}|j|}|Sr)rr)rKrs rNrdzRobertaIntermediate.forwards& =1 00? rOrrys@rNrrxs#9U\\ellrOrcneZdZfdZdej dej dej fdZxZS) RobertaOutputc(t|tj|j|j |_tj|j |j|_tj|j|_ yr) r2r3r4rrr7rr<r=r>r?r@rJs rNr3zRobertaOutput.__init__s` YYv779K9KL f&8&8f>S>STzz&"<"<= rOrrrScr|j|}|j|}|j||z}|Srrrs rNrdzRobertaOutput.forwardrrOrrys@rNrrrrOrceZdZd fd Z d dej dej dzdej dzdej dzdedzdee d ej fd Z d Z xZ S) RobertaLayerNct||j|_d|_t ||j ||_|j |_|j|_|jr.|j st|dt |d|d|_ t||_ t||_ y)Nr$rz> should be used as a decoder model if cross attention is addedFT)rrr)r2r3chunk_size_feed_forward seq_len_dimrr attentionadd_cross_attentionrcrossattentionr intermediaterr)rKrLrrMs rNr3zRobertaLayer.__init__s '-'E'E$)&F>  $ ,$  $  q 1 ??4@4!12 =dVDDD )<(;(;%%& ) !0 )  ) % "A 6 0  # #T%A%A4CSCSUe rOcL|j|}|j||}|Sr)rr)rKrintermediate_outputrs rNrzRobertaLayer.feed_forward_chunks,"//0@A{{#68HI rOrr) rprqrrr3rBrvrur rrrdrrxrys@rNrrs,,48:>;?(, %||%))D0% %0047 % !& 1 1D 8 %  %+,% %NrOrcneZdZeZdZdZdZdZdZ dZ e e e dZej fdZxZS)RobertaPreTrainedModelrobertaT)r attentionscross_attentionsct||t|tr t j |j yt|tryt j|jtj|jjdjdt j |jyy)zInitialize the weightsr.r-N)r2 _init_weightsr RobertaLMHeadinitzeros_biasr'copy_r,rBrCr]rEr0)rKrzrMs rNrz$RobertaPreTrainedModel._init_weightss f% fm , KK $  1 2 JJv**ELL9L9L9R9RSU9V,W,^,^_f,g h KK-- .3rO)rprqrrr% config_classbase_model_prefixsupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendrrr_can_record_outputsrBno_gradrrxrys@rNrrsX L!&*#N"&%*1 U]]_//rOrceZdZfdZ d dej dej dzdej dzdej dzdedzdedzd e e d e ej e zfd Z xZS) RobertaEncoderc t|||_tjt |j Dcgc]}t||c}|_ycc}w)N)r) r2r3rLr4 ModuleListrangenum_hidden_layersrlayer)rKrLirMs rNr3zRobertaEncoder.__init__sF  ]]uU[UmUmOn#o!L1$E#op #osANrr~rrr use_cacherrSc t|jD]\}} | |||f||d|}t||r|SdS)N)rr)last_hidden_stater) enumeraterr) rKrr~rrrrrr layer_modules rNrdzRobertaEncoder.forwardsn )4 OA|(%(> /   M 9+/8O  >B  rO)NNNNN)rprqrrr3rBrvrur boolrrrrrdrxrys@rNr r sq48:>;?(,!% || ))D0  %0047  !& 1 1D 8    $; +,  u|| H H rOr cVeZdZfdZdej dej fdZxZS) RobertaPoolerct|tj|j|j|_tj |_yr)r2r3r4rr7rTanh activationrJs rNr3zRobertaPooler.__init__s9 YYv1163E3EF '')rOrrSc\|dddf}|j|}|j|}|SNr)rr)rKrfirst_token_tensor pooled_outputs rNrdzRobertaPooler.forwards6+1a40 #56  6 rOrrys@rNrrs#$ U\\ellrOra  The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. ) custom_introceZdZddgZdfd ZdZdZeee dde jdzde jdzd e jdzd e jdzd e jdzd e jdzd e jdzde dzde dzdeedee jezfdZdZxZS) RobertaModelr'rct||||_d|_t ||_t ||_|r t|nd|_ |jy)zv add_pooling_layer (bool, *optional*, defaults to `True`): Whether to add a pooling layer FN) r2r3rLgradient_checkpointingr'rcr encoderrpooler post_init)rKrLadd_pooling_layerrMs rNr3zRobertaModel.__init__.sU   &+#+F3%f- /@mF+d  rOc.|jjSrrcr9rKs rNget_input_embeddingsz!RobertaModel.get_input_embeddings?s...rOc&||j_yrr*)rKr}s rNset_input_embeddingsz!RobertaModel.set_input_embeddingsBs*/'rONrPr~r0r,rQrrrrrrSc |du|duz r td|jjr| | n|jj} nd} | rd|b||jjr4t t |jt |jnt |j}||jnd} |j||||| } |j||| ||\}}|j| f||||| |d| } | j}|j|j|nd}t||| jS) Nz:You must specify exactly one of input_ids or inputs_embedsF)rLr)rPr,r0rQrR)r~rembedding_outputrr)r~rrrrr,)r pooler_outputr)rrLrris_encoder_decoderr r get_seq_lengthrc_create_attention_masksr%rr&rr)rKrPr~r0r,rQrrrrrrRr0encoder_outputssequence_outputrs rNrdzRobertaModel.forwardEs|  -t"; <YZ Z ;; ! !%.%:  @U@UII 0)4 8V8V$L $DlZ^ZeZeFfg!5  FUE`!?!?!Afg??%)'#9 + 261M1M)#9-"7+ 2N2 ..'$,,   )"7#9+%    *;;8< 8O O4UY ;-'+;;  rOc|jjrt|j|||}nt|j||}|t|j|||}||fS)N)rLrQr~r)rLrQr~)rLrQr~r)rLrrr)rKr~rr0rrs rNr4z$RobertaModel._create_attention_maskssu ;; ! !/{{.- / N7{{.-N " -%>{{.5&; & "555rO)T) NNNNNNNNN)rprqrr_no_split_modulesr3r,r.r"r#rrBrvr rrrrrrdr4rxrys@rNr"r"s.-n="/0 *..2.2,0-1596:(,!%? <<$&?  t+?  t+ ? llT) ? ||d* ?  %||d2? !& t 3? ? $;? +,?  u|| K K?  ? B6rOr"zS RoBERTa Model with a `language modeling` head on top for CLM fine-tuning. ceZdZdddZfdZdZdZee dde jdzd e jdzd e jdzd e jdzd e jdzd e jdzde jdzde jdzde e e jdzde dzdee jzdeede e jezfdZxZS)RobertaForCausalLM)roberta.embeddings.word_embeddings.weight lm_head.biaszlm_head.decoder.weightzlm_head.decoder.biasct|||jstj dt |d|_t||_|jy)NzOIf you want to use `RobertaLMHeadModel` as a standalone, add `is_decoder=True.`Fr( r2r3rloggerwarningr"rrlm_headr'rJs rNr3zRobertaForCausalLM.__init__sL    NNl m#FeD $V,  rOc.|jjSrrCdecoderr+s rNget_output_embeddingsz(RobertaForCausalLM.get_output_embeddings||###rOc&||j_yrrErKnew_embeddingss rNset_output_embeddingsz(RobertaForCausalLM.set_output_embeddings- rONrPr~r0r,rQrrlabelsrrlogits_to_keeprrSc |d} |j|f||||||| | dd | } | j}t| tr t | dn| }|j |dd|ddf}d}|*|j d|||jjd| }t||| j| j| j| jS)am token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value >= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` Example: ```python >>> from transformers import AutoTokenizer, RobertaForCausalLM, AutoConfig >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("FacebookAI/roberta-base") >>> config = AutoConfig.from_pretrained("FacebookAI/roberta-base") >>> config.is_decoder = True >>> model = RobertaForCausalLM.from_pretrained("FacebookAI/roberta-base", config=config) >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> prediction_logits = outputs.logits ```NFT) r~r0r,rQrrrr return_dict)logitsrNr6)lossrRrrrr)rrrrkslicerC loss_functionrLr6rrrrr)rKrPr~r0r,rQrrrNrrrOroutputsr slice_indicesrRrSs rNrdzRobertaForCausalLM.forwards`  I@L  A ))%'"7#9+ A  A  11 8B>SV8W~ot4]k mA}a,?@A  %4%%pVFt{{OeOepiopD0#33!//))$55   rO) NNNNNNNNNNr)rprqrr_tied_weights_keysr3rGrLr!rrBrtrurrrkrvrrrrdrxrys@rNr:r:s#N . $..237260426:>;?*.BF!%-.O ##d*O ))D0O ((4/ O &&- O ((4/ O  %0047O !& 1 1D 8O   4'O uU%6%6784?O $;O ell*O +,O  u|| @ @O O rOr:ceZdZdddZfdZdZdZee dde jdzd e jdzd e jdzd e jdzd e jdzd e jdzde jdzde jdzde e dee jezfdZxZS)RobertaForMaskedLMr;r<r=ct|||jrtj dt |d|_t||_|jy)NznIf you want to use `RobertaForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.Fr?r@rJs rNr3zRobertaForMaskedLM.__init__"sR     NN1  $FeD $V,  rOc.|jjSrrEr+s rNrGz(RobertaForMaskedLM.get_output_embeddings1rHrOc&||j_yrrErJs rNrLz(RobertaForMaskedLM.set_output_embeddings4rMrONrPr~r0r,rQrrrNrrSc t|j|f||||||dd| } | d} |j| } d} |a|j| j}t }|| j d|j j|j d} t| | | j| jS)a token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value >= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` T)r~r0r,rQrrrQrNr.rSrRrr) rrCr\rXrrrLr6rrr)rKrPr~r0r,rQrrrNrrWr6prediction_scoresmasked_lm_lossloss_fcts rNrdzRobertaForMaskedLM.forward7s:$,,   ))%'"7#9    "!* LL9  YY0778F')H%&7&<&;?*.5 ##d*5 ))D05 ((4/ 5 &&- 5 ((4/ 5  %00475 !& 1 1D 85   4'5 +,5  u|| ~ -5 5 rOr[c(eZdZdZfdZdZxZS)rz*Roberta Head for masked language modeling.ct|tj|j|j|_tj |j|j|_tj|j|j|_ tjtj|j|_yr)r2r3r4rr7rr<r= layer_normr6rF ParameterrBrFrrJs rNr3zRobertaLMHead.__init__ts YYv1163E3EF ,,v'9'9v?T?TUyy!3!3V5F5FG LLV->->!?@ rOc |j|}t|}|j|}|j|}|Sr)rr rfrFrKfeaturesrxs rNrdzRobertaLMHead.forward|s; JJx  G OOA  LLOrOrprqrrrsr3rdrxrys@rNrrqs4ArOrz RoBERTa Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. c*eZdZfdZee d dejdzdejdzdejdzdejdzdejdzdejdzd e e d e eje zfd ZxZS) RobertaForSequenceClassificationct|||j|_||_t |d|_t ||_|jyNFr?) r2r3 num_labelsrLr"rRobertaClassificationHead classifierr'rJs rNr3z)RobertaForSequenceClassification.__init__sJ   ++ #FeD 3F; rONrPr~r0r,rQrNrrSc |j|f||||dd|}|d} |j| } d} ||j| j}|jj |j dk(rd|j_nl|j dkDrL|jtjk(s|jtjk(rd|j_nd|j_|jj dk(rIt} |j dk(r&| | j|j} n| | |} n|jj dk(r=t} | | jd |j |jd } n,|jj dk(rt} | | |} t!| | |j"|j$ S) a token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value >= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Tr~r0r,rQrQrNr$ regressionsingle_label_classificationmulti_label_classificationr.r`)rrsr\rXrL problem_typerqr1rBrHrkrsqueezerrrrrr rKrPr~r0r,rQrNrrWr6rRrSrcs rNrdz(RobertaForSequenceClassification.forwards6$,,  ))%'   "!*1  YYv}}-F{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#FF3D))-JJ+- B @&++b/R))-II,./'!//))   rONNNNNN)rprqrrr3r!rrBrtrurrrrvrrdrxrys@rNrnrns .237260426*.C ##d*C ))D0C ((4/ C &&- C ((4/ C   4'C +,C  u|| 7 7C C rOrnc*eZdZfdZee d dejdzdejdzdejdzdejdzdejdzdejdzd e e d e eje zfd ZxZS) RobertaForMultipleChoicect||t||_t j |j |_t j|jd|_ |jy)Nr$) r2r3r"rr4r>r?r@rr7rsr'rJs rNr3z!RobertaForMultipleChoice.__init__sV  #F+ zz&"<"<= ))F$6$6: rONrPr0r~rNr,rQrrSc "||jdn|jd}|!|jd|jdnd} |!|jd|jdnd} |!|jd|jdnd} |!|jd|jdnd} |1|jd|jd|jdnd} |j| f| | | | dd|}|d}|j |}|j |}|jd|}d}|.|j |j}t}|||}t|||j|jS)a input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value >= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. Nr$r.T)r,r0r~rQrQr`) r]rrGrr@rsr\rXrrrr)rKrPr0r~rNr,rQr num_choicesflat_input_idsflat_position_idsflat_token_type_idsflat_attention_maskflat_inputs_embedsrWrrRreshaped_logitsrSrcs rNrdz RobertaForMultipleChoice.forwardsV-6,Aiooa(}GZGZ[\G] CLCXINN2,>?^bLXLdL--b,2C2CB2GHjnR`Rln11"n6I6I"6MNrvR`Rln11"n6I6I"6MNrv(   r=#5#5b#9=;M;Mb;Q R  $,,  *..,      ]3 / ++b+6  YY556F')HOV4D("!//))   rOr|)rprqrrr3r!rrBrtrurrrrvrrdrxrys@rNr~r~s.22637*.0426P ##d*P ((4/P ))D0 P   4' P &&- P ((4/P +,P  u|| 8 8P P rOr~c*eZdZfdZee d dejdzdejdzdejdzdejdzdejdzdejdzd e e d e eje zfd ZxZS) RobertaForTokenClassificationcdt|||j|_t|d|_|j |j n |j }tj||_ tj|j|j|_ |jyrp)r2r3rqr"rclassifier_dropoutr?r4r>r@rr7rsr'rKrLrrMs rNr3z&RobertaForTokenClassification.__init__Ds   ++#FeD )/)B)B)NF % %TZTnTn zz"45 ))F$6$68I8IJ rONrPr~r0r,rQrNrrSc ~|j|f||||dd|}|d} |j| } |j| } d} |W|j| j}t } | | j d|j|j d} t| | |j|jS)a- token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value >= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. TrurNr.r`) rr@rsr\rXrrrqrrrr{s rNrdz%RobertaForTokenClassification.forwardRs2$,,  ))%'   "!*,,71  YYv}}-F')HFKKDOOr@rqout_projrs rNr3z"RobertaClassificationHead.__init__s YYv1163E3EF )/)B)B)NF % %TZTnTn zz"45  &"4"4f6G6GH rOc |dddddf}|j|}|j|}tj|}|j|}|j |}|Sr)r@rrBtanhrris rNrdz!RobertaClassificationHead.forwardsY Q1W  LLO JJqM JJqM LLO MM! rOrlrys@rNrrrrs7IrOrrcJeZdZfdZee d dejdzdejdzdejdzdejdzdejdzdejdzd ejdzd e e d e eje zfd ZxZS)RobertaForQuestionAnsweringct|||j|_t|d|_t j |j|j|_|jyrp) r2r3rqr"rr4rr7 qa_outputsr'rJs rNr3z$RobertaForQuestionAnswering.__init__sU   ++#FeD ))F$6$68I8IJ rONrPr~r0r,rQstart_positions end_positionsrrSc |j|f||||dd|} | d} |j| } | jdd\} } | jdj } | jdj } d}||t |j dkDr|jd}t |j dkDr|jd}| j d}|jd|}|jd|}t|}|| |}|| |}||zd z }t|| | | j| j S) a[ token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value >= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) Trurr$r.riN) ignore_indexr)rS start_logits end_logitsrr) rrsplitrzrlenrGclamprrrr)rKrPr~r0r,rQrrrrWr6rRrr total_loss ignored_indexrc start_lossend_losss rNrdz#RobertaForQuestionAnswering.forwards0$,,  ))%'   "!*1#)<<r<#: j#++B/::< ''+668   &=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9 (--a0M-33A}EO)//=AM']CH!,@J M:H$x/14J+%!!//))   rO)NNNNNNN)rprqrrr3r!rrBrtrurrrrvrrdrxrys@rNrrs.2372604263715> ##d*> ))D0> ((4/ > &&- > ((4/ > ))D0> ''$.> +,>  u|| ; ;> > rOr)r:r[r~rrnrr"r)Nr)Pcollections.abcrrBtorch.nnr4rrrrr activationsr r cache_utilsr r r generationr masking_utilsrrmodeling_layersrmodeling_outputsrrrrrrrrmodeling_utilsrrprocessing_utilsr pytorch_utilsrutilsrrr utils.genericr!r"utils.output_capturingr#configuration_robertar% get_loggerrprAModuler'rvfloatrrrrrrrrrr rr"r:r[rrnr~rrrr__all__rTrOrNrs,% AA&'CC)J9   G&6@@I50   H %g8 g8`!% II% <<% % << % LL4' % T\ %%'(%8@)299@)FI)BIII)X  .ryy.: "))  BII >->B/_//2 RYY @ BII   |6)|6 |6~ i /i  i XR /R R jBII, Q '=Q  Q h] 5] ] @C $:C C L ,K "8K K \ rO