^jddlmZddlZddlmZddlmZmZmZddlm Z ddl m Z m Z ddlmZmZmZddlmZdd lmZmZdd lmZdd lmZmZmZmZmZmZm Z m!Z!dd l"m#Z#m$Z$dd l%m&Z&ddl'm(Z(ddl)m*Z*m+Z+m,Z,ddl-m.Z.m/Z/ddl0m1Z1ddl2m3Z3e,jhe5Z6GddejnZ8 dJdejndejrdejrdejrdejrdzde:dzde:de&e*fdZ;Gdd ejnZ<Gd!d"ejnZ=Gd#d$ejnZ>Gd%d&ejnZ?Gd'd(ejnZ@Gd)d*ejnZAGd+d,eZBGd-d.ejnZCe+Gd/d0e$ZDGd1d2ejnZEGd3d4ejnZFe+d56Gd7d8eDZGe+Gd9d:eDZHGd;dd?eDZJe+Gd@dAeDZKe+GdBdCeDZLe+GdDdEeDZMe+dF6GdGdHeDeZNgdIZOy)K)CallableN)BCEWithLogitsLossCrossEntropyLossMSELoss)initialization)ACT2FNgelu)Cache DynamicCacheEncoderDecoderCache)GenerationMixin)create_bidirectional_maskcreate_causal_mask)GradientCheckpointingLayer))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentions!CausalLMOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)apply_chunking_to_forward)TransformersKwargsauto_docstringlogging)can_return_tuplemerge_with_config_defaults)capture_outputs)CamembertConfigc eZdZdZfdZ d dej dzdej dzdej dzdejdzded ejf d Z e d Z e dd Z xZS)CamembertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.cTt|tj|j|j |j |_tj|j|j |_ tj|j |j|_ tj|j|_|jdt!j"|j$j'dd|jdt!j(|j*j-t j.d|j |_tj|j$|j |j0|_y) N) padding_idxeps position_idsr$F) persistenttoken_type_ids)dtype)super__init__nn Embedding vocab_size hidden_size pad_token_idword_embeddingstype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_buffertorcharangemax_position_embeddingsexpandzerosr,sizelongr)position_embeddingsselfconfig __class__s {/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/camembert/modeling_camembert.pyr3zCamembertEmbeddings.__init__;s4 !||F,=,=v?Q?Q_e_r_rs%'\\&2H2H&J\J\%]"f&8&8f>S>STzz&"<"<=   ELL)G)GHOOPWXej    ekk$*;*;*@*@*B%**Ubg  "..#%<<  * *F,>,>DL\L\$  N input_idsr0r, inputs_embedspast_key_values_lengthreturnc|<||j||j|}n|j||j}||j}n|jdd}|\}}|t |drm|j j |jj|jdd} tj| d|} | j||}n:tj|tj|jj}||j|}|j!|} || z} |j#|} | | z} |j%| } |j'| } | S)Nr.r0rr$)dimindexr1device)"create_position_ids_from_input_idsr)&create_position_ids_from_inputs_embedsrGhasattrr0torXrEshaperBgatherrFrHr,r9r;rIr<r@) rKrPr0r,rQrR input_shape batch_size seq_lengthbuffered_token_type_idsr; embeddingsrIs rNforwardzCamembertEmbeddings.forwardOs  $#FFt//1G   $JJ=Z^ZjZjk  #..*K',,.s3K!, J  !t-.*.*=*=*@*@ATAT*U*\*\]i]o]opq]rtv*w'*/,,7NTU]i*j'!8!?!? J!W!&[ SWSdSdSkSk!l   00;M $ : :> J"%:: "66|D"55 ^^J/ \\*- rOc|jdd}|d}tj|dz||zdztj|j}|j dj |S)z We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids. Args: inputs_embeds: torch.Tensor Returns: torch.Tensor Nr.r$rWr)rGrBrCrHrX unsqueezerE)rQr)r_sequence_lengthr,s rNrZz:CamembertEmbeddings.create_position_ids_from_inputs_embedssp$((*3B/ %a.|| !O_{:Q>ejjYfYmYm %%a(// <L==((6??([L,,|U3K''1-88:K  $$rOc eZdZd fd Z d dej dej dzdedzdee de ej f dZ xZ S) CamembertSelfAttentionNc@t||j|jzdk7r2t |ds&t d|jd|jd||_|j|_t|j|jz |_|j|jz|_ |jdz|_ tj|j|j|_ tj|j|j|_tj|j|j|_tj |j"|_|j&|_||_||_yNrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()r)r2r3r7num_attention_headsr[ ValueErrorrLrkattention_head_size all_head_sizerr4Linearr{r|r}r>attention_probs_dropout_probr@ is_decoder is_causal layer_idxrKrLrrrMs rNr3zCamembertSelfAttention.__init__sP     : : :a ?PVXhHi#F$6$6#78 445Q8  #)#=#= #&v'9'9F|} t|tr |j} | j|| |j\}} tj|jjt } | |||| |f|j"sdn|j$j&|j(d|\} } | j*g|dj-} | | fS)Nr.r$rr@r)r]rr{viewrr|r} isinstancer self_attention_cacheupdaterr get_interfacerL_attn_implementationrrr@rrreshaper)rKrr~rrr_ hidden_shape query_layer key_layer value_layercurrent_past_key_valuesattention_interfacerrs rNrdzCamembertSelfAttention.forwards$))#2. CCbC$*B*BC 5djj/44lCMMaQRS 0DHH]+00,?II!QO 4djj/44lCMMaQRS  &&5 #/+>?*9*N*N'&=%C%CI{\`\j\j%k "I{(?(M(M KK , ,.E) %8      % $}}C$,,..LL %  % ! \*k));;;;FFH L((rOFN)NN rprqrrr3rBrvrur rrtuplerdrxrys@rNrrsg#648(, ')||')))D0') ') +, ') u||  ')rOrceZdZd fd Z d dej dej dzdej dzdedzdee de ej f d Z xZ S) CamembertCrossAttentionNct||j|jzdk7r2t |ds&t d|jd|jd||_|j|_t|j|jz |_|j|jz|_ |jdz|_ tj|j|j|_ tj|j|j|_tj|j|j|_tj |j"|_||_||_yr)r2r3r7rr[rrLrkrrrr4rr{r|r}r>rr@rrrs rNr3z CamembertCrossAttention.__init__sC     : : :a ?PVXhHi#F$6$6#78 445Q8  #)#=#= #&v'9'9FB**4>>:(?(M(M KK , ,.E) %8      % $}}C$,,..LL %  % ! \*k));;;;FFH L((rOr)NNN) rprqrrr3rBrvrur rrrrdrxrys@rNrrs#4;?376: 1)||1) %00471)))D0 1) -t3 1) +, 1) u|| 1)rOrcneZdZfdZdej dej dej fdZxZS)CamembertSelfOutputc(t|tj|j|j|_tj |j|j|_tj|j|_ yNr*) r2r3r4rr7denser<r=r>r?r@rJs rNr3zCamembertSelfOutput.__init__Ns` YYv1163E3EF f&8&8f>S>STzz&"<"<= rOr input_tensorrScr|j|}|j|}|j||z}|SNrr@r<rKrrs rNrdzCamembertSelfOutput.forwardT7 =1  ]3 }|'CD rOrprqrrr3rBrvrdrxrys@rNrrM1> U\\RWR^R^rOrceZdZd fd Z d dej dej dzdej dzdej dzdedzdee d e ej fd Z xZ S) CamembertAttentionNct|||_|rtnt}|||||_t ||_y)Nrr)r2r3is_cross_attentionrrrKroutput)rKrLrrrattention_classrMs rNr3zCamembertAttention.__init__\s= "45G1Mc#Fi9U )&1 rOrr~rencoder_attention_maskrrrSc |js|n|}|j|f|||d|\}}|j||}||fS)N)rr~r)rrKr) rKrr~rrrrattention_outputrs rNrdzCamembertAttention.forwardcsd04/F/FLb)2 * "7)+ *  * &, ;;'7G--rO)FNFNNNNrrys@rNrr[s248:>;?(, .||.))D0. %0047 . !& 1 1D 8 .  .+,. u|| .rOrcVeZdZfdZdej dej fdZxZS)CamembertIntermediatect|tj|j|j |_t|jtrt|j|_ y|j|_ yr) r2r3r4rr7intermediate_sizerr hidden_actstrr intermediate_act_fnrJs rNr3zCamembertIntermediate.__init__ys] YYv1163K3KL f'' -'-f.?.?'@D $'-'8'8D $rOrrScJ|j|}|j|}|Sr)rr)rKrs rNrdzCamembertIntermediate.forwards& =1 00? rOrrys@rNrrxs#9U\\ellrOrcneZdZfdZdej dej dej fdZxZS)CamembertOutputc(t|tj|j|j |_tj|j |j|_tj|j|_ yr) r2r3r4rrr7rr<r=r>r?r@rJs rNr3zCamembertOutput.__init__s` YYv779K9KL f&8&8f>S>STzz&"<"<= rOrrrScr|j|}|j|}|j||z}|Srrrs rNrdzCamembertOutput.forwardrrOrrys@rNrrrrOrceZdZd fd Z d dej dej dzdej dzdej dzdedzdee d ej fd Z d Z xZ S)CamembertLayerNct||j|_d|_t ||j ||_|j |_|j|_|jr.|j st|dt |d|d|_ t||_ t||_ y)Nr$rz> should be used as a decoder model if cross attention is addedFT)rrr)r2r3chunk_size_feed_forward seq_len_dimrr attentionadd_cross_attentionrcrossattentionr intermediaterr)rKrLrrMs rNr3zCamembertLayer.__init__s '-'E'E$+Ff>O>O[de ++#)#=#=  # #?? D6)g!hii"4##' #D  2&9%f- rOrr~rrrrrSc |j||fd|i|\}}|} |jr:|8t|dstd|d|j|d||fd|i|\} }| } t |j |j|j| } | S)Nrrz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`) rrr[rrrfeed_forward_chunkrr) rKrr~rrrrself_attention_output_rcross_attention_output layer_outputs rNrdzCamembertLayer.forwards$24>>  $ ,$  $  q 1 ??4@4!12 =dVDDD )<(;(;%%& ) !0 )  ) % "A 6 0  # #T%A%A4CSCSUe rOcL|j|}|j||}|Sr)rr)rKrintermediate_outputrs rNrz!CamembertLayer.feed_forward_chunks,"//0@A{{#68HI rOrr) rprqrrr3rBrvrur rrrdrrxrys@rNrrs.,48:>;?(, %||%))D0% %0047 % !& 1 1D 8 %  %+,% %NrOrc(eZdZdZfdZdZxZS)CamembertLMHeadz,Camembert Head for masked language modeling.ct|tj|j|j|_tj |j|j|_tj|j|j|_ tjtj|j|_yr)r2r3r4rr7rr<r= layer_normr6decoder ParameterrBrFbiasrJs rNr3zCamembertLMHead.__init__s YYv1163E3EF ,,v'9'9v?T?TUyy!3!3V5F5FG LLV->->!?@ rOc |j|}t|}|j|}|j|}|Sr)rr rrrKfeaturesrxs rNrdzCamembertLMHead.forwards; JJx  G OOA  LLOrOrprqrrrsr3rdrxrys@rNrrs6ArOrcneZdZeZdZdZdZdZdZ dZ e e e dZej fdZxZS)CamembertPreTrainedModelrobertaT)r attentionscross_attentionsct||t|tr t j |j yt|tryt j|jtj|jjdjdt j |jyy)zInitialize the weightsr.r-N)r2 _init_weightsrrinitzeros_rr'copy_r,rBrCr]rEr0)rKrzrMs rNrz&CamembertPreTrainedModel._init_weightss f% fo . KK $  3 4 JJv**ELL9L9L9R9RSU9V,W,^,^_f,g h KK-- .5rO)rprqrrr% config_classbase_model_prefixsupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendrrr_can_record_outputsrBno_gradrrxrys@rNrrsX"L!&*#N"&',3 U]]_//rOrceZdZfdZ d dej dej dzdej dzdej dzdedzdedzd e e d e ej e zfd Z xZS) CamembertEncoderc t|||_tjt |j Dcgc]}t||c}|_ycc}w)N)r) r2r3rLr4 ModuleListrangenum_hidden_layersrlayer)rKrLirMs rNr3zCamembertEncoder.__init__sG  ]]QVW]WoWoQp#qAN6Q$G#qr #qsANrr~rrr use_cacherrSc t|jD]\}} | |||f||d|}t||r|SdS)N)rr)last_hidden_stater) enumeraterr) rKrr~rrrrrr layer_modules rNrdzCamembertEncoder.forward sn )4 OA|(%(> /   M 9+/8O  >B  rO)NNNNN)rprqrrr3rBrvrur boolrrrrrdrxrys@rNrrss48:>;?(,!% || ))D0  %0047  !& 1 1D 8    $; +,  u|| H H rOrcVeZdZfdZdej dej fdZxZS)CamembertPoolerct|tj|j|j|_tj |_yr)r2r3r4rr7rTanh activationrJs rNr3zCamembertPooler.__init__'s9 YYv1163E3EF '')rOrrSc\|dddf}|j|}|j|}|SNr)rr%)rKrfirst_token_tensor pooled_outputs rNrdzCamembertPooler.forward,s6+1a40 #56  6 rOrrys@rNr"r"&s#$ U\\ellrOr"a  The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. ) custom_introceZdZddgZdfd ZdZdZeee dde jdzde jdzd e jdzd e jdzd e jdzd e jdzd e jdzde dzde dzdeedee jezfdZdZxZS)CamembertModelr'rct||||_d|_t ||_t ||_|r t|nd|_ |jy)zv add_pooling_layer (bool, *optional*, defaults to `True`): Whether to add a pooling layer FN) r2r3rLgradient_checkpointingr'rcrencoderr"pooler post_init)rKrLadd_pooling_layerrMs rNr3zCamembertModel.__init__DsU   &+#-f5'/ 1Bof-  rOc.|jjSrrcr9rKs rNget_input_embeddingsz#CamembertModel.get_input_embeddingsUs...rOc&||j_yrr4)rKr}s rNset_input_embeddingsz#CamembertModel.set_input_embeddingsXs*/'rONrPr~r0r,rQrrrrrrSc |du|duz r td|jjr| | n|jj} nd} | rd|b||jjr4t t |jt |jnt |j}||jnd} |j||||| } |j||| ||\}}|j| f||||| |d| } | j}|j|j|nd}t||| jS) Nz:You must specify exactly one of input_ids or inputs_embedsF)rLr)rPr,r0rQrR)r~rembedding_outputrr)r~rrrrr,)r pooler_outputr)rrLrris_encoder_decoderr r get_seq_lengthrc_create_attention_masksr/rr0rr)rKrPr~r0r,rQrrrrrrRr:encoder_outputssequence_outputr)s rNrdzCamembertModel.forward[s|  -t"; <YZ Z ;; ! !%.%:  @U@UII 0)4 8V8V$L $DlZ^ZeZeFfg!5  FUE`!?!?!Afg??%)'#9 + 261M1M)#9-"7+ 2N2 ..'$,,   )"7#9+%    *;;8< 8O O4UY ;-'+;;  rOc|jjrt|j|||}nt|j||}|t|j|||}||fS)N)rLrQr~r)rLrQr~)rLrQr~r)rLrrr)rKr~rr:rrs rNr>z&CamembertModel._create_attention_maskssu ;; ! !/{{.- / N7{{.-N " -%>{{.5&; & "555rO)T) NNNNNNNNN)rprqrr_no_split_modulesr3r6r8r"r#rrBrvr r rrrrrdr>rxrys@rNr,r,5s//0@A"/0 *..2.2,0-1596:(,!%? <<$&?  t+?  t+ ? llT) ? ||d* ?  %||d2? !& t 3? ? $;? +,?  u|| K K?  ? B6rOr,ceZdZdddZfdZdZdZee dde jdzd e jdzd e jdzd e jdzd e jdzd e jdzde jdzde jdzde e dee jezfdZxZS)CamembertForMaskedLM)roberta.embeddings.word_embeddings.weight lm_head.biaszlm_head.decoder.weightzlm_head.decoder.biasct|||jrtj dt ||_t|d|_|jy)NzpIf you want to use `CamembertForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.Fr2 r2r3rloggerwarningrlm_headr,rr1rJs rNr3zCamembertForMaskedLM.__init__sR     NN1 'v. %fF  rOc.|jjSrrMrr5s rNget_output_embeddingsz*CamembertForMaskedLM.get_output_embeddings||###rOc&||j_yrrOrKnew_embeddingss rNset_output_embeddingsz*CamembertForMaskedLM.set_output_embeddings- rONrPr~r0r,rQrrlabelsrrSc t|j|f||||||dd| } | d} |j| } d} |a|j| j}t }|| j d|j j|j d} t| | | j| jS)a token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value >= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` T)r~r0r,rQrr return_dictrNr.losslogitsrr) rrMr\rXrrrLr6rrr)rKrPr~r0r,rQrrrWroutputsr@prediction_scoresmasked_lm_lossloss_fcts rNrdzCamembertForMaskedLM.forwards:$,,   ))%'"7#9    "!* LL9  YY0778F')H%&7&<&;?*.5 ##d*5 ))D05 ((4/ 5 &&- 5 ((4/ 5  %00475 !& 1 1D 85   4'5 +,5  u|| ~ -5 5 rOrDc(eZdZdZfdZdZxZS)CamembertClassificationHeadz-Head for sentence-level classification tasks.cZt|tj|j|j|_|j |j n |j}tj||_ tj|j|j|_ yr) r2r3r4rr7rclassifier_dropoutr?r>r@ num_labelsout_projrKrLrerMs rNr3z$CamembertClassificationHead.__init__s YYv1163E3EF )/)B)B)NF % %TZTnTn zz"45  &"4"4f6G6GH rOc |dddddf}|j|}|j|}tj|}|j|}|j |}|Sr')r@rrBtanhrgrs rNrdz#CamembertClassificationHead.forward"sY Q1W  LLO JJqM JJqM LLO MM! rOrrys@rNrcrcs7IrOrcz Camembert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. c*eZdZfdZee d dejdzdejdzdejdzdejdzdejdzdejdzd e e d e eje zfd ZxZS) "CamembertForSequenceClassificationct|||j|_||_t ||_t |d|_|jyNFrI) r2r3rfrLrc classifierr,rr1rJs rNr3z+CamembertForSequenceClassification.__init__3sJ   ++ 5f=%fF  rONrPr~r0r,rQrWrrSc |j|f||||dd|}|d} |j| } d} ||j| j}|jj |j dk(rd|j_nl|j dkDrL|jtjk(s|jtjk(rd|j_nd|j_|jj dk(rIt} |j dk(r&| | j|j} n| | |} n|jj dk(r=t} | | jd |j |jd } n,|jj dk(rt} | | |} t!| | |j"|j$ S) a token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value >= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Tr~r0r,rQrYrNr$ regressionsingle_label_classificationmulti_label_classificationr.rZ)rror\rXrL problem_typerfr1rBrHrkrsqueezerrrrrr rKrPr~r0r,rQrWrr]r@r\r[r`s rNrdz*CamembertForSequenceClassification.forward>s6$,,  ))%'   "!*1  YYv}}-F{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#FF3D))-JJ+- B @&++b/R))-II,./'!//))   rONNNNNN)rprqrrr3r!rrBrtrurrrrvrrdrxrys@rNrlrl,s .237260426*.C ##d*C ))D0C ((4/ C &&- C ((4/ C   4'C +,C  u|| 7 7C C rOrlc*eZdZfdZee d dejdzdejdzdejdzdejdzdejdzdejdzd e e d e eje zfd ZxZS) CamembertForMultipleChoicect||tj|j|_tj |jd|_t|d|_ |jy)Nr$TrI) r2r3r4r>r?r@rr7ror,rr1rJs rNr3z#CamembertForMultipleChoice.__init__sX  zz&"<"<= ))F$6$6:%fE  rONrPr0r~rWr,rQrrSc "||jdn|jd}|!|jd|jdnd} |!|jd|jdnd} |!|jd|jdnd} |!|jd|jdnd} |1|jd|jd|jdnd} |j| f| | | | dd|}|d}|j |}|j |}|jd|}d}|.|j |j}t}|||}t|||j|jS)a input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value >= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. Nr$r.T)r,r0r~rQrYrZ) r]rrGrr@ror\rXrrrr)rKrPr0r~rWr,rQr num_choicesflat_input_idsflat_position_idsflat_token_type_idsflat_attention_maskflat_inputs_embedsr]r)r\reshaped_logitsr[r`s rNrdz"CamembertForMultipleChoice.forwardsV-6,Aiooa(}GZGZ[\G] CLCXINN2,>?^bLXLdL--b,2C2CB2GHjnR`Rln11"n6I6I"6MNrvR`Rln11"n6I6I"6MNrv(   r=#5#5b#9=;M;Mb;Q R  $,,  *..,      ]3 / ++b+6  YY556F')HOV4D("!//))   rOrx)rprqrrr3r!rrBrtrurrrrvrrdrxrys@rNrzrzs.22637*.0426P ##d*P ((4/P ))D0 P   4' P &&- P ((4/P +,P  u|| 8 8P P rOrzc*eZdZfdZee d dejdzdejdzdejdzdejdzdejdzdejdzd e e d e eje zfd ZxZS) CamembertForTokenClassificationcdt|||j|_|j |jn |j}t j ||_t j|j|j|_ t|d|_ |jyrn)r2r3rfrer?r4r>r@rr7ror,rr1rhs rNr3z(CamembertForTokenClassification.__init__s   ++)/)B)B)NF % %TZTnTn zz"45 ))F$6$68I8IJ%fF  rONrPr~r0r,rQrWrrSc ~|j|f||||dd|}|d} |j| } |j| } d} |W|j| j}t } | | j d|j|j d} t| | |j|jS)a- token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value >= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. TrqrNr.rZ) rr@ror\rXrrrfrrrrws rNrdz'CamembertForTokenClassification.forwards2$,,  ))%'   "!*,,71  YYv}}-F')HFKKDOO= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) Trqrr$r.riN) ignore_indexr)r[ start_logits end_logitsrr) rrsplitrvrlenrGclamprrrr)rKrPr~r0r,rQrrrr]r@r\rr total_loss ignored_indexr` start_lossend_losss rNrdz%CamembertForQuestionAnswering.forward:s0$,,  ))%'   "!*1#)<<r<#: j#++B/::< ''+668   &=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9 (--a0M-33A}EO)//=AM']CH!,@J M:H$x/14J+%!!//))   rO)NNNNNNN)rprqrrr3r!rrBrtrurrrrvrrdrxrys@rNrr.s.2372604263715> ##d*> ))D0> ((4/ > &&- > ((4/ > ))D0> ''$.> +,>  u|| ; ;> > rOrzU Camembert Model with a `language modeling` head on top for CLM fine-tuning. ceZdZdddZfdZdZdZee dde jdzd e jdzd e jdzd e jdzd e jdzd e jdzde jdzde jdzde e e jdzde dzdee jzdeede e jezfdZxZS)CamembertForCausalLMrErFrGct|||jstj dt ||_t|d|_|jy)NzQIf you want to use `CamembertLMHeadModel` as a standalone, add `is_decoder=True.`FrIrJrJs rNr3zCamembertForCausalLM.__init__sL    NNn o&v. %fF  rOc.|jjSrrOr5s rNrPz*CamembertForCausalLM.get_output_embeddingsrQrOc&||j_yrrOrSs rNrUz*CamembertForCausalLM.set_output_embeddingsrVrONrPr~r0r,rQrrrWrrlogits_to_keeprrSc |d} |j|f||||||| | dd | } | j}t| tr t | dn| }|j |dd|ddf}d}|*|j d|||jjd| }t||| j| j| j| jS)aq token_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value >= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` Example: ```python >>> from transformers import AutoTokenizer, CamembertForCausalLM, AutoConfig >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("almanach/camembert-base") >>> config = AutoConfig.from_pretrained("almanach/camembert-base") >>> config.is_decoder = True >>> model = CamembertForCausalLM.from_pretrained("almanach/camembert-base", config=config) >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> prediction_logits = outputs.logits ```NFT) r~r0r,rQrrrrrY)r\rWr6)r[r\rrrr)rrrrkslicerM loss_functionrLr6rrrrr)rKrPr~r0r,rQrrrWrrrrr]r slice_indicesr\r[s rNrdzCamembertForCausalLM.forwards`  I@L  A ))%'"7#9+ A  A  11 8B>SV8W~ot4]k mA}a,?@A  %4%%pVFt{{OeOepiopD0#33!//))$55   rO) NNNNNNNNNNr)rprqrrrar3rPrUr!rrBrtrurr rkrvrrrrdrxrys@rNrr}s#N . $..237260426:>;?*.BF!%-.O ##d*O ))D0O ((4/ O &&- O ((4/ O  %0047O !& 1 1D 8O   4'O uU%6%6784?O $;O ell*O +,O  u|| @ @O O rOr)rrDrzrrlrr,r)Nr)Pcollections.abcrrBtorch.nnr4rrrrr activationsr r cache_utilsr r r generationr masking_utilsrrmodeling_layersrmodeling_outputsrrrrrrrrmodeling_utilsrrprocessing_utilsr pytorch_utilsrutilsrrr utils.genericr!r"utils.output_capturingr#configuration_camembertr% get_loggerrprKModuler'rvfloatrrrrrrrrrrrr"r,rDrcrlrzrrr__all__rrOrNrs,% AA&'CC)J9   G&6@@I54   H %g8"))g8`!% II% <<% % << % LL4' % T\ %%'(%8@)RYY@)FI)biiI)X ")) ..: BII  bii >/>Bbii,///2 ryy @ bii   |6-|6 |6~R 3R R j")), Q )AQ  Q h] !9] ] @C &>C C LK $<K K \ i 3_i  i X rO