^judZddlZddlZddlmZddlmZmZmZddlm Z ddl m Z ddl mZdd lmZmZmZmZmZmZmZdd lmZdd lmZmZd d lmZej<eZ GddejBZ"GddejBZ#GddejHZ%GddejBZ&GddejBZ'GddejBZ(GddejBZ)GddejBZ*GddejBZ+Gd d!ejBZ,Gd"d#ejBZ-Gd$d%ejBZ.eGd&d'eZ/eGd(d)e/Z0eGd*d+e/Z1ed,-Gd.d/e/Z2eGd0d1e/Z3eGd2d3e/Z4eGd4d5e/Z5gd6Z6y)7zPyTorch SqueezeBert model.N)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)initialization)ACT2FN)create_bidirectional_mask)BaseModelOutputBaseModelOutputWithPoolingMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)auto_docstringlogging)SqueezeBertConfigc*eZdZdZfdZddZxZS)SqueezeBertEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c|t|tj|j|j |j |_tj|j|j |_ tj|j|j |_ tj|j|j|_ tj|j |_|j%dt'j(|jj+ddy)N) padding_idxeps position_idsrF) persistent)super__init__r Embedding vocab_sizeembedding_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings LayerNorm hidden_sizelayer_norm_epsDropouthidden_dropout_probdropoutregister_buffertorcharangeexpandselfconfig __class__s /var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/squeezebert/modeling_squeezebert.pyr"zSqueezeBertEmbeddings.__init__0s !||F,=,=v?T?Tbhbubuv#%<<0N0NPVPePe#f %'\\&2H2H&J_J_%`"f&8&8f>S>STzz&"<"<=   ELL)G)GHOOPWXej  c||j}n|jdd}|d}||jddd|f}|:tj|tj|jj }||j |}|j|}|j|}||z|z} |j| } |j| } | S)Nrrdtypedevice) sizerr3zeroslongr?r'r)r+r,r1) r7 input_idstoken_type_idsr inputs_embeds input_shape seq_lengthr)r+ embeddingss r:forwardzSqueezeBertEmbeddings.forward>s  #..*K',,.s3K ^  ,,Q  ^ J"%88;PP ^^J/ \\*- r;)NNNN__name__ __module__ __qualname____doc__r"rI __classcell__r9s@r:rr-sQ  r;rc(eZdZdZfdZdZxZS) MatMulWrapperz Wrapper for torch.matmul(). This makes flop-counting easier to implement. Note that if you directly call torch.matmul() in your code, the flop counter will typically ignore the flops of the matmul. c"t|yN)r!r")r7r9s r:r"zMatMulWrapper.__init__]s r;c.tj||S)a0 :param inputs: two torch tensors :return: matmul of these tensors Here are the typical dimensions found in BERT (the B is optional) mat1.shape: [B, , M, K] mat2.shape: [B, , K, N] output shape: [B, , M, N] )r3matmul)r7mat1mat2s r:rIzMatMulWrapper.forward`s||D$''r;rJrPs@r:rRrRWs (r;rRceZdZdZddZdZy)SqueezeBertLayerNormz This is a nn.LayerNorm subclass that accepts NCW data layout and performs normalization in the C dimension. N = batch C = channels W = sequence length cHtjj|||y)N)normalized_shaper)rr,r")r7r-rs r:r"zSqueezeBertLayerNorm.__init__rs d[cJr;c|jddd}tjj||}|jdddS)Nrr)permuterr,rI)r7xs r:rIzSqueezeBertLayerNorm.forwardus= IIaA  LL q )yyAq!!r;N)g-q=)rKrLrMrNr"rIr;r:rZrZks K"r;rZc(eZdZdZfdZdZxZS)ConvDropoutLayerNormz8 ConvDropoutLayerNorm: Conv, Dropout, LayerNorm ct|tj||d||_t ||_tj||_yNr in_channels out_channels kernel_sizegroups) r!r"rConv1dconv1drZ layernormr/r1)r7cincoutrj dropout_probr9s r:r"zConvDropoutLayerNorm.__init__sB iiCdPQZ`a -d3zz,/ r;cv|j|}|j|}||z}|j|}|SrT)rlr1rm)r7 hidden_states input_tensorr`s r:rIzConvDropoutLayerNorm.forwards: KK & LLO  NN1 r;rJrPs@r:rcrc{s0r;rcc(eZdZdZfdZdZxZS)ConvActivationz* ConvActivation: Conv, Activation czt|tj||d||_t ||_yre)r!r"rrkrlr act)r7rnrorjrwr9s r:r"zConvActivation.__init__s1 iiCdPQZ`a #;r;cF|j|}|j|SrT)rlrw)r7r`outputs r:rIzConvActivation.forwardsQxxr;rJrPs@r:rurus  r;ruc8eZdZdfd ZdZdZdZdZxZS)SqueezeBertSelfAttentionct|||jzdk7rtd|d|jd|j|_t ||jz |_|j|j z|_tj||d||_ tj||d||_ tj||d||_ tj|j|_tjd|_t#|_t#|_y ) z config = used for some things; ignored for others (work in progress...) cin = input channels = output channels groups = number of groups to use in conv1d layers rzcin (z6) is not a multiple of the number of attention heads ()rrfrdimN)r!r"num_attention_heads ValueErrorintattention_head_size all_head_sizerrkquerykeyvaluer/attention_probs_dropout_probr1SoftmaxsoftmaxrR matmul_qk matmul_qkv)r7r8rnq_groupsk_groupsv_groupsr9s r:r"z!SqueezeBertSelfAttention.__init__s   ++ +q 0uRSYSmSmRnnop $*#=#= #&sV-G-G'G#H !558P8PPYY3SaX`a 993AV^_YY3SaX`a zz&"E"EF zzb) &'/r;c|jd|j|j|jdf}|j|}|j ddddS)z - input: [N, C, W] - output: [N, C1, W, C2] where C1 is the head index, and C2 is one head's contents rrrrr^)r@rrviewr_r7r` new_x_shapes r:transpose_for_scoresz-SqueezeBertSelfAttention.transpose_for_scoress] vvx{D$<$V>VXYX^X^X`acXde AFFK yyAq!$$r;c|jd|j|j|jdf}|j|}|S)z - input: [N, C, W] - output: [N, C1, C2, W] where C1 is the head index, and C2 is one head's contents rr)r@rrrrs r:transpose_key_for_scoresz1SqueezeBertSelfAttention.transpose_key_for_scoressM vvx{D$<$V>VXYX^X^X`acXde AFFK r;c|jddddj}|jd|j|jdf}|j|}|S)zE - input: [N, C1, W, C2] - output: [N, C, W] rrrr^)r_ contiguousr@rrrs r:transpose_outputz)SqueezeBertSelfAttention.transpose_outputs\ IIaAq ! , , .vvx{D$6$6 D AFFK r;c|j|}|j|}|j|}|j|}|j |}|j|} |j ||} | t j|jz } || |z} |j| } |j| } |j| | } |j| } d| i} |r| | d<| S)z expects hidden_states in [N, C, W] data layout. The attention_mask data layout is [N, W], and it does not need to be transposed. context_layerattention_score) rrrrrrmathsqrtrrr1rr)r7rrattention_maskoutput_attentionsmixed_query_layermixed_key_layermixed_value_layer query_layer key_layer value_layerrattention_probsrresults r:rIz SqueezeBertSelfAttention.forwards !JJ}5((=1 JJ}5//0AB 11/B //0AB ..i@)DIId6N6N,OO  %->O,,7,,7E --m< !=1 (7F$ % r;)rrr) rKrLrMr"rrrrIrOrPs@r:r{r{s*0%"r;r{c$eZdZfdZdZxZS)SqueezeBertModulect||j}|j}|j}|j}t |||j |j |j|_t|||j|j|_ t|||j|j|_t|||j"|j|_y)a - hidden_size = input chans = output chans for Q, K, V (they are all the same ... for now) = output chans for the module - intermediate_size = output chans for intermediate layer - groups = number of groups for all layers in the BertModule. (eventually we could change the interface to allow different groups for different layers) )r8rnrrr)rnrorjrp)rnrorjrwN)r!r"r-intermediate_sizer{rrr attentionrcpost_attention_groupsr0post_attentionruintermediate_groups hidden_act intermediate output_groupsry)r7r8c0c1c2c3r9s r:r"zSqueezeBertModule.__init__s         % %   1rFOOfoo`f`o`o 3F$@$@vOiOi +r6C]C]cictctu*F$8$8vGaGa  r;c|j|||}|d}|j||}|j|}|j||}d|i} |r|d| d<| S)Nr feature_mapr)rrrry) r7rrrrattattention_outputpost_attention_outputintermediate_output layer_output output_dicts r:rIzSqueezeBertModule.forwards|nn]Nz.SqueezeBertEncoder.__init__..-s#g!$5f$=#gs) r!r"r%r-r ModuleListrangenum_hidden_layerslayersr6s `r:r"zSqueezeBertEncoder.__init__$sW $$(:(::  2 : mm#guVMeMeGf#gg r;c|jddd}|rdnd}|rdnd}|jD]T}|r,|jddd}||fz }|jddd}|j|||} | d}|sL|| dfz }V|jddd}|r||fz }|std|||fDSt |||S) Nrr^rrarrc3&K|] }|| ywrTra)rvs r:rz-SqueezeBertEncoder.forward..QshqZ[Zghs)last_hidden_staterr attentions)r_rrItupler ) r7rrrroutput_hidden_states return_dictall_hidden_statesall_attentionslayerrs r:rIzSqueezeBertEncoder.forward/s&--aA6 "6BD0d[[ EE# - 5 5aA > !m%55! - 5 5aA >  ==HYZL(7M <0A#B"DD E&--aA6  -!1 1 h]4E~$Vhh h+;LYg  r;)NFFTrrPs@r:rr#s h" % r;rc$eZdZfdZdZxZS)SqueezeBertPoolerct|tj|j|j|_tj |_yrT)r!r"rLinearr-denseTanh activationr6s r:r"zSqueezeBertPooler.__init__Xs9 YYv1163E3EF '')r;c\|dddf}|j|}|j|}|S)Nr)rr)r7rrfirst_token_tensor pooled_outputs r:rIzSqueezeBertPooler.forward]s6+1a40 #56  6 r;rrPs@r:rrWs $ r;rc$eZdZfdZdZxZS)"SqueezeBertPredictionHeadTransformcht|tj|j|j|_t |jtrt|j|_ n|j|_ tj|j|j|_ y)Nr) r!r"rrr-r isinstancerstrr transform_act_fnr,r.r6s r:r"z+SqueezeBertPredictionHeadTransform.__init__gs{ YYv1163E3EF f'' -$*6+<+<$=D !$*$5$5D !f&8&8f>S>STr;cl|j|}|j|}|j|}|SrT)rrr,r7rrs r:rIz*SqueezeBertPredictionHeadTransform.forwardps4 =1 --m< }5 r;rrPs@r:rrfsUr;rc$eZdZfdZdZxZS)SqueezeBertLMPredictionHeadct|t||_t j |j |jd|_t jtj|j|_ y)NT)bias) r!r"r transformrrr-r$decoder Parameterr3rArr6s r:r"z$SqueezeBertLMPredictionHead.__init__xs[ ;FCyy!3!3V5F5FTR LLV->->!?@ r;cJ|j|}|j|}|SrT)rrrs r:rIz#SqueezeBertLMPredictionHead.forwards$}5  ]3 r;rrPs@r:rrwsAr;rc$eZdZfdZdZxZS)SqueezeBertOnlyMLMHeadcBt|t||_yrT)r!r"r predictionsr6s r:r"zSqueezeBertOnlyMLMHead.__init__s 6v>r;c(|j|}|SrT)r)r7sequence_outputprediction_scoress r:rIzSqueezeBertOnlyMLMHead.forwards ,,_=  r;rrPs@r:rrs ?!r;rcVeZdZUeed<dZejfdZxZ S)SqueezeBertPreTrainedModelr8 transformercXt||t|tr t j |j yt|trZt j|jtj|jjdjdyy)zInitialize the weightsrrN)r! _init_weightsrrinitzeros_rrcopy_rr3r4shaper5)r7moduler9s r:rz(SqueezeBertPreTrainedModel._init_weightssx f% f9 : KK $  5 6 JJv**ELL9L9L9R9RSU9V,W,^,^_f,g h7r;) rKrLrMr__annotations__base_model_prefixr3no_gradrrOrPs@r:rrs+ %U]]_iir;rc eZdZfdZdZdZe ddejdzdejdzdejdzdejdzd ejdzd e dzd e dzd e dzd e e zfdZ xZS)SqueezeBertModelct||t||_t ||_t ||_|jyrT) r!r"rrHrencoderrpooler post_initr6s r:r"zSqueezeBertModel.__init__s@  /7)&1 '/  r;c.|jjSrTrHr'r7s r:get_input_embeddingsz%SqueezeBertModel.get_input_embeddingss...r;c&||j_yrTr r7new_embeddingss r:set_input_embeddingsz%SqueezeBertModel.set_input_embeddingss*8'r;NrCrrDrrErrrreturnc  ||n|jj}||n|jj}||n|jj}| | t d|#|j |||j } n!||j dd} n t d| |jn |j} |tj| | }|&tj| tj| }|j||||} t|j| |}|j| ||||} | d }|j|}|s ||f| d dzSt!||| j"| j$ S) NzDYou cannot specify both input_ids and inputs_embeds at the same timerz5You have to specify either input_ids or inputs_embeds)r?r=)rCrrDrE)r8rEr)rrrrrrrr)r pooler_outputrrr)r8rrrr%warn_if_padding_and_no_attention_maskr@r?r3onesrArBrHr rr r rrr)r7rCrrDrrErrrkwargsrFr?embedding_outputencoder_outputsrrs r:rIzSqueezeBertModel.forwards2C1N-TXT_T_TqTq$8$D $++JjJj &1%cd d  "  6 6y. Q#..*K  &',,.s3KTU U%.%:!! @T@T  !"ZZ FCN  !"[[EJJvVN??l>iv+ 3;;*)  ,,*)/!5# ' *!, O4 #]3oab6II I)-')77&11   r;)NNNNNNNN)rKrLrMr"rrrr3Tensor FloatTensorboolrr rIrOrPs@r:rrs/9*..2.2,026)-,0#'? <<$&?  t+?  t+ ? llT) ? ((4/ ?  $;? #Tk? D[?  + +? ? r;rc6eZdZdddZfdZdZdZe ddejdzd ejdzd ejdzd ejdzd ejdzd ejdzde dzde dzde dzde e zfdZ xZS)SqueezeBertForMaskedLMzcls.predictions.biasz-transformer.embeddings.word_embeddings.weight)zcls.predictions.decoder.biaszcls.predictions.decoder.weightct||t||_t ||_|j yrT)r!r"rrrclsr r6s r:r"zSqueezeBertForMaskedLM.__init__s5  +F3)&1 r;cB|jjjSrT)r!rrr s r:get_output_embeddingsz,SqueezeBertForMaskedLM.get_output_embeddingssxx##+++r;c||jj_|j|jj_yrT)r!rrrrs r:set_output_embeddingsz,SqueezeBertForMaskedLM.set_output_embeddings s,'5$$2$7$7!r;NrCrrDrrElabelsrrrrc | | n|jj} |j|||||||| } | d} |j| } d}|Ft }|| j d|jj |j d}| s| f| ddz}||f|zS|St|| | j| jS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` NrrDrrErrrrrr^losslogitsrrr) r8rrr!rrr$r rrr)r7rCrrDrrEr&rrrroutputsrrmasked_lm_lossloss_fctrys r:rIzSqueezeBertForMaskedLM.forwards(&1%| | n|jj} |j|||||||| } | d} |j| } |j | } d}||jj |j dk(rd|j_nl|j dkDrL|jtjk(s|jtjk(rd|j_nd|j_|jj dk(rIt}|j dk(r&|| j|j}n|| |}n|jj dk(r=t}|| jd|j |jd}n,|jj dk(rt}|| |}| s| f| ddz}||f|zS|St!|| | j"| j$ S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Nr(r regressionsingle_label_classificationmulti_label_classificationrr^r))r8rrr1r6 problem_typer5r>r3rBrrsqueezerrrrrrr)r7rCrrDrrEr&rrrrr,rr+r*r.rys r:rIz,SqueezeBertForSequenceClassification.forwardWs(&1%G>SINN2y~~b'9:Y] M[Mg,,R1D1DR1HImqM[Mg,,R1D1DR1HImqGSG_|((\->->r-BCei (   r=#5#5b#9=;M;Mb;Q R  "" ))%'/!5##     ]3 / ++b+6  ')HOV4D%''!"+5F)-)9TGf$ Ev E("!//))   r;r/) rKrLrMr"rr3rrrrrIrOrPs@r:r>r>s*..2.2,0-1&*)-,0#'W <<$&W  t+W  t+ W llT) W ||d* W  t#W  $;W #TkW D[W  * *W W r;r>c eZdZfdZe ddej dzdej dzdej dzdej dzdej dzdej dzd edzd edzd edzd ee zfd Z xZ S)!SqueezeBertForTokenClassificationc,t|||j|_t||_t j |j|_t j|j|j|_ |jyrT) r!r"r5rrrr/r0r1rr-r6r r6s r:r"z*SqueezeBertForTokenClassification.__init__ sj   +++F3zz&"<"<= ))F$6$68I8IJ r;NrCrrDrrEr&rrrrc | | n|jj} |j|||||||| } | d} |j| } |j | } d}|rIr3rErrr)7rNrr3rtorch.nnrrrrr activationsr masking_utilsr modeling_outputsr r r rrrrmodeling_utilsrutilsrrconfiguration_squeezebertr get_loggerrKloggerModulerrRr,rZrcrur{rrrrrrrrrr3r>rErI__all__rar;r:res! AA&!6.9   H %'BII'T(BII(( "2<< " 299(  RYY  XryyXv' 'T1 1 h  ""))&!RYY! i i iQ 1Q Q hH 7H H V S +ES  S lc #=c c L> (B> > BI &@I I X r;