^j`ddlZddlmZmZddlmZddlZddlmZddl mcm Z ddl m ZddlmZddlmZddlmZdd lmZmZdd lmZdd lmZmZmZmZdd lm Z dd l!m"Z"ddl#m$Z$ejJe&Z'eeGddeZ(eeGddeZ)GddejTZ+GddejTZ,edGddejTZ-GddejTZ. d1dejTdej^dej^d ej^d!ej^dzd"e0d#e0e1zd$eefd%Z2Gd&d'ejTZ3Gd(d)ejTZ4eGd*d+eZ5eGd,d-e5Z6Gd.d/e5Z7gd0Z8y)2N)CallableSequence) dataclass)initialization)use_kernel_forward_from_hub)FlashAttentionKwargs)BaseModelOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)merge_with_config_defaults)capture_outputs) TimesFmConfigcbeZdZUdZdZej dzed<dZej dzed<y) TimesFmOutputz loc (`torch.Tensor` of shape `(batch_size, )`): The mean of the time series inputs. scale (`torch.Tensor` of shape `(batch_size,)`): The scale of the time series inputs. Nlocscale) __name__ __module__ __qualname____doc__rtorchTensor__annotations__rw/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/timesfm/modeling_timesfm.pyrr,s/ $C #!%E5<<$ %r"rceZdZUdZdZej dzed<dZej dzed<dZ ej e zdzed<y)TimesFmOutputForPredictiona mean_predictions (`torch.Tensor` of shape `(batch_size, sequence_length)`): The mean predictions of the time series. full_predictions (`torch.Tensor` of shape `(batch_size, sequence_length)`): The full predictions of the time series including the mean and the quantiles. loss (`torch.Tensor` of shape `(1,)`, *optional*, returned when `future_values` is provided): The loss of the TimesFM model. Nmean_predictionsfull_predictionsloss) rrrrr&rrr r'r(floatr!r"r#r%r%:sI-1ellT)0,0ellT)0(,D%,,  %,r"r%c0eZdZdZdeffd ZddZxZS) TimesFmMLPzPax MLP in pytorch.configct||j}|j}t j |||_t j |||_t j|d|_ y)Nư>)normalized_shapeeps) super__init__ hidden_sizeintermediate_sizennLinear gate_proj down_proj LayerNorm layer_norm)selfr,r3r4 __class__s r#r2zTimesFmMLP.__init__Ns] (( "44;0AB#4kB,, Nr"c|j|}|j|}tj|}|j |}||d|dddddfz z}||zS)N?)r:r7Frelur8)r;xpaddingsgate_inpgateoutputss r#forwardzTimesFmMLP.forwardWsc??1%~~h'vvd|..&  x1d ';!;% Ht?T?T4T(UU {{]--k:::r"c^t|jjd|jS)Nz, eps=)tuplerashaperb)r;s r# extra_reprzTimesFmRMSNorm.extra_reprs* ))*+6$2G2G1HIIr")r.) rrrr)r2rrrFrsrIrJs@r#r\r\ws7$$$$;U\\;ell;Jr"r\c0eZdZdZdeffd ZddZxZS)TimesFmPositionalEmbeddingz6Generates position embedding for a given 1-d sequence.r,c t||j}|j}||c|_|_|j|_|j dz}t jt|t|z t|dz dz }|jd|tjtj|tj| zzy)Nrerinv_timescalesrh)r1r2 min_timescale max_timescaler3embedding_dimsmathlogr)maxregister_bufferrexparangerj)r;r,ryrznum_timescaleslog_timescale_incrementr<s r#r2z#TimesFmPositionalEmbedding.__init__s ,, ,, 1> .D.$00,,1"&((5+?% BV+V"WZ]^lop^prsZt"t   EIIell>&W[rZr&rs s r"cN| | td|Jtj|tj|jj j d}n'|jdk7rtd|j|jg|jd|jjdddz}tjtj|tj|gd}tj|ddd|jdzf}|S) aGenerates a Tensor of sinusoids with different frequencies. Args: seq_length: an optional Python int defining the output sequence length. if the `position` argument is specified. position: [B, seq_length], optional position for each token in the sequence, only required when the sequence is packed. Returns: [B, seqlen, D] if `position` is specified, else [1, seqlen, D] z.Either position or seq_length must be providedrhdevicerrez*position must be 2-dimensional, got shape rrfdim) ValueErrorrrrjrwr unsqueezendimrrviewcatsincosr?padr{)r; seq_lengthposition scaled_timesignals r#rFz"TimesFmPositionalEmbedding.forwards   2MN N  ||JemmDL_L_LfLfgqqrstH ]]a I(..IYZ[ [#hmm7X^^7Q7$:M:M:R:RSTVWY[:\\ EIIk2EIIk4JKQRSv1a)<)@P@PQ r"queryr]ctj|jjdt j |j z }||dddddfzS)Ng^$3eG?)r?softplusrmulr|sqrtr)r;rrs r# _scale_queryzTimesFmAttention._scale_querysJ 4<<(,,[499T]];S-STuT4q0111r"Nrcrrc |jdd}g|d|j}|j|j|j dd}|j |}|j |j|j dd}|j|j|j dd}tj|jjt} | |||||f|jsdn |jdd|\} } | jg|dj!} |j#| } | | fS)Nrfrrer>)rr)rrrrrrrrrr get_interfacer,_attn_implementationrrrreshaperr) r;rcrr input_shape hidden_shaperrrattention_interfacerrs r#rFzTimesFmAttention.forwardsW $))#2. 88b8$--8 {{=166|DNNqRST ((6 [[/44\BLLQPQR {{=166|DNNqRST (?(M(M KK , ,.L) %8      % $}}C$2H2H %  % ! \*k));;;;FFH kk+. L((r"rG)rrrrrintr2rrrr r rqrFrIrJs@r#rrsvR}RR(2%,,25<<2/3)||) t+)-. ) u||U\\D00 1 )r"rceZdZdZdedeffd Zdejdejdejdejfd Z xZ S) TimesFmDecoderLayerzTransformer layer.r,rct|t|||_t ||_t |j|j|_ y)N)r)r0) r1r2r self_attnr+mlpr\r3 rms_norm_epsinput_layernormrs r#r2zTimesFmDecoderLayer.__init__sC )&IFf%-f.@.@fFYFYZr"rcrrBr]c |}|j|}|j||\}}||z}|j||}|S)N)rcr)rB)rrr)r;rcrrBrrY_s r#rFzTimesFmDecoderLayer.forwards]!,,]; >>')*  q!=0 B r") rrrrrrr2rrrFrIrJs@r#rr sW[}[[|| ,,   r"rcreZdZUeed<dZdgZdZdZdZ e e dZ e jfdZxZS) TimesFmPreTrainedModelr,timesfmr past_values)timeT)rc attentionsc "t||t|tr t j |j yt|tr|jdz}|j|j}}tjt|t|z t|dz dz }t j|j |t#j$t#j&|t"j(| zzyy)Nrerrx)r1 _init_weights isinstancerinitones_rrur{rzryr|r}r)r~copy_rwrrrrj)r;rrrzryrr<s r#rz$TimesFmPreTrainedModel._init_weights9s f% f. / JJv~~ &  : ;#22a7N+1+?+?AUAU=M&*hhu]/CeMFZ/Z&[^a"A_' # JJ%%))ELLu}}MQhPhhij  rrrrT)rsequence_lengthrhrrN)rrB)last_hidden_staterr)rrrr,rrrrrrrhrrrrrrrrconcat_timesfm_shift_padded_seqr_prepare_4d_attention_maskrrr)r;rrrrbsizepatched_inputsrstats concat_inputs model_inputpatched_paddingpos_embf_embrcrlayers r#rFzTimesFmModel.forwardqs&!!!$$))%T[[5M5MN*//r4;;;S;ST  IIlS( )DKK,A,A A LLN$8$8AVAV W   {{ IInt{{':':: ;dkk>S>S S LLL$6$6|?R?R S  !% 7 7 U(3+=> ><"@bI ))-8  ))Lb9!< ;; / /'' (9(9!(<=GllG9{/@/@/C#CKG44_gNG 7 "K d#u $ 88*)//2%% '' 9 [[!@4;;#@#@A E!-( M +a(  r"rNr rhrrc|jrtj|jntj|j}|&|j |j dddd}||z}|rbtjtj||f|||zd}|j dd||}|tj||}|S|}|S)a Creates 4D attention mask and combines causal and padding masks if needed. Args: attention_mask: Optional tensor of shape (batch_size, seq_length) containing padding mask sequence_length: Length of the sequence dtype: Data type of the mask device: Device of the mask is_causal: Whether to apply causal masking Returns: 4D attention mask of shape (batch_size, 1, seq_length, seq_length) rrrfr)diagonal) is_floating_pointrfinforiinforrrtriur`minimum)rr rhrr min_value causal_masks r#r z'TimesFmModel._prepare_4d_attention_masks,/4.E.EEKK&**5;;W\K]KaKa   %+001E1Ea1H!QPRSN+i7N ** O_=USYZ]ffK&**1a/RK)!&~{!K"-r"paddingcFdtjfd}tjd|z d}||}tj|jd}|||ddf}|||ddf}d|z }tj|d} tj | d } tj||zd} | | z } || j d z |z} tj| dzd| z } tj | d } tj| }| |fS) aCalculates mean and standard deviation of `inputs` across axis 1. It excludes values where `padding` is 1. Args: inputs: A PyTorch tensor of shape [b, n, p]. padding: A PyTorch tensor of shape [b, n, p] with values 0 or 1. Returns: A tuple containing the mean and standard deviation. We return the statistics of the first patch with more than three non-padded values. arrc(tj|dk\jtjd}|dk\jtjj d}tj |dk(|j ddz |S)Nrrrr)rargmaxriint32sumrrr)r"indicesrow_sums r#_get_patch_indexz?TimesFmModel._timesfm_masked_mean_std.._get_patch_indexskllC1H==#=1EGaxmmEKK0444;G;;w!|SYYq\A-=wG Gr"rrerrNr>rrfr)rrr&rrrrrr)rr r)pad_sum patch_indicesbidxsr"rmasknum_valid_elements masked_sum masked_meanmasked_centered_arr masked_var masked_stds r#rz%TimesFmModel._timesfm_masked_mean_stds  H%,, H ))AKQ/(1  V\\!_-UM1,-e]A-.3w#YYt3"[[);EYYsTzq1  #55  #[%:%:2%>>$FYY2A51=@RR [[5 ZZ + J&&r"r-seqc|j\}}}|dk(}|jtjj d}d||j d<tj ||jjdddj|d|}||ddddfz |z}|jd|} | S)zShifts rows of seq based on the first 0 in each row of the mask. Args: mask: mask tensor of shape [B, N] seq: seq tensor of shape [B, N, P] Returns: The shifted sequence. rrrrf)rN) rrrirr%r$anyrrrexpandgather) r-r4 batch_sizenum_seq feature_dimnew_maskr' idx_range shifted_idx shifted_seqs r#r z&TimesFmModel._timesfm_shift_padded_seqs,/99( G[%)QY++ekk*11a18)+!$$%LL<AA!RKRRS]_acno !71dD=#99WD jjK0 r")T)rrrrr2rrrqrrrr LongTensorr rrrF staticmethodrrhrboolr rr rIrJs@r#rrLs}&$ll$27,,$ u||U5<<#=>> ?$  F \\F #--F ll F +, F  F  F P  + t+++{{+ +  +   ++Z,',' ,'QVW\WcWcejeqeqWqQr,','\ 5<<ELLr"rcReZdZdZdeffd Z ddeejdee dzde dzde ejd ffd Z d ejd e ejejfdejfd Z dejdejdejfdZ ee ddeejdeeje zdzde dzdejdzde dzdededeedefdZedejde deejfdZxZS)TimesFmModelForPredictionz/TimesFM model for quantile and mean prediction.r,cJt||||_|j|_|j |_t||_t|j|j dt|jzz|j|_|jy)Nrr)r1r2r,context_length context_lenhorizon_length horizon_lenrdecoderrLr3len quantilesr4horizon_ff_layerr)r;r,r<s r#r2z"TimesFmModelForPrediction.__init__=s   !00!00#F+ !5))--S9I9I5J1JK00!  r"NrrrGr].cX| |j}gg}}|D]}|jd}tj||jz|j |j }||kr||z } tjtj| |j |j |gd}tjtj| |j |j |gd}n||kDr|| d}|||jz d}|j||j|tj|dtj|df} |E| tj|dt|tjjddfz} | S)aPad/truncate input time series to `context_len` and build a padding mask. Args: inputs: A list of 1d Tensors. Each Tensor is the context time series of a single forecast task. freq: Optional list of frequencies (returned as a tensor when provided). context_len: Optional context length override (defaults to `self.context_len`). Returns: Tuple of (padded_inputs, padding_mask) and optionally a freq tensor. Nrrrrxrfr)rGrrrzerosrIrhrrr`appendstackrrKr%r) r;rrrGinput_ts input_paddingts input_lenr  num_front_padresults r# _preprocessz%TimesFmModelForPrediction._preprocessPs  **K"$b- *B Ikk)d.>.>">bhhWYW`W`aG;& +i 7 YY MRTR[R[ \^`aghi))UZZ RXXV]VdVd%egn$ouvw[( &!K$2B2B$B"C"EF OOB    ) *++hA. Mq0QR  u||D3v;,?u{{S[[\^`abddF r" model_outputrc|j|}|j\}}}|j|||jjt |jj dz}|\}}||dddddfz|dddddfzS)z*Postprocess output of stacked transformer.rN)rMrrrr,rHrKrL) r;rYr output_tsbnrrrs r#_postprocess_outputz-TimesFmModelForPrediction._postprocess_outputus )),7 //1aNN1a)C)CSI^I^E_bcEcd  E5D$!4551dD$;N8OOOr" predictionstargetsc*g}t|jjD]M\}}||d|fz }tj|dz |z||z}|j |j Otj|j S)N.r) enumerater,rLrr~rPrlrQ)r;r_r`lossesiqerrorsr(s r#_quantile_lossz(TimesFmModelForPrediction._quantile_losssdkk334 'DAq{3622F99a!ev-q6z:D MM$))+ & '{{6"''))r"r window_size future_valuesforecast_context_lenreturn_forecast_on_contexttruncate_negativerc > | |j} n|} |dj} |D cgc]} | | d } } tjtj| D cgc]} tj| c} } |Yg}g}t | D]A\}} |j |j| ||*|j ||gdzC|} ||}|$tjddgt| z}|j| |\}}}|j| }|j| }|j| }|}|jd}g}|jd|jd|jzk7r8td|jdd|jdd|j|j j"}|j|zdz |z}t%|D]/}|ddd|jdf}|dd| df}|dd| df}|j&d|||d |}|j)|j*|j,|j.f}|rl|dk(rg|dddd d|j j0ddf}|j3|j5dd |j5d }|j7||ddd d|df}|ddd d|ddf}|j7|tj8||gd }2|rHtj8|d ddd||j j0z |jzddf}n-tj8|d ddd|jddf}|dddddf}|*|dddd f|dddd fz}|dddd f|dddd fz}|rQ| dk\} tj:| |j=d|}tj:| |j=d|}d}!|9t?j@||}"|jC|ddddddf|}#|"|#z}!tEj*|jF|jH|||!Scc} wcc} w)a past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Past values of the time series that serves as input to the model. freq (`torch.LongTensor` of shape `(batch_size,)`): Frequency indices for the time series data. window_size (`int`, *optional*): Window size of trend + residual decomposition. If None then we do not do decomposition. future_values (`torch.Tensor`, *optional*): Optional future time series values to be used for loss computation. forecast_context_len (`int`, *optional*): Optional max context length. return_forecast_on_context (`bool`, *optional*): True to return the forecast on the context when available, i.e. after the first input patch. truncate_negative (`bool`, *optional*): Truncate to only non-negative values if any of the contexts have non-negative values, otherwise do nothing. Example: ```python >>> from transformers import TimesFmModelForPrediction >>> model = TimesFmModelForPrediction.from_pretrained("google/timesfm-2.0-500m-pytorch") >>> forecast_input = [torch.linspace(0, 20, 100).sin(), torch.linspace(0, 20, 200).sin(), torch.linspace(0, 20, 400).sin()] >>> frequency_input = torch.tensor([0, 1, 2], dtype=torch.long) >>> # Generate >>> with torch.no_grad(): >>> outputs = model(past_values=forecast_input, freq=frequency_input, return_dict=True) >>> point_forecast_conv = outputs.mean_predictions >>> quantile_forecast_conv = outputs.full_predictions ``` Nrrez6No frequency provided via `freq`. Default to high (0).rz=Length of paddings must match length of input + horizon_len: z != z + )rrrrfr)axis.r)r rrcr&r'r(r!)%rGrrrrQrbextend_timesfm_moving_averageloggerinforKrXrirrrIrr,rHrrJr^r rrrrsizerP concatenater clamp_minr?mse_lossrgr%rrc)$r;rrrhrirjrkrlr fcontext_lenrrTrinp_min new_inputs new_freqsrdrRrSinp_freq final_outrG full_outputsoutput_patch_lennum_decode_patches step_indexcurrent_paddingdecoder_output fprop_outputs new_full_tsnew_ts mean_outputsrr(rv quantile_losss$ r#rFz!TimesFmModelForPrediction.forwards^ '++L/LQ&&/:;"l]^$;;))EKK(H22(HIJ  "JI"6* 42!!$">">r;"OP#$$d1gY]3 4 F  < KKP Q3V$D,0,<,M,8DLL-$$1- -N !4400##^%9%9:M *jAo+Ass4Ndkk6N6N4NPQ,QR )11+2B2B12Er;K[K[\]K^_ ##K0"1b*;+;*;Q#>?F'2/@0@/@!(CDK    ,))9f*=BGI/ H2 & ,,\BPkDKK$<$<H // Q12X0F VMm+D),>>%00(66))   g<(Hs RR r"c4tj||dz dfdd}tj||j|j |z }tj |jddd|jdddj}|||z gS)zCCalculates the moving average using PyTorch's convolution function.rrconstantrrf) r?rrr`rhrconv1drsqueeze)r"rh arr_paddedkernel smoothed_arrs r#rpz1TimesFmModelForPrediction._timesfm_moving_averagesUU3q! 4j!D KsyyL{Zxx 1b 96;;q!R;PQYY[ cL011r"r)NNNNFF)rrrrrr2rrrrrqrXr^rgrrrBr rr%rFrAlistrprIrJs@r#rDrD:s9}(lp#u||,#4rs* .! &7B/F&RR750   H %  &O &  &  - -  -,!299!,Y'JRYYJ(J(++j% II%,,% %,, % LL4' %  %S[%'(%,9)ryy9)x"))@_>j)jjZm2 6m2` Rr"