^jwdZddlZddlmZmZddlmZddlZddlm Z ddl m cm Z ddl mZddlmZddlmZdd lmZmZdd lmZdd lmZmZmZmZdd lmZdd l m!Z!ddl"m#Z#ddl$m%Z%ddl&m'Z'ejPe)Z*eeGddeZ+eeGddeZ,Gdde jZZ.Gdde jZZ/Gdde#Z0Gdde jZZ1Gdd e jZZ2Gd!d"e jZZ3eGd#d$eZ4eGd%d&e4Z5Gd'd(e4Z6gd)Z7y)*zPyTorch TimesFM model.N)CallableSequence) dataclass)initialization)FlashAttentionKwargs)BaseModelOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)merge_with_config_defaults)capture_outputs) LlamaRMSNorm)simple_eager_attention_forward) TimesFmConfigcbeZdZUdZdZej dzed<dZej dzed<y) TimesFmOutputz loc (`torch.Tensor` of shape `(batch_size, )`): The mean of the time series inputs. scale (`torch.Tensor` of shape `(batch_size,)`): The scale of the time series inputs. Nlocscale) __name__ __module__ __qualname____doc__rtorchTensor__annotations__rv/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/timesfm/modular_timesfm.pyrr(s/ $C #!%E5<<$ %r$rceZdZUdZdZej dzed<dZej dzed<dZ ej e zdzed<y)TimesFmOutputForPredictiona mean_predictions (`torch.Tensor` of shape `(batch_size, sequence_length)`): The mean predictions of the time series. full_predictions (`torch.Tensor` of shape `(batch_size, sequence_length)`): The full predictions of the time series including the mean and the quantiles. loss (`torch.Tensor` of shape `(1,)`, *optional*, returned when `future_values` is provided): The loss of the TimesFM model. Nmean_predictionsfull_predictionsloss) rrrrr(r r!r"r)r*floatr#r$r%r'r'6sI-1ellT)0,0ellT)0(,D%,,  %,r$r'c0eZdZdZdeffd ZddZxZS) TimesFmMLPzPax MLP in pytorch.configct||j}|j}t j |||_t j |||_t j|d|_ y)Ngư>)normalized_shapeeps) super__init__ hidden_sizeintermediate_sizennLinear gate_proj down_proj LayerNorm layer_norm)selfr.r4r5 __class__s r%r3zTimesFmMLP.__init__Js] (( "44;0AB#4kB,, Nr$c|j|}|j|}tj|}|j |}||d|dddddfz z}||zS)N?)r;r8Frelur9)r<xpaddingsgate_inpgateoutputss r%forwardzTimesFmMLP.forwardSsc??1%~~h'vvd|..&  x1d ';!; .D.$00,,1"&((5+?% BV+V"WZ]^lop^prsZt"t   EIIell>&W[rZr&rs s r$cN| | td|Jtj|tj|jj j d}n'|jdk7rtd|j|jg|jd|jjdddz}tjtj|tj|gd}tj|ddd|jdzf}|S) aGenerates a Tensor of sinusoids with different frequencies. Args: seq_length: an optional Python int defining the output sequence length. if the `position` argument is specified. position: [B, seq_length], optional position for each token in the sequence, only required when the sequence is packed. Returns: [B, seqlen, D] if `position` is specified, else [1, seqlen, D] z.Either position or seq_length must be providedrbdevicerrz*position must be 2-dimensional, got shape rdim) ValueErrorr rkrlr`rq unsqueezendimshapeviewcatsincosr@padre)r< seq_lengthposition scaled_timesignals r%rGz"TimesFmPositionalEmbedding.forwards   2MN N  ||JemmDL_L_LfLfgqqrstH ]]a I(..IYZ[ [#hmm7X^^7Q7$:M:M:R:RSTVWY[:\\ EIIk2EIIk4JKQRSv1a)<)@P@PQ r$queryreturnctj|jjdt j |j z }||dddddfzS)Ng^$3eG?)r@softplusrmulrfsqrtr)r<rrs r% _scale_queryzTimesFmAttention._scale_querysJ 4<<(,,[499T]];S-STuT4q0111r$N hidden_statesattention_maskkwargsc |jdd}g|d|j}|j|j|j dd}|j |}|j |j|j dd}|j|j|j dd}tj|jjt} | |||||f|jsdn |jdd|\} } | jg|dj!} |j#| } | | fS)Nrrrrr?)dropoutr)rxrrry transposerrrr get_interfacer._attn_implementationrtrainingrreshape contiguousr) r<rrr input_shape hidden_shape query_states key_states value_statesattention_interface attn_output attn_weightss r%rGzTimesFmAttention.forwardsW $))#2. 88b8$--8 {{=166|DNNqRST ((6 [[/44\BLLQPQR {{=166|DNNqRST (?(M(M KK , ,.L) %8      % $}}C$2H2H %  % ! \*k));;;;FFH kk+. L((r$rH)rrrrrintr3r r!rr rtuplerGrJrKs@r%rrsvR}RR(2%,,25<<2/3)||) t+)-. ) u||U\\D00 1 )r$rceZdZdZdedeffd Zdejdejdejdejfd Z xZ S) TimesFmDecoderLayerzTransformer layer.r.rct|t|||_t ||_t |j|j|_ y)N)r)r1) r2r3r self_attnr-mlpr\r4 rms_norm_epsinput_layernormrs r%r3zTimesFmDecoderLayer.__init__sC )&IFf%-f.@.@fFYFYZr$rrrCrc |}|j|}|j||\}}||z}|j||}|S)N)rr)rC)rrr)r<rrrCrrZ_s r%rGzTimesFmDecoderLayer.forwards]!,,]; >>')*  q!=0 B r$) rrrrrrr3r r!rGrJrKs@r%rrsW[}[[|| ,,   r$rcreZdZUeed<dZdgZdZdZdZ e e dZ e jfdZxZS) TimesFmPreTrainedModelr.timesfmr past_values)timeT)r attentionsc "t||t|tr t j |j yt|tr|jdz}|j|j}}tjt|t|z t|dz dz }t j|j |t#j$t#j&|t"j(| zzyy)Nrrra)r2 _init_weights isinstancerinitones_rr^rerdrcrfrgr+rhcopy_r`r rjrkrl)r<modulermrdrcrnr=s r%rz$TimesFmPreTrainedModel._init_weightss f% f. / JJv~~ &  : ;#22a7N+1+?+?AUAU=M&*hhu]/CeMFZ/Z&[^a"A_' # JJ%%))ELLu}}MQhPhhij  S>S S LLL$6$6|?R?R S  !% 7 7 U(3+=> ><"@bI ))-8  ))Lb9!< ;; / /'' (9(9!(<=GllG9{/@/@/C#CKG44_gNG 7 "K d#u $ 88*)//2%% '' 9 [[!@4;;#@#@A E!-( M +a(  r$rNrrbrqrc|jrtj|jntj|j}|&|j |j dddd}||z}|rbtjtj||f|||zd}|j dd||}|tj||}|S|}|S)a Creates 4D attention mask and combines causal and padding masks if needed. Args: attention_mask: Optional tensor of shape (batch_size, seq_length) containing padding mask sequence_length: Length of the sequence dtype: Data type of the mask device: Device of the mask is_causal: Whether to apply causal masking Returns: 4D attention mask of shape (batch_size, 1, seq_length, seq_length) rrrrrp)diagonal) is_floating_pointr finforiinforyrxtriuonesminimum)rrrbrqr min_value causal_masks r%rz'TimesFmModel._prepare_4d_attention_masks,/4.E.EEKK&**5;;W\K]KaKa   %+001E1Ea1H!QPRSN+i7N ** O_=USYZ]ffK&**1a/RK)!&~{!K"-r$paddingcFdtjfd}tjd|z d}||}tj|jd}|||ddf}|||ddf}d|z }tj|d} tj | d } tj||zd} | | z } || j d z |z} tj| dzd| z } tj | d } tj| }| |fS) aCalculates mean and standard deviation of `inputs` across axis 1. It excludes values where `padding` is 1. Args: inputs: A PyTorch tensor of shape [b, n, p]. padding: A PyTorch tensor of shape [b, n, p] with values 0 or 1. Returns: A tuple containing the mean and standard deviation. We return the statistics of the first patch with more than three non-padded values. arrc(tj|dk\jtjd}|dk\jtjj d}tj |dk(|j ddz |S)Nrrrsr)r argmaxtoint32sumrrx)rindicesrow_sums r%_get_patch_indexz?TimesFmModel._timesfm_masked_mean_std.._get_patch_indexskllC1H==#=1EGaxmmEKK0444;G;;w!|SYYq\A-=wG Gr$rrrsrNr?rrrr)r r!rrkrxrrvr)rr rpad_sum patch_indicesbidxsrr}masknum_valid_elements masked_sum masked_meanmasked_centered_arr masked_var masked_stds r%rz%TimesFmModel._timesfm_masked_mean_stds  H%,, H ))AKQ/(1  V\\!_-UM1,-e]A-.3w#YYt3"[[);EYYsTzq1  #55  #[%:%:2%>>$FYY2A51=@RR [[5 ZZ + J&&r$rseqc|j\}}}|dk(}|jtjj d}d||j d<tj ||jjdddj|d|}||ddddfz |z}|jd|} | S)zShifts rows of seq based on the first 0 in each row of the mask. Args: mask: mask tensor of shape [B, N] seq: seq tensor of shape [B, N, P] Returns: The shifted sequence. rrrsrr)rqN) rxrr rranyrkrqryexpandgather) rr! batch_sizenum_seq feature_dimnew_maskr idx_range shifted_idx shifted_seqs r%rz&TimesFmModel._timesfm_shift_padded_seqs,/99( G[%)QY++ekk*11a18)+!$$%LL<AA!RKRRS]_acno !71dD=#99WD jjK0 r$)T)rrrrr3r r!rrrrr LongTensorr r rrG staticmethodrrbrqboolrrrrJrKs@r%rr!s}&$ll$27,,$ u||U5<<#=>> ?$  F \\F #--F ll F +, F  F  F P  + t+++{{+ +  +   ++Z,',' ,'QVW\WcWcejeqeqWqQr,','\ 5<<ELLr$rcReZdZdZdeffd Z ddeejdee dzde dzde ejd ffd Z d ejd e ejejfdejfd Z dejdejdejfdZ ee ddeejdeeje zdzde dzdejdzde dzdededeedefdZedejde deejfdZxZS)TimesFmModelForPredictionz/TimesFM model for quantile and mean prediction.r.cJt||||_|j|_|j |_t||_t|j|j dt|jzz|j|_|jy)Nrr)r2r3r.context_length context_lenhorizon_length horizon_lenrdecoderrMr4len quantilesr5horizon_ff_layerr)r<r.r=s r%r3z"TimesFmModelForPrediction.__init__s   !00!00#F+ !5))--S9I9I5J1JK00!  r$Nrrr4r.cX| |j}gg}}|D]}|jd}tj||jz|j |j }||kr||z } tjtj| |j |j |gd}tjtj| |j |j |gd}n||kDr|| d}|||jz d}|j||j|tj|dtj|df} |E| tj|dt|tjjddfz} | S)aPad/truncate input time series to `context_len` and build a padding mask. Args: inputs: A list of 1d Tensors. Each Tensor is the context time series of a single forecast task. freq: Optional list of frequencies (returned as a tensor when provided). context_len: Optional context length override (defaults to `self.context_len`). Returns: Tuple of (padded_inputs, padding_mask) and optionally a freq tensor. Nrrprsrarrr)r4rxr zerosr6rbrqrzrappendstackrr8rr) r<rrr4input_ts input_paddingts input_lenr  num_front_padresults r% _preprocessz%TimesFmModelForPrediction._preprocess%s  **K"$b- *B Ikk)d.>.>">bhhWYW`W`aG;& +i 7 YY MRTR[R[ \^`aghi))UZZ RXXV]VdVd%egn$ouvw[( &!K$2B2B$B"C"EF OOB    ) *++hA. Mq0QR  u||D3v;,?u{{S[[\^`abddF r$ model_outputrc|j|}|j\}}}|j|||jjt |jj dz}|\}}||dddddfz|dddddfzS)z*Postprocess output of stacked transformer.rN)r:rxryr.r5r8r9) r<rFr output_tsbnrrrs r%_postprocess_outputz-TimesFmModelForPrediction._postprocess_outputJs )),7 //1aNN1a)C)CSI^I^E_bcEcd  E5D$!4551dD$;N8OOOr$ predictionstargetsc*g}t|jjD]M\}}||d|fz }tj|dz |z||z}|j |j Otj|j S)N.r) enumerater.r9r rhr=meanr>)r<rLrMlossesiqerrorsr*s r%_quantile_lossz(TimesFmModelForPrediction._quantile_lossYsdkk334 'DAq{3622F99a!ev-q6z:D MM$))+ & '{{6"''))r$r window_size future_valuesforecast_context_lenreturn_forecast_on_contexttruncate_negativerc > | |j} n|} |dj} |D cgc]} | | d } } tjtj| D cgc]} tj| c} } |Yg}g}t | D]A\}} |j |j| ||*|j ||gdzC|} ||}|$tjddgt| z}|j| |\}}}|j| }|j| }|j| }|}|jd}g}|jd|jd|jzk7r8td|jdd|jdd|j|j j"}|j|zdz |z}t%|D]/}|ddd|jdf}|dd| df}|dd| df}|j&d|||d |}|j)|j*|j,|j.f}|rl|dk(rg|dddd d|j j0ddf}|j3|j5dd |j5d }|j7||ddd d|df}|ddd d|ddf}|j7|tj8||gd }2|rHtj8|d ddd||j j0z |jzddf}n-tj8|d ddd|jddf}|dddddf}|*|dddd f|dddd fz}|dddd f|dddd fz}|rQ| dk\} tj:| |j=d|}tj:| |j=d|}d}!|9t?j@||}"|jC|ddddddf|}#|"|#z}!tEj*|jF|jH|||!Scc} wcc} w)a past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Past values of the time series that serves as input to the model. freq (`torch.LongTensor` of shape `(batch_size,)`): Frequency indices for the time series data. window_size (`int`, *optional*): Window size of trend + residual decomposition. If None then we do not do decomposition. future_values (`torch.Tensor`, *optional*): Optional future time series values to be used for loss computation. forecast_context_len (`int`, *optional*): Optional max context length. return_forecast_on_context (`bool`, *optional*): True to return the forecast on the context when available, i.e. after the first input patch. truncate_negative (`bool`, *optional*): Truncate to only non-negative values if any of the contexts have non-negative values, otherwise do nothing. Example: ```python >>> from transformers import TimesFmModelForPrediction >>> model = TimesFmModelForPrediction.from_pretrained("google/timesfm-2.0-500m-pytorch") >>> forecast_input = [torch.linspace(0, 20, 100).sin(), torch.linspace(0, 20, 200).sin(), torch.linspace(0, 20, 400).sin()] >>> frequency_input = torch.tensor([0, 1, 2], dtype=torch.long) >>> # Generate >>> with torch.no_grad(): >>> outputs = model(past_values=forecast_input, freq=frequency_input, return_dict=True) >>> point_forecast_conv = outputs.mean_predictions >>> quantile_forecast_conv = outputs.full_predictions ``` Nrrz6No frequency provided via `freq`. Default to high (0).rz=Length of paddings must match length of input + horizon_len: z != z + )rrrrrr)axis.r)rrrr(r)r*r#)%r4rqr rr>rOextend_timesfm_moving_averageloggerinfor8rErrxr6rur.r5rr7rKrrrrrsizer= concatenater clamp_minr@mse_lossrUr'rr)$r<rrrVrWrXrYrZr fcontext_lenrqrArinp_min new_inputs new_freqsrRr?r@inp_freq final_outr4 full_outputsoutput_patch_lennum_decode_patches step_indexcurrent_paddingdecoder_output fprop_outputs new_full_tsnew_ts mean_outputsrr*rd quantile_losss$ r%rGz!TimesFmModelForPrediction.forwardas^ '++L/LQ&&/:;"l]^$;;))EKK(H22(HIJ  "JI"6* 42!!$">">r;"OP#$$d1gY]3 4 F  < KKP Q3V$D,0,<,M,8DLL-$$1- -N !4400##^%9%9:M *jAo+Ass4Ndkk6N6N4NPQ,QR )11+2B2B12Er;K[K[\]K^_ ##K0"1b*;+;*;Q#>?F'2/@0@/@!(CDK    ,))9f*=BGI/ H2 & ,,\BPkDKK$<$<H // Q12X0F VMm+D),>>%00(66))   g<(Hs RR rc4tj||dz dfdd}tj||j|j |z }tj |jddd|jdddj}|||z gS)zCCalculates the moving average using PyTorch's convolution function.rrconstantrprr) r@r}r rrbrqconv1drysqueeze)rrV arr_paddedkernel smoothed_arrs r%r^z1TimesFmModelForPrediction._timesfm_moving_averagesUU3q! 4j!D KsyyL{Zxx 1b 96;;q!R;PQYY[ cL011r$r)NNNNFF)rrrrrr3rr r!rrrErKrUrrr/r r r'rGr.listr^rJrKs@r%r1r1s9}(lp#u||,#4rsO .! &B/F&RR75/U0   H %  &O &  &  - -  -,!299!, \ ++\9)ryy9)x"))@_>j)jjZm2 6m2` Rr$