Ë
    ÜÓ^j—I ã                   óÀ  — d Z ddlZddlmZ ddlmZ ddlZddlmZ ddlm	Z
 ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZ ddlmZ ddlmZmZmZ ddlmZmZmZm Z  ddl!m"Z"  e jF                  e$«      Z%	 	 dodejL                  dejN                  dejN                  dejN                  dejN                  dz  de(dz  de(dee   fd„Z) G d„ dejL                  «      Z* G d„ dejL                  «      Z+	 	 	 dpdejN                  d e(d!e,dz  d"e-d#e.f
d$„Z/	 	 dqdejN                  d%e,e.z  d!e,dz  d#e.fd&„Z0 G d'„ d(ejL                  «      Z1 G d)„ d*ejL                  «      Z2 G d+„ d,ejL                  «      Z3e G d-„ d.e«      «       Z4 G d/„ d0ejL                  «      Z5 G d1„ d2ejL                  «      Z6 G d3„ d4e4«      Z7 ed5¬6«      e G d7„ d8e«      «       «       Z8 ed9¬6«      e G d:„ d;e«      «       «       Z9 ed<¬6«      e G d=„ d>e«      «       «       Z: ed?¬6«      e G d@„ dAe«      «       «       Z; edB¬6«      e G dC„ dDe«      «       «       Z< edE¬6«      e G dF„ dGe«      «       «       Z=dHej|                  j~                  dIejN                  dJejN                  fdK„Z@drdLejN                  dMejN                  dz  dJejN                  fdN„ZA G dO„ dPejL                  «      ZB G dQ„ dRejL                  «      ZC G dS„ dTejL                  «      ZD G dU„ dVejL                  «      ZEe G dW„ dXe4«      «       ZF G dY„ dZejL                  «      ZG ed[¬6«       G d\„ d]e4«      «       ZH G d^„ d_ejL                  «      ZI ed`¬6«       G da„ dbe4«      «       ZJ edc¬6«       G dd„ deejL                  «      «       ZK edf¬6«       G dg„ dhe4«      «       ZL G di„ djejL                  «      ZM edk¬6«       G dl„ dme4«      «       ZNg dn¢ZOy)szPyTorch PatchTST model.é    N)ÚCallable)Ú	dataclass)Únné   )Úinitialization)ÚACT2CLS)Úis_deepspeed_zero3_enabled)ÚFlashAttentionKwargs)ÚBaseModelOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚNegativeBinomialOutputÚNormalOutputÚStudentTOutput)ÚModelOutputÚTransformersKwargsÚauto_docstringÚloggingé   )ÚPatchTSTConfigÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 óœ  — |€|j                  d«      dz  }t        j                  ||j                  dd«      «      |z  }|�||z   }t        j
                  j                  |d¬«      }t        j
                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )Néÿÿÿÿç      à¿é   r   ©Údim)ÚpÚtrainingr   )
ÚsizeÚtorchÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr   r'   Ú
contiguous)
r   r   r   r   r   r   r   r   Úattn_weightsÚattn_outputs
             úy/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/patchtst/modeling_patchtst.pyÚeager_attention_forwardr2   '   s»   € ð €Ø—*‘*˜R“. DÑ(ˆô —<‘<  s§}¡}°Q¸Ó':Ó;¸gÑE€LàÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2Ð(Ó>€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$ó    c                   ó(  ‡ — e Zd ZdZ	 	 	 	 	 ddedededededed	edz  fˆ fd
„Z	 	 	 dde	j                  de	j                  dz  de	j                  dz  dedz  dee   dee	j                  e	j                  dz  ee	j                     dz  f   fd„Zˆ xZS )ÚPatchTSTAttentionz=Multi-headed attention from 'Attention Is All You Need' paperNÚ	embed_dimÚ	num_headsr   Ú
is_decoderÚbiasÚ	is_causalÚconfigc                 ó
  •— t         ‰| �  «        || _        || _        || _        ||z  | _        || _        | j
                  |z  | j                  k7  rt        d| j                  › d|› d�«      ‚| j
                  dz  | _        || _	        || _
        t        j                  |||¬«      | _        t        j                  |||¬«      | _        t        j                  |||¬«      | _        t        j                  |||¬«      | _        y )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: ú).r"   ©r9   )ÚsuperÚ__init__r6   r7   r   Úhead_dimr;   Ú
ValueErrorr   r8   r:   r   ÚLinearÚk_projÚv_projÚq_projÚout_proj)	Úselfr6   r7   r   r8   r9   r:   r;   Ú	__class__s	           €r1   r@   zPatchTSTAttention.__init__G   sä   ø€ ô 	‰ÑÔØ"ˆŒØ"ˆŒØˆŒØ! YÑ.ˆŒØˆŒà�M‰M˜IÑ%¨$¯.©.Ò8ÜØMÈdÏnÉnÐM]Ø$ Y K¨rð3óð ð —}‘} dÑ*ˆŒØ$ˆŒØ"ˆŒä—i‘i 	¨9¸4Ô@ˆŒÜ—i‘i 	¨9¸4Ô@ˆŒÜ—i‘i 	¨9¸4Ô@ˆŒÜŸ	™	 )¨Y¸TÔBˆ�r3   Úhidden_statesÚkey_value_statesr   Úoutput_attentionsr   Úreturnc                 óò  — |du}|j                   dd }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }	|r|n|}
g |
j                   dd ¢d‘| j                  ‘­}| j                  |
«      j                  |«      j	                  dd«      }| j                  |
«      j                  |«      j	                  dd«      }t        j                  | j                  j                  t        «      } || |	|||f| j                  sdn| j                  | j                  |dœ|¤Ž\  }} |j                  g |¢d‘­Ž j!                  «       }| j#                  |«      }||dfS )z#Input shape: Batch x Time x ChannelNr!   r   r#   ç        )r   r   rL   )ÚshaperA   rF   Úviewr+   rD   rE   r   Úget_interfacer;   Ú_attn_implementationr2   r'   r   r   Úreshaper.   rG   )rH   rJ   rK   r   rL   r   Úis_cross_attentionÚinput_shapeÚhidden_shapeÚquery_statesÚcurrent_statesÚkv_shapeÚ
key_statesÚvalue_statesÚattention_interfacer0   r/   s                    r1   ÚforwardzPatchTSTAttention.forwardf   sŠ  € ð .°TÐ9Ðð $×)Ñ)¨#¨2Ð.ˆà8˜Ð8 bÐ8¨$¯-©-Ñ8ˆð —{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆá-?Ñ)À]ˆØB�^×)Ñ)¨#¨2Ð.ÐB°ÐB°D·M±MÑBˆØ—[‘[ Ó0×5Ñ5°hÓ?×IÑIÈ!ÈQÓOˆ
Ø—{‘{ >Ó2×7Ñ7¸ÓA×KÑKÈAÈqÓQˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$·,±,Ø—L‘LØ/ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—m‘m KÓ0ˆà˜L¨$Ð.Ð.r3   )rO   FTFN)NNF)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚintÚfloatÚboolr   r@   r)   ÚTensorr   r
   Útupler^   Ú__classcell__©rI   s   @r1   r5   r5   D   sÿ   ø„ ÙGð Ø ØØØ(,ñCàðCð ðCð ð	Cð
 ðCð ðCð ðCð  Ñ%õCðD 15Ø.2Ø).ñ0/à—|‘|ð0/ð  Ÿ,™,¨Ñ-ð0/ð Ÿ™ tÑ+ð	0/ð
   $™;ð0/ð Ð-Ñ.ð0/ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷0/r3   r5   c                   óH   ‡ — e Zd ZdZdefˆ fd„Zdej                  fd„Zˆ xZ	S )ÚPatchTSTBatchNormzP
    Compute batch normalization over the sequence length (time) dimension.
    r;   c                 ó‚   •— t         ‰| �  «        t        j                  |j                  |j
                  ¬«      | _        y )N©Úeps)r?   r@   r   ÚBatchNorm1dÚd_modelÚnorm_epsÚ	batchnorm©rH   r;   rI   s     €r1   r@   zPatchTSTBatchNorm.__init__ž   s(   ø€ Ü‰ÑÔÜŸ™¨¯©¸F¿O¹OÔLˆ�r3   Úinputsc                 ól   — |j                  dd«      }| j                  |«      }|j                  dd«      S )a  
        Parameters:
            inputs (`torch.Tensor` of shape `(batch_size, sequence_length, d_model)`):
                input for Batch norm calculation
        Returns:
            `torch.Tensor` of shape `(batch_size, sequence_length, d_model)`
        r   r#   )r+   rr   )rH   rt   Úoutputs      r1   r^   zPatchTSTBatchNorm.forward¢   s7   € ð ×!Ñ! ! QÓ'ˆØ—‘ Ó'ˆØ×Ñ  1Ó%Ð%r3   ©
r_   r`   ra   rb   r   r@   r)   rf   r^   rh   ri   s   @r1   rk   rk   ™   s&   ø„ ñðM˜~õ Mð
&˜eŸl™l÷ 
&r3   rk   rt   Ú
mask_ratioÚunmasked_channel_indicesÚchannel_consistent_maskingÚ
mask_valuec                 ó°  — |dk  s|dk\  rt        d|› d�«      ‚| j                  \  }}}}| j                  }	t        |d|z
  z  «      }
|r-t	        j
                  |d||	¬«      }|j                  d|d«      }nt	        j
                  ||||	¬«      }t	        j                  ||||	¬«      }d|dd…dd…d|
…f<   t	        j                  |d¬«      }t	        j                  |d¬«      }t	        j                  |d|¬	«      }|j                  d«      j                  ddd|«      }|�d|dd…|dd…dd…f<   | j                  |j                  «       |«      }||d
   fS )aÆ  random_masking: Mask the input considering the control variables.

    Args:
        inputs (`torch.Tensor` of shape `(batch_size, num_channels, sequence_length, num_features)`):
            The input tensor to mask.
        mask_ratio (`float`):
            Masking ratio applied to mask the input data during random pretraining. It is the number between 0 and 1.
        unmasked_channel_indices (list, *optional*):
            Indices of channels that will not be masked.
        channel_consistent_masking (bool, *optional*, defaults to `False`):
            When true, masking will be same across all channels of a timeseries. Otherwise, masking positions will vary
            across channels.
        mask_value (int, *optional*, defaults to 0):
            Define the value of masked patches for pretraining.

    Returns:
        `tuple(torch.Tensor)`: inputs_mask, masked input, same shape as input Tensor and mask tensor of shape [bs x c x
        n]
    r   r   zMask ratio z has to be between 0 and 1.©ÚdeviceNr!   r$   )r%   Úindex©.r   )rB   rP   r~   rc   r)   ÚrandÚrepeatÚonesÚargsortÚgatherÚ	unsqueezeÚmasked_fillre   )rt   rx   ry   rz   r{   Ú
batch_sizeÚnum_channelsÚsequence_lengthÚnum_featuresr~   Úlen_keepÚnoiseÚmaskÚids_shuffleÚids_restoreÚinputs_masks                   r1   Úrandom_maskingr’   ¯   sQ  € ð4 �A‚~˜ qšÜ˜; z lÐ2MÐNÓOÐOà>D¿l¹lÑ;€J�˜o¨|Ø�]‰]€Fä�? a¨*¡nÑ5Ó6€Há!Ü—
‘
˜: q¨/À&ÔIˆØ—‘˜Q ¨aÓ0‰ô —
‘
˜: |°_ÈVÔTˆô �:‰:�j ,°ÈÔO€DØ€DŠŠAˆy�ˆyˆÑô —-‘- ¨2Ô.€KÜ—-‘- °Ô4€Kä�<‰<˜ "¨KÔ8€DØ�>‰>˜"Ó×$Ñ$ Q¨¨1¨lÓ;€DØÐ+Ø23ˆŠQÐ(ª!ªQÐ.Ñ/à×$Ñ$ T§Y¡Y£[°*Ó=€KØ˜˜V™Ð$Ð$r3   Únum_forecast_mask_patchesc                 óP  — t        |t        «      r|g}|D �cg c]  }d‘Œ }}| j                  \  }}}}	t        j                  |||| j
                  ¬«      }
g }d}t        |«      }t        ||«      D ]H  \  }}|dk  s||k\  rt        d|› d�«      ‚t        ||z  |z  «      }|j                  |||g«       ||z  }ŒJ t        |d„ ¬«      }||k  r|d   d   ||z
  z   |d   d<   n||kD  r|d	   d   ||z
  z   |d	   d<   d}|D ]  \  }}}||z   }d|
||…d
d
…| d
…f<   |}Œ t        j                  |
j                  d   «      }|
|   }
|
j                  d	«      j                  ddd|	«      }
|�d|
d
d
…|d
d
…d
d
…f<   | j                  |
j                  «       |«      }||
d   fS c c}w )a¡  Forecast masking that masks the last K patches where K is from the num_forecast_mask_patches.
    If num_forecast_mask_patches is a list, samples in the batch will be randomly masked by numbers defined in the list.

    Parameters:
        inputs (`torch.Tensor`):
            Input of shape `(bs, num_channels, num_patch, patch_length)`
        num_forecast_mask_patches (`list`):
            Number of patches to be masked at the end of each batch sample. e.g. 4 or [3, 5].
        unmasked_channel_indices (`list`, *optional*):
            Indices of channels that are not masked.
        mask_value (`int`, *optional*, defaults to 0):
            Values in the masked patches will be filled by `mask_value`.

    Returns:
        `tuple(torch.Tensor)`: inputs_mask, masked input, same shape as inputs Tensor and Mask tensor of shape `(bs,
        num_channels , num_patch)` or `(bs, tsg1, tsg2, num_channels, num_patch)`
    r   r}   r   znum_forecast_mask_patches z6 should be greater than 0 and less than total patches.c                 ó   — | d   S )Nr#   © )Úxs    r1   ú<lambda>z"forecast_masking.<locals>.<lambda>  s
   € ¨!¨A©$€ r3   )r   r#   r!   Nr€   )Ú
isinstancerc   rP   r)   Úzerosr~   ÚsumÚziprB   ÚappendÚsortedÚrandpermr†   r‚   r‡   re   )rt   r“   ry   r{   Ú_Úforecast_mask_ratiosrˆ   r‰   rŠ   r‹   rŽ   Út_listÚtotal_lengthÚtotal_ratioÚpatch_lengthÚratioÚtemp_lenÚbatch1Ú	patch_lenÚbatch2Úpermr‘   s                         r1   Úforecast_maskingr¬   é   s  € ô0 Ð+¬SÔ1Ø%>Ð$?Ð!Ø'@ÖA !šAÐAÐÐAà>D¿l¹lÑ;€J�˜o¨|Ü�;‰;�z <°ÈÏÉÔW€Dà€FØ€LÜÐ*Ó+€Kä"Ð#<Ð>RÓSò !Ñˆ�eØ˜1Ò °Ò ?ÜØ,¨\¨NÐ:pÐqóð ô �z EÑ)¨KÑ7Ó8ˆØ�‰�| U¨HÐ5Ô6Ø˜Ñ ‰ð!ô �F¡Ô/€Fà�jÒ Ø˜a‘y ‘| z°LÑ'@ÑAˆˆq‰	�!ŠØ	˜
Ò	"Ø˜r™
 1™¨¸
Ñ)BÑCˆˆr‰
�1‰à€FØ"(ò Ñˆ	�1�hØ˜(Ñ"ˆØ./ˆˆV�Fˆ]šA 	˜z™{Ð*Ñ+Ø‰ðô
 �>‰>˜$Ÿ*™* Q™-Ó(€DØ�‰:€Dà�>‰>˜"Ó×$Ñ$ Q¨¨1¨lÓ;€DØÐ+Ø23ˆŠQÐ(ª!ªQÐ.Ñ/à×$Ñ$ T§Y¡Y£[°*Ó=€KØ˜˜V™Ð$Ð$ùòO Bs   ˜	F#c                   óH   ‡ — e Zd ZdZdefˆ fd„Zdej                  fd„Zˆ xZ	S )ÚPatchTSTPatchifyz³
    A class to patchify the time series sequence into different patches

    Returns:
        `torch.Tensor` of shape `(batch_size, num_channels, num_patches, patch_length)`
    r;   c                 ó  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        | j                  | j                  k  r&t        d| j                  › d| j                  › d�«      ‚t        | j                  | j                  «      | j                  z
  | j
                  z  dz   | _        | j                  | j
                  | j                  dz
  z  z   }| j                  |z
  | _	        y )NzSequence length (z+) has to be greater than the patch length (ú)r   )
r?   r@   Úcontext_lengthrŠ   r¥   Úpatch_striderB   ÚmaxÚnum_patchesÚsequence_start)rH   r;   Únew_sequence_lengthrI   s      €r1   r@   zPatchTSTPatchify.__init__5  sò   ø€ Ü‰ÑÔà%×4Ñ4ˆÔØ"×/Ñ/ˆÔØ"×/Ñ/ˆÔà×Ñ 4×#4Ñ#4Ò4ÜØ# D×$8Ñ$8Ð#9Ð9dÐei×evÑevÐdwÐwxÐyóð ô
   × 4Ñ 4°d×6GÑ6GÓHÈ4×K\ÑK\Ñ\Ðae×arÑarÑrÐuvÑvˆÔØ"×/Ñ/°$×2CÑ2CÀt×GWÑGWÐZ[ÑG[Ñ2\Ñ\ÐØ"×2Ñ2Ð5HÑHˆÕr3   Úpast_valuesc                 ó:  — |j                   d   }|| j                  k7  rt        d|› d| j                  › d�«      ‚|dd…| j                  d…dd…f   }|j	                  d| j
                  | j                  ¬«      }|j                  dd«      j                  «       }|S )a!  
        Parameters:
            past_values (`torch.Tensor` of shape `(batch_size, sequence_length, num_channels)`, *required*):
                Input for patchification

        Returns:
            `torch.Tensor` of shape `(batch_size, num_channels, num_patches, patch_length)`
        éþÿÿÿzInput sequence length (z%) doesn't match model configuration (r=   N)Ú	dimensionr(   Ústepéýÿÿÿ)	rP   rŠ   rB   rµ   Úunfoldr¥   r²   r+   r.   )rH   r·   rŠ   rv   s       r1   r^   zPatchTSTPatchify.forwardF  s¨   € ð &×+Ñ+¨BÑ/ˆØ˜d×2Ñ2Ò2ÜØ)¨/Ð):Ð:_Ð`d×`tÑ`tÐ_uÐuwÐxóð ð šQ × 3Ñ 3Ñ 5²qÐ8Ñ9ˆà—‘¨°$×2CÑ2CÈ$×J[ÑJ[�Ó\ˆà×!Ñ! " bÓ)×4Ñ4Ó6ˆØˆr3   rw   ri   s   @r1   r®   r®   -  s&   ø„ ñðI˜~õ Ið" 5§<¡<÷ r3   r®   c                   óH   ‡ — e Zd ZdZdefˆ fd„Zdej                  fd„Zˆ xZ	S )ÚPatchTSTMaskinga�  
    Class to perform random or forecast masking.

    Parameters:
        config (`PatchTSTConfig`): model config
    Returns:
        x_mask (`torch.Tensor` of shape `(batch_size, num_channels, num_patches, patch_length)`)
            Masked patched input
        mask (`torch.Tensor` of shape `(batch_size, num_channels, num_patches)`)
            Bool tensor indicating True on masked points
    r;   c                 ó<  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        |j                  | _        | j                  �t        | j                  «      | _        y y ©N)	r?   r@   Úrandom_mask_ratiorz   Ú	mask_typer“   ry   r{   rž   rs   s     €r1   r@   zPatchTSTMasking.__init__j  s„   ø€ Ü‰ÑÔØ!'×!9Ñ!9ˆÔØ*0×*KÑ*KˆÔ'Ø×)Ñ)ˆŒØ)/×)IÑ)IˆÔ&Ø(.×(GÑ(GˆÔ%Ø ×+Ñ+ˆŒØ×(Ñ(Ð4Ü,2°4×3PÑ3PÓ,QˆDÕ)ð 5r3   Úpatch_inputc                 ór  — | j                   dk(  r<t        || j                  | j                  | j                  | j
                  ¬«      \  }}nY| j                   dk(  r1t        || j                  | j                  | j
                  ¬«      \  }}nt        d| j                   › d�«      ‚|j                  «       }||fS )aä  
        Parameters:
            patch_input (`torch.Tensor` of shape `(batch_size, num_channels, num_patches, patch_length)`, *required*):
                Patch input

        Return:
            masked_input (`torch.Tensor` of shape `(batch_size, num_channels, num_patches, patch_length)`)
                Masked patched input
            mask (`torch.Tensor` of shape `(batch_size, num_channels, num_patches)`)
                Bool tensor indicating True on masked points

        Úrandom)rt   rx   ry   rz   r{   Úforecast)rt   r“   ry   r{   zInvalid mask type ú.)
rÃ   r’   rÂ   ry   rz   r{   r¬   r“   rB   re   )rH   rÄ   Úmasked_inputrŽ   s       r1   r^   zPatchTSTMasking.forwardu  s±   € ð �>‰>˜XÒ%Ü!/Ø"Ø×1Ñ1Ø)-×)FÑ)FØ+/×+JÑ+JØŸ?™?ô"ÑˆL™$ð �^‰^˜zÒ)Ü!1Ø"Ø*.×*HÑ*HØ)-×)FÑ)FØŸ?™?ô	"ÑˆL™$ô Ð1°$·.±.Ð1AÀÐCÓDÐDð �y‰y‹{ˆØ˜TÐ!Ð!r3   rw   ri   s   @r1   r¿   r¿   ]  s&   ø„ ñ
ð	R˜~õ 	Rð!" 5§<¡<÷ !"r3   r¿   c                   óT   ‡ — e Zd ZdZdefˆ fd„Zddej                  dedz  fd„Z	ˆ xZ
S )	ÚPatchTSTEncoderLayerz 
    PatchTST encoder layer
    r;   c           
      ó  •— t         ‰| �  «        |j                  | _        t        |j                  |j
                  |j                  |¬«      | _        |j                  dkD  rt        j                  |j                  «      nt        j                  «       | _        |j                  dk(  rt        |«      | _        nX|j                  dk(  r1t        j                   |j                  |j"                  ¬«      | _        nt%        |j                  › d�«      ‚| j                  r¿|j                  dkD  rt        j                  |j                  «      nt        j                  «       | _        |j                  dk(  rt        |«      | _        nX|j                  dk(  r1t        j                   |j                  |j"                  ¬«      | _        nt%        |j                  › d�«      ‚t        j*                  t        j,                  |j                  |j.                  |j0                  ¬«      t3        |j4                     «       |j6                  dkD  rt        j                  |j6                  «      nt        j                  «       t        j,                  |j.                  |j                  |j0                  ¬«      «      | _        |j                  dkD  rt        j                  |j                  «      nt        j                  «       | _        |j                  dk(  rt        |«      | _        nX|j                  dk(  r1t        j                   |j                  |j"                  ¬«      | _        nt%        |j                  › d�«      ‚|j>                  | _        y )N)r6   r7   r   r;   r   rr   Ú	layernormrm   z$ is not a supported norm layer type.r>   ) r?   r@   Úchannel_attentionr5   rp   Únum_attention_headsÚattention_dropoutÚ	self_attnÚpath_dropoutr   ÚDropoutÚIdentityÚdropout_path1Ú	norm_typerk   Únorm_sublayer1Ú	LayerNormrq   rB   Údropout_path2Únorm_sublayer2Ú
SequentialrC   Úffn_dimr9   r   Úactivation_functionÚ
ff_dropoutÚffÚdropout_path3Únorm_sublayer3Úpre_normrs   s     €r1   r@   zPatchTSTEncoderLayer.__init__ž  s‘  ø€ Ü‰ÑÔà!'×!9Ñ!9ˆÔä*Ø—n‘nØ×0Ñ0Ø×,Ñ,Øô	
ˆŒð AG×@SÑ@SÐVWÒ@WœRŸZ™Z¨×(;Ñ(;Ô<Ô]_×]hÑ]hÓ]jˆÔØ×Ñ˜{Ò*Ü"3°FÓ";ˆDÕØ×Ñ Ò,Ü"$§,¡,¨v¯~©~À6Ç?Á?Ô"SˆDÕä × 0Ñ 0Ð1Ð1UÐVÓWÐWð ×!Ò!ØDJ×DWÑDWÐZ[ÒD[¤§¡¨F×,?Ñ,?Ô!@Ôac×alÑalÓanˆDÔØ×Ñ ;Ò.Ü&7¸Ó&?�Õ#Ø×!Ñ! [Ò0Ü&(§l¡l°6·>±>ÀvÇÁÔ&W�Õ#ä  F×$4Ñ$4Ð#5Ð5YÐ!ZÓ[Ð[ô —-‘-Ü�I‰I�f—n‘n f§n¡n¸6¿;¹;ÔGÜ�F×.Ñ.Ñ/Ó1Ø-3×->Ñ->ÀÒ-BŒB�J‰J�v×(Ñ(Ô)ÌÏÉËÜ�I‰I�f—n‘n f§n¡n¸6¿;¹;ÔGó	
ˆŒð AG×@SÑ@SÐVWÒ@WœRŸZ™Z¨×(;Ñ(;Ô<Ô]_×]hÑ]hÓ]jˆÔØ×Ñ˜{Ò*Ü"3°FÓ";ˆDÕØ×Ñ Ò,Ü"$§,¡,¨v¯~©~À6Ç?Á?Ô"SˆDÕä × 0Ñ 0Ð1Ð1UÐVÓWÐWàŸ™ˆ�r3   NÚhidden_staterL   c                 óØ  — |j                   \  }}}}|j                  ||z  ||«      }| j                  r;| j                  | j	                  |«      |¬«      \  }}}	|| j                  |«      z   }n:| j                  ||¬«      \  }}}	| j	                  || j                  |«      z   «      }|j                  ||||«      }| j                  rë|j                  dd«      j                  «       }|j                  ||z  ||«      }| j                  r;| j                  | j                  |«      |¬«      \  }}
}	|| j                  |«      z   }n:| j                  ||¬«      \  }}
}	| j                  || j                  |«      z   «      }|j                  ||||«      }|j                  dd«      j                  «       }|j                  ||z  ||«      }| j                  r3|| j                  | j                  | j                  |«      «      «      z   }n2| j                  || j                  | j                  |«      «      z   «      }|j                  ||||«      }|f}|r|| j                  r|
fn|fz  }|S )a¯  
        Parameters:
            hidden_state (`torch.Tensor` of shape `(batch_size, num_channels, sequence_length, d_model)`, *required*):
                Past values of the time series
            output_attentions (`bool`, *optional*):
                Whether or not to return the output attention of all layers
        Return:
            `torch.Tensor` of shape `(batch_size, num_channels, sequence_length, d_model)`

        )rJ   rL   r#   r   )rP   rQ   râ   rÑ   r×   rÕ   rT   rÎ   r+   r.   rÚ   rÙ   rà   rß   rá   )rH   rã   rL   rˆ   Únum_input_channelsrŠ   rp   r0   r/   r    Úchannel_attn_weightsÚoutputss               r1   r^   zPatchTSTEncoderLayer.forwardÐ  s®  € ð DP×CUÑCUÑ@ˆ
Ð&¨¸ð $×(Ñ(¨Ð6HÑ)HÈ/Ð[bÓcˆà�=Š=à+/¯>©>Ø"×1Ñ1°,Ó?ÐSdð ,:ó ,Ñ(ˆK˜ qð (¨$×*<Ñ*<¸[Ó*IÑI‰Lð ,0¯>©>Ø*Ð>Oð ,:ó ,Ñ(ˆK˜ qð  ×.Ñ.¨|¸d×>PÑ>PÐQ\Ó>]Ñ/]Ó^ˆLð $×+Ñ+¨JÐ8JÈOÐ]dÓeˆð ×!Ò!à'×1Ñ1°!°QÓ7×BÑBÓDˆLà'×,Ñ,¨Z¸/Ñ-IÐK]Ð_fÓgˆLØ�}Š}à7;·~±~Ø"&×"5Ñ"5°lÓ"CÐWhð 8Fó 8Ñ4�Ð1°1ð  ,¨d×.@Ñ.@ÀÓ.MÑM‘ð 8<·~±~Ø".ÐBSð 8Fó 8Ñ4�Ð1°1ð  $×2Ñ2°<À$×BTÑBTÐU`ÓBaÑ3aÓb�ð (×/Ñ/°
¸OÐM_ÐahÓiˆLà'×1Ñ1°!°QÓ7×BÑBÓDˆLð $×(Ñ(¨Ð6HÑ)HÈ/Ð[bÓcˆØ�=Š=ð (¨$×*<Ñ*<¸T¿W¹WÀT×EXÑEXÐYeÓEfÓ=gÓ*hÑh‰Lð  ×.Ñ.¨|¸d×>PÑ>PÐQU×QXÑQXÐYeÓQfÓ>gÑ/gÓhˆLð $×+Ñ+¨JÐ8JÈOÐ]dÓeˆà�/ˆÙØ¸t×?UÒ?U˜Ð&:Ñ;Ð\hÐ[jÑjˆGàˆr3   rÁ   )r_   r`   ra   rb   r   r@   r)   rf   re   r^   rh   ri   s   @r1   rË   rË   ™  s3   ø„ ñð0(˜~õ 0(ñdQ E§L¡Lð QÀTÈDÁ[÷ Qr3   rË   c                   ó�   ‡ — e Zd ZU eed<   dZdZdZdZdZ	dZ
dZ ej                  «       dej                  fˆ fd„«       Zd
d	„Zˆ xZS )ÚPatchTSTPreTrainedModelr;   Úmodelr·   )ÚtimeFTr   c                 ó  •— t         ‰| �  |«       t        |t        «      �rZt	        | j
                  j                  | j
                  j                  «      | j
                  j                  z
  | j
                  j                  z  dz   }| j
                  j                  r&t        j                  |j                  d¬«       |dz  }|j                  | j
                  |«      }t        «       rrddl}|j                   j#                  |j$                  d¬«      5  |j$                  j'                  «       dkD  r t        j(                  |j$                  |«       ddd«       yt        j(                  |j$                  |«       yy# 1 sw Y   yxY w)z$
        Initialize weights
        r   g{®Gáz”?)Ústdr   N)Úmodifier_rank)r?   Ú_init_weightsr™   ÚPatchTSTPositionalEncodingr³   r;   r±   r¥   r²   Úuse_cls_tokenÚinitÚnormal_Ú	cls_tokenÚ_init_per	   Ú	deepspeedÚzeroÚGatheredParametersÚposition_encÚnumelÚcopy_)rH   r   r´   rù   rö   rI   s        €r1   rï   z%PatchTSTPreTrainedModel._init_weights/  s5  ø€ ô
 	‰Ñ˜fÔ%Ü�fÔ8Õ9ô �D—K‘K×.Ñ.°·±×0HÑ0HÓIÈDÏKÉK×LdÑLdÑdØ—‘×)Ñ)ñ*à,-ñ.ˆKð �{‰{×(Ò(Ü—‘˜V×-Ñ-°4Õ8Ø˜qÑ �à!Ÿ?™?¨4¯;©;¸ÓDˆLÜ)Ô+Û à—^‘^×6Ñ6°v×7JÑ7JÐZ^Ð6Ó_ñ FØ×*Ñ*×0Ñ0Ó2°QÒ6ÜŸ
™
 6×#6Ñ#6¸ÔE÷Fð Fô —
‘
˜6×.Ñ.°Õ=ð% :÷Fð Fús   Ä>E=Å=Fc                 ó4   — t        |t        «      r||_        y y rÁ   )r™   ÚPatchTSTEncoderÚgradient_checkpointing)rH   r   r   s      r1   Ú_set_gradient_checkpointingz3PatchTSTPreTrainedModel._set_gradient_checkpointingI  s   € Ü�fœÔ0Ø,1ˆFÕ)ð 1r3   )F)r_   r`   ra   r   Ú__annotations__Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnr)   Úno_gradr   ÚModulerï   rÿ   rh   ri   s   @r1   ré   ré   $  s[   ø… àÓØÐØ#€OØ ÐØ&+Ð#ØÐØ€NØÐà€U‡]�]ƒ_ð> B§I¡Iô >ó ð>÷22r3   ré   c                   óD   ‡ — e Zd Zdefˆ fd„Zdej                  fd„Zˆ xZS )ÚPatchTSTEmbeddingr;   c                 óÊ  •— t         ‰| �  «        |j                  | _        |j                  | _        | j                  r0t	        j
                  |j                  |j                  «      | _        y t	        j                  «       | _        t        |j                  «      D ]E  }| j                  j                  t	        j
                  |j                  |j                  «      «       ŒG y rÁ   )r?   r@   rå   Úshare_embeddingr   rC   r¥   rp   Úinput_embeddingÚ
ModuleListÚranger�   )rH   r;   r    rI   s      €r1   r@   zPatchTSTEmbedding.__init__O  s£   ø€ Ü‰ÑÔØ"(×";Ñ";ˆÔØ%×5Ñ5ˆÔà×ÒÜ#%§9¡9¨V×-@Ñ-@À&Ç.Á.Ó#QˆDÕ ä#%§=¡=£?ˆDÔ Ü˜6×4Ñ4Ó5ò \�Ø×$Ñ$×+Ñ+¬B¯I©I°f×6IÑ6IÈ6Ï>É>Ó,ZÕ[ñ\r3   rÄ   c                 ó`  — |j                   d   }|| j                  k7  rt        d| j                  › d|› d�«      ‚| j                  r| j	                  |«      }|S t        |«      D �cg c]$  } | j                  |   |dd…|dd…dd…f   «      ‘Œ& }}t        j                  |d¬«      }|S c c}w )a%  
        Parameters:
            patch_input (`torch.Tensor` of shape `(batch_size, num_channels, num_patches, patch_length)`, *required*):
                Patch input for embedding
        return:
            `torch.Tensor` of shape `(batch_size, num_channels, num_patches, d_model)`
        r   z&The defined number of input channels (zQ) in the config has to be the same as the number of channels in the batch input (r°   Nr$   )rP   rå   rB   r  r  r  r)   Ústack)rH   rÄ   rå   Ú
embeddingsÚis        r1   r^   zPatchTSTEmbedding.forward[  sÐ   € ð )×.Ñ.¨qÑ1ÐØ ×!8Ñ!8Ò8ÜØ8¸×9PÑ9PÐ8Qð RTØTfÐSgÐghðjóð ð ×ÒØ×-Ñ-¨kÓ:ˆJð Ðô UZÐZlÓTmÖnÈqÐ1˜$×.Ñ.¨qÑ1°+ºaÀÂAÂq¸jÑ2IÕJÐnˆJÐnÜŸ™ Z°QÔ7ˆJØÐùò os   Á')B+©	r_   r`   ra   r   r@   r)   rf   r^   rh   ri   s   @r1   r  r  N  s!   ø„ ð
\˜~õ 
\ð 5§<¡<÷ r3   r  c                   ó~   ‡ — e Zd ZdZdedefˆ fd„Zedededej                  fd„«       Z
dej                  fd„Zˆ xZS )	rð   z'
    Class for positional encoding
    r;   r´   c                 óÄ  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  r?t	        j
                  t        j                  ddd|j                  «      «      | _	        |dz  }| j                  ||«      | _        |j                  dkD  r%t	        j                  |j                  «      | _        y t	        j                  «       | _        y )Nr   r   )r?   r@   rñ   rå   r   Ú	Parameterr)   rš   rp   rô   rõ   rù   Úpositional_dropoutrÓ   rÔ   ©rH   r;   r´   rI   s      €r1   r@   z#PatchTSTPositionalEncoding.__init__w  s±   ø€ Ü‰ÑÔØ#×1Ñ1ˆÔØ"(×";Ñ";ˆÔØ×ÒäŸ\™\¬%¯+©+°a¸¸A¸v¿~¹~Ó*NÓOˆDŒNØ˜1ÑˆKà ŸM™M¨&°+Ó>ˆÔð 6<×5NÑ5NÐQRÒ5RŒB�J‰J�v×0Ñ0Ó1ð 	ÕÜXZ×XcÑXcÓXeð 	Õr3   rM   c                 ó$  — | j                   dk(  r7t        j                  t        j                  || j
                  «      d¬«      }|S | j                   dk(  �r#t        j                  || j
                  «      }t        j                  d|«      j                  d«      }t        j                  t        j                  d| j
                  d«      t        j                  d«      | j
                  z   z  «      }t        j                  ||z  «      |d d …dd d…f<   t        j                  ||z  «      |d d …dd d…f<   ||j                  «       z
  }||j                  «       d	z  z  }t        j                  |d
¬«      }|S t!        | j                   › d�«      ‚)NrÆ   T©Úrequires_gradÚsincosr   r   r#   g     ˆÃ@é
   FzN is not a valid positional encoder. Available types are 'random' and 'sincos'.)Úpositional_encoding_typer   r  r)   Úrandnrp   rš   Úaranger†   ÚexpÚmathÚlogÚsinÚcosÚmeanrí   rB   )r;   r´   rù   ÚpositionÚdiv_terms        r1   rõ   z#PatchTSTPositionalEncoding._init_pe†  sd  € ð ×*Ñ*¨hÒ6ÜŸ<™<¬¯©°KÀÇÁÓ(PÐ`dÔeˆLð Ðð ×,Ñ,°Ó8Ü Ÿ;™; {°F·N±NÓCˆLÜ—|‘| A {Ó3×=Ñ=¸aÓ@ˆHÜ—y‘y¤§¡¨a°·±ÀÓ!CÌÏÉÐQXÓHYÐ\b×\jÑ\jÑHjÐFkÑ!kÓlˆHÜ$)§I¡I¨h¸Ñ.AÓ$BˆLš˜A˜D˜q˜D˜Ñ!Ü$)§I¡I¨h¸Ñ.AÓ$BˆLš˜A˜D˜q˜D˜Ñ!Ø'¨,×*;Ñ*;Ó*=Ñ=ˆLØ'¨<×+;Ñ+;Ó+=ÀÑ+BÑCˆLÜŸ<™<¨ÀEÔJˆLð
 Ðô Ø×2Ñ2Ð3ð  4Bð  Cóð r3   rÄ   c                 óx  — | j                   r�| j                  || j                  dd …d d …f   z   «      }| j                  | j                  d d…d d …f   z   }|j	                  |j
                  d   | j                  dd«      }t        j                  ||fd¬«      }|S | j                  || j                  z   «      }|S )Nr   r   r!   r#   r$   )	rñ   r  rù   rô   ÚexpandrP   rå   r)   Úcat)rH   rÄ   rô   Ú
cls_tokensrã   s        r1   r^   z"PatchTSTPositionalEncoding.forwardš  s¿   € Ø×Òà×1Ñ1°+À×@QÑ@QÐRSÑRTÒVWÐRWÑ@XÑ2XÓYˆKàŸ™¨×):Ñ):¸2¸A¸2ºq¸5Ñ)AÑAˆIà"×)Ñ)¨+×*;Ñ*;¸AÑ*>À×@WÑ@WÐY[Ð]_Ó`ˆJä Ÿ9™9 j°+Ð%>ÀAÔFˆLð Ðð  ×2Ñ2°;À×ARÑARÑ3RÓSˆLØÐr3   )r_   r`   ra   rb   r   rc   r@   Ústaticmethodr   r  rõ   r)   rf   r^   rh   ri   s   @r1   rð   rð   r  sX   ø„ ñð
˜~ð 
¸Cõ 
ð ð˜ð °cð ¸b¿l¹lò ó ðð& 5§<¡<÷ r3   rð   c            	       ój   ‡ — e Zd ZdZdedefˆ fd„Z	 	 ddej                  de	dz  de	dz  d	e
fd
„Zˆ xZS )rý   z
    PatchTST Encoder
    r;   r´   c                 ó&  •— t         ‰| �  |«       d| _        t        |«      | _        t        ||«      | _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        | j                  «        y c c}w )NF)r?   r@   rþ   r  Úembedderrð   Úpositional_encoderr   r  r  Únum_hidden_layersrË   ÚlayersÚ	post_init)rH   r;   r´   r  rI   s       €r1   r@   zPatchTSTEncoder.__init__¯  st   ø€ Ü‰Ñ˜Ô Ø&+ˆÔ#ô *¨&Ó1ˆŒä"<¸VÀ[Ó"QˆÔä—m‘mÌ5ÐQW×QiÑQiÓKjÖ$kÀaÔ%9¸&Õ%AÒ$kÓlˆŒð 	�‰Õùò %ls   ÁBNrÄ   Úoutput_hidden_statesrL   rM   c                 óJ  — |�|n| j                   j                  }|�|n| j                   j                  }| j                  |«      }| j	                  |«      }|rdnd}|rdnd}| j
                  D ]%  }|r||fz   } |||¬«      }	|	d   }|sŒ||	d   fz   }Œ' t        |||¬«      S )a²  
        Parameters:
            patch_input (`torch.Tensor` of shape `(batch_size, num_channels, num_patches, patch_length)`, *required*):
                Past values of the time series
            output_hidden_states (bool, optional): Indicates if hidden states should be outputted.
            output_attentions (bool, optional): Indicates if attentions should be outputted.

        return:
            `BaseModelOutput`
        Nr–   )rã   rL   r   r   )Úlast_hidden_staterJ   Ú
attentions)r;   rL   r7  r2  r3  r5  r   )
rH   rÄ   r7  rL   r   rã   Úencoder_statesÚall_attentionsÚencoder_layerÚlayer_outputss
             r1   r^   zPatchTSTEncoder.forward½  sÐ   € ð" 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð
 —m‘m KÓ0ˆà×.Ñ.¨{Ó;ˆá3™¸ˆÙ0™°dˆØ!Ÿ[™[ò 
	FˆMÙ#Ø!/°<°/Ñ!A�á)°|ÐWhÔiˆMð )¨Ñ+ˆLâ Ø!/°=ÀÑ3CÐ2EÑ!E‘ð
	Fô °È^ÐhvÔwÐwr3   ©NN)r_   r`   ra   rb   r   rc   r@   r)   rf   re   r   r^   rh   ri   s   @r1   rý   rý   ª  s_   ø„ ñð˜~ð ¸Cõ ð" -1Ø)-ñ	)xà—\‘\ð)xð # T™kð)xð   $™;ð	)xð 
÷)xr3   rý   zG
    Base class for model's outputs, with potential hidden states.
    )Úcustom_introc                   ó6  — e Zd ZU dZdZej                  dz  ed<   dZe	ej                     dz  ed<   dZ
e	ej                     dz  ed<   dZej                  dz  ed<   dZej                  dz  ed<   dZej                  dz  ed<   dZej                  dz  ed	<   y)
ÚPatchTSTModelOutputa>  
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, num_patches, patch_length)`):
        Sequence of hidden-states at the output of the last layer of the model.
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
        one for the output of each layer) of shape `(batch_size, num_channels, height, width)`. Hidden-states of
        the model at the output of each layer plus the optional initial embedding outputs.
    mask (`torch.FloatTensor` of shape `(batch_size, num_channels, num_patches)`, *optional*):
        Bool masked tensor indicating which patches are masked
    loc (`torch.FloatTensor` of shape `(batch_size, 1, num_channels)`, *optional*):
        Mean of the input data (batch_size, sequence_length, num_channels) over the sequence_length
    scale (`torch.FloatTensor` of shape `(batch_size, 1, num_channels)`, *optional*):
        Std of the input data (batch_size, sequence_length, num_channels) over the sequence_length
    patch_input (`torch.FloatTensor` of shape `(batch_size, num_channels, num_patches, patch_length)`):
        Patched input to the Transformer
    Nr9  rJ   r:  rŽ   ÚlocÚscalerÄ   )r_   r`   ra   rb   r9  r)   ÚFloatTensorr   rJ   rg   r:  rŽ   rC  rD  rÄ   r–   r3   r1   rB  rB  é  s§   … ñð" 37Ð�u×(Ñ(¨4Ñ/Ó6Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ó6Ø%)€Dˆ%×
Ñ
˜dÑ
"Ó)Ø$(€Cˆ×	Ñ	˜TÑ	!Ó(Ø&*€Eˆ5×Ñ˜tÑ#Ó*Ø,0€K�×"Ñ" TÑ)Ô0r3   rB  z4
    Output type of [`PatchTSTForPretraining`].
    c                   ó¾   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                     dz  ed<   dZe
ej                     dz  ed<   y)ÚPatchTSTForPretrainingOutputa  
    loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
        MSE loss.
    prediction_output (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
        Prediction outputs of the time series modeling heads.
    NÚlossÚprediction_outputrJ   r:  )r_   r`   ra   rb   rH  r)   rE  r   rI  rJ   rg   r:  r–   r3   r1   rG  rG  
  sh   … ñð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø26Ð�u×(Ñ(¨4Ñ/Ó6Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ô6r3   rG  z3
    Output type of [`PatchTSTForRegression`].
    c                   ó¾   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                     dz  ed<   dZe
ej                     dz  ed<   y)ÚPatchTSTForRegressionOutputa  
    loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
        MSE loss.
    regression_outputs (`torch.FloatTensor` of shape `(batch_size, num_targets)`):
        Regression outputs of the time series modeling heads.
    NrH  Úregression_outputsrJ   r:  )r_   r`   ra   rb   rH  r)   rE  r   rL  rJ   rg   r:  r–   r3   r1   rK  rK    sh   … ñð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø37Ð˜×)Ñ)¨DÑ0Ó7Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ô6r3   rK  z3
    Output type of [`PatchTSTForPrediction`].
    c                   ó  — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                     dz  ed<   dZe
ej                     dz  ed<   dZej                  dz  ed<   dZej                  dz  ed<   y)	ÚPatchTSTForPredictionOutputa!  
    loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
        MSE loss.
    prediction_outputs (`torch.FloatTensor` of shape `(batch_size, prediction_length, -1)`):
        Prediction outputs of the time series modeling heads.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.

        Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
        heads.
    loc: (`torch.FloatTensor` of shape `(batch_size, 1, num_channels)`, *optional*)
        Mean of the input data (batch_size, sequence_length, num_channels) over the sequence_length
    scale: (`torch.FloatTensor` of shape `(batch_size, 1, num_channels)`, *optional*)
        Std of the input data (batch_size, sequence_length, num_channels) over the sequence_length
    NrH  Úprediction_outputsrJ   r:  rC  rD  )r_   r`   ra   rb   rH  r)   rE  r   rO  rJ   rg   r:  rC  rD  r–   r3   r1   rN  rN  2  s’   … ñð" &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø37Ð˜×)Ñ)¨DÑ0Ó7Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ó6Ø$(€Cˆ×	Ñ	˜TÑ	!Ó(Ø&*€Eˆ5×Ñ˜tÑ#Ô*r3   rN  z7
    Output type of [`PatchTSTForClassification`].
    c                   ó¾   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                     dz  ed<   dZe
ej                     dz  ed<   y)ÚPatchTSTForClassificationOutputa‹  
    loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`):
        Total loss as the sum of the masked language modeling loss and the next sequence prediction
        (classification) loss.
    prediction_logits (`torch.FloatTensor` of shape `(batch_size, num_targets)`):
        Prediction scores of the PatchTST modeling head (scores before SoftMax).
    NrH  Úprediction_logitsrJ   r:  )r_   r`   ra   rb   rH  r)   rE  r   rR  rJ   rg   r:  r–   r3   r1   rQ  rQ  R  sh   … ñð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø26Ð�u×(Ñ(¨4Ñ/Ó6Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ô6r3   rQ  zƒ
    Base class for time series model's predictions outputs that contains the sampled values from the chosen
    distribution.
    c                   ó:   — e Zd ZU dZdZej                  dz  ed<   y)ÚSamplePatchTSTOutputz¤
    sequences (`torch.FloatTensor` of shape `(batch_size, num_samples, prediction_length, num_targets)`):
        Sampled values from the chosen distribution.
    NÚ	sequences)r_   r`   ra   rb   rU  r)   rE  r   r–   r3   r1   rT  rT  g  s   … ñð
 +/€Iˆu× Ñ  4Ñ'Ô.r3   rT  ÚinputÚtargetrM   c                 ó&   — | j                  |«       S )zc
    Computes the negative log likelihood loss from input distribution with respect to target.
    )Úlog_prob)rV  rW  s     r1   ÚnllrZ  x  s   € ð �N‰N˜6Ó"Ð"Ð"r3   Úinput_tensorÚweightsc                 óP  — |�“t        j                  |dk7  | |z  t        j                  | «      «      }t        j                  |r|j	                  |¬«      n|j	                  «       d¬«      }|r|j	                  |¬«      |z  S |j	                  «       |z  S | j                  |¬«      S )aj  
    Computes the weighted average of a given tensor across a given `dim`, masking values associated with weight zero,
    meaning instead of `nan * 0 = nan` you will get `0 * 0 = 0`.

    Args:
        input_tensor (`torch.FloatTensor`):
            Input tensor, of which the average must be computed.
        weights (`torch.FloatTensor`, *optional*):
            Weights tensor, of the same shape as `input_tensor`.
        dim (`int`, *optional*):
            The dim along which to average `input_tensor`.

    Returns:
        `torch.FloatTensor`: The tensor with values averaged along the specified `dim`.
    r   r$   ç      ð?©Úmin)r)   ÚwhereÚ
zeros_likeÚclampr›   r(  )r[  r\  r%   Úweighted_tensorÚsum_weightss        r1   Úweighted_averagerf  €  s›   € ð  ÐÜŸ+™+ g°¡l°LÀ7Ñ4JÌE×L\ÑL\Ð]iÓLjÓkˆÜ—k‘k¹# '§+¡+°# +Ô"6À7Ç;Á;Ã=ÐVYÔZˆÙ03�×#Ñ#¨Ð#Ó,ÐR]Ñ]Ð]¸×9LÑ9LÓ9NÐR]Ñ]Ð]à× Ñ  SÐ Ó)Ð)r3   c            	       ó¬   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  deej                  ej                  ej                  f   fd„Z	ˆ xZ
S )ÚPatchTSTStdScalerz½
    Standardize features by calculating the mean and scaling along the first dimension, and then normalizes it by
    subtracting from the mean and dividing by the standard deviation.
    r;   c                 óè   •— t         ‰| �  «        t        |d«      r|j                  nd| _        t        |d«      r|j
                  nd| _        t        |d«      r|j                  | _        y d| _        y )NÚscaling_dimr   ÚkeepdimTÚminimum_scalegñhãˆµøä>)r?   r@   Úhasattrrj  r%   rk  rl  rs   s     €r1   r@   zPatchTSTStdScaler.__init__Ÿ  s[   ø€ Ü‰ÑÔÜ)0°¸Ô)G�6×%Ò%ÈQˆŒÜ)0°¸Ô)C�v—~’~ÈˆŒÜ5<¸VÀ_Ô5U˜V×1Ñ1ˆÕÐ[_ˆÕr3   ÚdataÚobserved_indicatorrM   c                 óŒ  — |j                  | j                  | j                  ¬«      }|j                  d«      }||z  j                  | j                  | j                  ¬«      |z  }||z
  |z  dz  j                  | j                  | j                  ¬«      |z  }t	        j
                  || j                  z   «      }||z
  |z  ||fS )áC  
        Parameters:
            data (`torch.Tensor` of shape `(batch_size, sequence_length, num_input_channels)`):
                input for Batch norm calculation
            observed_indicator (`torch.BoolTensor` of shape `(batch_size, sequence_length, num_input_channels)`):
                Calculating the scale on the observed indicator.
        Returns:
            tuple of `torch.Tensor` of shapes
                (`(batch_size, sequence_length, num_input_channels)`,`(batch_size, 1, num_input_channels)`,
                `(batch_size, 1, num_input_channels)`)
        ©rk  r^  r#   )r›   r%   rk  Ú	clamp_minr)   Úsqrtrl  )rH   rn  ro  ÚdenominatorrC  ÚvariancerD  s          r1   r^   zPatchTSTStdScaler.forward¥  s¾   € ð )×,Ñ,¨T¯X©X¸t¿|¹|Ð,ÓLˆØ!×+Ñ+¨CÓ0ˆØÐ(Ñ(×-Ñ-¨d¯h©hÀÇÁÐ-ÓMÐP[Ñ[ˆà˜S‘jÐ$6Ñ6¸1Ñ<×AÑAÀ$Ç(Á(ÐTX×T`ÑT`ÐAÓaÐdoÑoˆÜ—
‘
˜8 d×&8Ñ&8Ñ8Ó9ˆØ�s‘
˜eÑ# S¨%Ð/Ð/r3   ©r_   r`   ra   rb   r   r@   r)   rf   rg   r^   rh   ri   s   @r1   rh  rh  ™  sS   ø„ ñð
`˜~õ `ð0Ø—L‘Lð0Ø6;·l±lð0à	ˆu�|‰|˜UŸ\™\¨5¯<©<Ð7Ñ	8÷0r3   rh  c            	       ó¬   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  deej                  ej                  ej                  f   fd„Z	ˆ xZ
S )ÚPatchTSTMeanScalerzŠ
    Computes a scaling factor as the weighted average absolute value along the first dimension, and scales the data
    accordingly.
    r;   c                 ó&  •— t         ‰| �  «        t        |d«      r|j                  nd| _        t        |d«      r|j
                  nd| _        t        |d«      r|j                  nd| _        t        |d«      r|j                  | _        y d | _        y )Nrj  r   rk  Trl  ç»½×Ùß|Û=Údefault_scale)r?   r@   rm  rj  r%   rk  rl  r|  rs   s     €r1   r@   zPatchTSTMeanScaler.__init__Ã  su   ø€ Ü‰ÑÔÜ)0°¸Ô)G�6×%Ò%ÈQˆŒÜ)0°¸Ô)C�v—~’~ÈˆŒÜ5<¸VÀ_Ô5U˜V×1Ò1Ð[`ˆÔÜ5<¸VÀ_Ô5U˜V×1Ñ1ˆÕÐ[_ˆÕr3   rn  ro  rM   c                 óÊ  — ||z  j                  «       j                  | j                  d¬«      }|j                  | j                  d¬«      }|t        j                  |d¬«      z  }| j
                  €Q|j                  d¬«      }t        j                  |j                  d«      d¬«      }t        j                  ||z  «      }n"| j
                  t        j                  |«      z  }t        j                  |dkD  ||«      }t        j                  || j                  ¬«      }||z  }	| j                  s|j                  | j                  ¬«      }|	t        j                  |«      |fS )rq  Trr  r   r_  r   r$   )Úabsr›   r%   r)   rc  r|  ÚsqueezeÚ	ones_likera  rl  rk  rb  )
rH   rn  ro  Úts_sumÚnum_observedrD  Ú	batch_sumÚbatch_observationsr|  Úscaled_datas
             r1   r^   zPatchTSTMeanScaler.forwardÊ  s.  € ð Ð+Ñ+×0Ñ0Ó2×6Ñ6°t·x±xÈÐ6ÓNˆØ)×-Ñ-¨d¯h©hÀÐ-ÓEˆàœŸ™ \°qÔ9Ñ9ˆð ×ÑÐ%ØŸ
™
 q˜
Ó)ˆIÜ!&§¡¨\×-=Ñ-=¸aÓ-@ÀaÔ!HÐÜ!ŸM™M¨)Ð6HÑ*HÓI‰Mà ×.Ñ.´·±ÀÓ1GÑGˆMô —‘˜L¨1Ñ,¨e°]ÓCˆô —‘˜E t×'9Ñ'9Ô:ˆØ˜U‘lˆà�|Š|Ø—M‘M d§h¡h�MÓ/ˆEàœE×,Ñ,¨UÓ3°UÐ:Ð:r3   rw  ri   s   @r1   ry  ry  ½  sS   ø„ ñð
`˜~õ `ð&;Ø—L‘Lð&;Ø6;·l±lð&;à	ˆu�|‰|˜UŸ\™\¨5¯<©<Ð7Ñ	8÷&;r3   ry  c            
       ó¶   ‡ — e Zd ZdZdefˆ fd„Z	 d	dej                  dej                  dz  deej                  ej                  ej                  f   fd„Z	ˆ xZ
S )
ÚPatchTSTNOPScalerz|
    Assigns a scaling factor equal to 1 along the first dimension, and therefore applies no scaling to the input data.
    r;   c                 óª   •— t         ‰| �  «        t        |d«      r|j                  nd| _        t        |d«      r|j
                  | _        y d| _        y )Nrj  r   rk  T)r?   r@   rm  rj  r%   rk  rs   s     €r1   r@   zPatchTSTNOPScaler.__init__ù  s@   ø€ Ü‰ÑÔÜ)0°¸Ô)G�6×%Ò%ÈQˆŒÜ)0°¸Ô)C�v—~‘~ˆ�Èˆ�r3   Nrn  ro  rM   c                 óü   — t        j                  |d¬«      j                  | j                  | j                  ¬«      }t        j
                  |d¬«      j                  | j                  | j                  ¬«      }|||fS )a�  
        Parameters:
            data (`torch.Tensor` of shape `(batch_size, sequence_length, num_input_channels)`):
                input for Batch norm calculation
        Returns:
            tuple of `torch.Tensor` of shapes
                (`(batch_size, sequence_length, num_input_channels)`,`(batch_size, 1, num_input_channels)`,
                `(batch_size, 1, num_input_channels)`)
        Fr  )r%   rk  )r)   r€  r(  r%   rk  rb  )rH   rn  ro  rD  rC  s        r1   r^   zPatchTSTNOPScaler.forwardþ  si   € ô —‘ °EÔ:×?Ñ?ÀDÇHÁHÐVZ×VbÑVbÐ?ÓcˆÜ×Ñ˜t°5Ô9×>Ñ>À4Ç8Á8ÐUY×UaÑUaÐ>ÓbˆØ�S˜%ÐÐr3   rÁ   rw  ri   s   @r1   r‡  r‡  ô  s_   ø„ ñðN˜~õ Nð MQñ Ø—L‘Lð Ø6;·l±lÀTÑ6Ið à	ˆu�|‰|˜UŸ\™\¨5¯<©<Ð7Ñ	8÷ r3   r‡  c            	       ó¨   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  deej                  ej                  ej                  f   fd„Zˆ xZ	S )ÚPatchTSTScalerr;   c                 óÞ   •— t         ‰| �  «        |j                  dk(  s|j                  du rt        |«      | _        y |j                  dk(  rt        |«      | _        y t        |«      | _        y )Nr(  Trí   )r?   r@   r   ry  Úscalerrh  r‡  rs   s     €r1   r@   zPatchTSTScaler.__init__  sU   ø€ Ü‰ÑÔØ�>‰>˜VÒ# v§~¡~¸Ñ'=Ü,¨VÓ4ˆD�KØ�^‰^˜uÒ$Ü+¨FÓ3ˆD�Kä+¨FÓ3ˆD�Kr3   rn  ro  rM   c                 ó8   — | j                  ||«      \  }}}|||fS )a>  
        Parameters:
            data (`torch.Tensor` of shape `(batch_size, sequence_length, num_input_channels)`):
                Input for scaler calculation
            observed_indicator (`torch.BoolTensor` of shape `(batch_size, sequence_length, num_input_channels)`):
                Calculating the scale on the observed indicator.
        Returns:
            tuple of `torch.Tensor` of shapes
                (`(batch_size, sequence_length, num_input_channels)`,`(batch_size, 1, num_input_channels)`,
                `(batch_size, 1, um_input_channels)`)
        )r�  )rH   rn  ro  rC  rD  s        r1   r^   zPatchTSTScaler.forward  s)   € ð  Ÿ;™; tÐ-?Ó@Ñˆˆc�5Ø�S˜%ÐÐr3   )
r_   r`   ra   r   r@   r)   rf   rg   r^   rh   ri   s   @r1   r‹  r‹    sL   ø„ ð4˜~õ 4ð Ø—L‘Lð Ø6;·l±lð à	ˆu�|‰|˜UŸ\™\¨5¯<©<Ð7Ñ	8÷ r3   r‹  c                   ó´   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 ddej                  dej                  dz  dej                  dz  dedz  dedz  d	edz  d
ee	z  fd„Z
ˆ xZS )ÚPatchTSTModelr;   c                 ób  •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        |j                  | _        | j
                  j                  }| j                  rt        |«      | _	        nt        j                  «       | _	        t        ||¬«      | _        | j                  «        y )N)r´   )r?   r@   r‹  r�  r®   Ú
patchifierÚdo_mask_inputr´   r¿   Úmaskingr   rÔ   rý   Úencoderr6  r  s      €r1   r@   zPatchTSTModel.__init__-  s�   ø€ Ü‰Ñ˜Ô ä$ VÓ,ˆŒÜ*¨6Ó2ˆŒØ#×1Ñ1ˆÔà—o‘o×1Ñ1ˆà×ÒÜ*¨6Ó2ˆD�LäŸ;™;›=ˆDŒLÜ& v¸;ÔGˆŒð 	�‰Õr3   Nr·   Úpast_observed_maskÚfuture_valuesr7  rL   Úreturn_dictrM   c           	      óŠ  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|€t	        j
                  |«      }| j                  ||«      \  }}	}
| j                  |«      }| j                  r| j                  |«      \  }}n| j                  |«      d}}| j                  |||¬«      }|s>|j                  |j                  |j                  f}|||	|
|fz   }t        d„ |D «       «      S t        |j                  |j                  |j                  ||	|
|¬«      S )a  
        Parameters:
            past_values (`torch.Tensor` of shape `(bs, sequence_length, num_input_channels)`, *required*):
                Input sequence to the model
            past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length, num_input_channels)`, *optional*):
                Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected
                in `[0, 1]`:

                - 1 for values that are **observed**,
                - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros).
            future_values (`torch.BoolTensor` of shape `(batch_size, prediction_length, num_input_channels)`, *optional*):
                Future target values associated with the `past_values`
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers
            output_attentions (`bool`, *optional*):
                Whether or not to return the output attention of all layers
            return_dict (`bool`, *optional*):
                Whether or not to return a `ModelOutput` instead of a plain tuple.

        Returns:
            `PatchTSTModelOutput` or tuple of `torch.Tensor` (if `return_dict`=False or `config.return_dict`=False)

        Examples:

        ```python
        >>> from huggingface_hub import hf_hub_download
        >>> import torch
        >>> from transformers import PatchTSTModel

        >>> file = hf_hub_download(
        ...     repo_id="hf-internal-testing/etth1-hourly-batch", filename="train-batch.pt", repo_type="dataset"
        ... )
        >>> batch = torch.load(file)

        >>> model = PatchTSTModel.from_pretrained("namctin/patchtst_etth1_pretrain")

        >>> # during training, one provides both past and future values
        >>> outputs = model(
        ...     past_values=batch["past_values"],
        ...     future_values=batch["future_values"],
        ... )

        >>> last_hidden_state = outputs.last_hidden_state
        ```N)rÄ   r7  rL   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrÁ   r–   )Ú.0Úvs     r1   ú	<genexpr>z(PatchTSTModel.forward.<locals>.<genexpr>�  s   è ø€ Ò=˜q¨q©}œÑ=ùs   ‚Š)r9  rJ   r:  rŽ   rC  rD  rÄ   )r;   r˜  rL   r7  r)   r€  r�  r’  r“  r”  r•  r9  rJ   r:  rg   rB  )rH   r·   r–  r—  r7  rL   r˜  r   Úscaled_past_valuesrC  rD  Úpatched_valuesÚmasked_valuesrŽ   Úencoder_outputrç   s                   r1   r^   zPatchTSTModel.forward?  sY  € ðn &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆØ1BÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð Ð%Ü!&§¡°Ó!=Ðð *.¯©°[ÐBTÓ)UÑ&Ð˜C ð Ÿ™Ð);Ó<ˆØ×ÒØ"&§,¡,¨~Ó">ÑˆM™4à"&§,¡,¨~Ó">À˜4ˆMàŸ™Ø%Ð<PÐduð &ó 
ˆñ Ø%×7Ñ7¸×9UÑ9UÐWe×WpÑWpÐqˆGØ  s¨E°>Ð BÑBˆGÜÑ= GÔ=Ó=Ð=ä"Ø,×>Ñ>Ø(×6Ñ6Ø%×0Ñ0ØØØØ&ô
ð 	
r3   ©NNNNN)r_   r`   ra   r   r@   r)   rf   re   rg   rB  r^   rh   ri   s   @r1   r�  r�  +  s™   ø„ ð˜~õ ð* 37Ø-1Ø,0Ø)-Ø#'ñ[
à—\‘\ð[
ð "ŸL™L¨4Ñ/ð[
ð —|‘| dÑ*ð	[
ð
 # T™kð[
ð   $™;ð[
ð ˜D‘[ð[
ð 
Ð$Ñ	$÷[
r3   r�  c                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚPatchTSTMaskPretrainHeadz-
    Pretraining head for mask modelling
    r;   c                 ó0  •— t         ‰| �  «        |j                  dkD  rt        j                  |j                  «      nt        j
                  «       | _        t        j                  |j                  |j                  «      | _
        |j                  | _        y ©Nr   )r?   r@   Úhead_dropoutr   rÓ   rÔ   r   rC   rp   r¥   Úlinearrñ   rs   s     €r1   r@   z!PatchTSTMaskPretrainHead.__init__¢  sh   ø€ Ü‰ÑÔØ:@×:MÑ:MÐPQÒ:Q”r—z‘z &×"5Ñ"5Ô6ÔWY×WbÑWbÓWdˆŒÜ—i‘i §¡°×0CÑ0CÓDˆŒØ#×1Ñ1ˆÕr3   Ú	embeddingrM   c                 ó€   — | j                  | j                  |«      «      }| j                  r|dd…dd…dd…dd…f   }|S )aÛ  
        Parameters:
            embedding (`torch.Tensor` of shape `(bs, num_channels, num_patches, d_model)` or
                    `(bs, num_channels, num_patches+1, d_model)` if `cls_token` is set to True, *required*):
                Embedding from the model
        Returns:
            `torch.Tensor` of shape `(bs, num_channels, num_patches, d_model)` or
                            `(bs, num_channels, num_patches+1, d_model)` if `cls_token` is set to True

        Nr   )r¨  r   rñ   )rH   r©  s     r1   r^   z PatchTSTMaskPretrainHead.forward¨  s>   € ð —K‘K §¡¨YÓ 7Ó8ˆ	Ø×ÒØ!¢!¢Q¨©ªA +Ñ.ˆIØÐr3   rw   ri   s   @r1   r¤  r¤  �  s/   ø„ ñð2˜~õ 2ð §¡ð °%·,±,÷ r3   r¤  z*
    The PatchTST for pretrain model.
    c                   ó”   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 ddej                  dej                  dz  dedz  dedz  dedz  d	ee	z  fd
„Z
ˆ xZS )ÚPatchTSTForPretrainingr;   c                 ó”   •— t         ‰| �  |«       d|_        t        |¬«      | _        t        |«      | _        | j                  «        y )NT)r;   )r?   r@   r“  r�  rê   r¤  Úheadr6  rs   s     €r1   r@   zPatchTSTForPretraining.__init__¿  s<   ø€ Ü‰Ñ˜Ô à#ˆÔÜ"¨&Ô1ˆŒ
Ü,¨VÓ4ˆŒ	ð 	�‰Õr3   Nr·   r–  r7  rL   r˜  rM   c                 óü  — |�|n| j                   j                  }| j                  ||||d¬«      }| j                  |j                  «      }t        j                  d¬«      }	 |	||j                  «      }
|
j                  d¬«      |j                  z  j                  «       |j                  j                  «       dz   z  }|j                  }|s|f|dd	 z   }|�|f|z   }|S |}|S t        ||||j                  ¬
«      S )aª	  
        Parameters:
            past_values (`torch.Tensor` of shape `(bs, sequence_length, num_input_channels)`, *required*):
                Input sequence to the model
            past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length, num_input_channels)`, *optional*):
                Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected
                in `[0, 1]`:

                - 1 for values that are **observed**,
                - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros).
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers
            output_attentions (`bool`, *optional*):
                Whether or not to return the output attention of all layers
            return_dict (`bool`, *optional*): Whether or not to return a `ModelOutput` instead of a plain tuple.

        Returns:
            `PatchTSTForPretrainingOutput` or tuple of `torch.Tensor` (if `return_dict`=False or
            `config.return_dict`=False)

        Examples:

        ```python
        >>> from huggingface_hub import hf_hub_download
        >>> import torch
        >>> from transformers import PatchTSTConfig, PatchTSTForPretraining

        >>> file = hf_hub_download(
        ...     repo_id="hf-internal-testing/etth1-hourly-batch", filename="train-batch.pt", repo_type="dataset"
        ... )
        >>> batch = torch.load(file)

        >>> # Config for random mask pretraining
        >>> config = PatchTSTConfig(
        ...     num_input_channels=7,
        ...     context_length=512,
        ...     patch_length=12,
        ...     stride=12,
        ...     mask_type='random',
        ...     random_mask_ratio=0.4,
        ...     use_cls_token=True,
        ... )
        >>> # Config for forecast mask pretraining
        >>> config = PatchTSTConfig(
        ...     num_input_channels=7,
        ...     context_length=512,
        ...     patch_length=12,
        ...     stride=12,
        ...     mask_type='forecast',
        ...     num_forecast_mask_patches=5,
        ...     use_cls_token=True,
        ... )
        >>> model = PatchTSTForPretraining(config)

        >>> # during training, one provides both past and future values
        >>> outputs = model(past_values=batch["past_values"])

        >>> loss = outputs.loss
        >>> loss.backward()
        ```T©r·   r–  r7  rL   r˜  Únone©Ú	reductionr!   r$   r{  r   éüÿÿÿ)rH  rI  rJ   r:  )r;   r˜  rê   r®  r9  r   ÚMSELossrÄ   r(  rŽ   r›   rJ   rG  r:  )rH   r·   r–  r7  rL   r˜  r   Úmodel_outputÚx_hatrH  Úloss_valÚmasked_lossr;  rç   s                 r1   r^   zPatchTSTForPretraining.forwardÉ  s  € ðL &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆð —z‘zØ#Ø1Ø!5Ø/Øð "ó 
ˆð —	‘	˜,×8Ñ8Ó9ˆô �z‰z FÔ+ˆÙ˜˜|×7Ñ7Ó8ˆØ—}‘}¨�}Ó,¨|×/@Ñ/@Ñ@×EÑEÓGÈ<×K\ÑK\×K`ÑK`ÓKbÐejÑKjÑkˆà%×3Ñ3ˆÙØ�h ¨a°Ð!3Ñ3ˆGØ2=Ð2I�{�n wÑ.ˆGØˆNð PWˆGØˆNÜ+Ø°À^Ð`l×`wÑ`wô
ð 	
r3   )NNNN)r_   r`   ra   r   r@   r)   rf   re   rg   rG  r^   rh   ri   s   @r1   r¬  r¬  ¹  s‚   ø„ ð˜~õ ð 37Ø,0Ø)-Ø#'ñb
à—\‘\ðb
ð "ŸL™L¨4Ñ/ðb
ð # T™kð	b
ð
   $™;ðb
ð ˜D‘[ðb
ð 
Ð-Ñ	-÷b
r3   r¬  c                   óD   ‡ — e Zd Zdefˆ fd„Zdej                  fd„Zˆ xZS )ÚPatchTSTClassificationHeadr;   c                 ó¢  •— t         ‰| �  «        |j                  | _        |j                  | _        t	        j
                  d¬«      | _        |j                  dkD  rt	        j                  |j                  «      nt	        j                  «       | _
        t	        j                  |j                  |j                  z  |j                  «      | _        y ©Nr   ©Ú	start_dimr   )r?   r@   rñ   Úpooling_typer   ÚFlattenÚflattenr§  rÓ   rÔ   r   rC   rå   rp   Únum_targetsr¨  rs   s     €r1   r@   z#PatchTSTClassificationHead.__init__/  s‘   ø€ Ü‰ÑÔØ#×1Ñ1ˆÔØ"×/Ñ/ˆÔÜ—z‘z¨AÔ.ˆŒØ:@×:MÑ:MÐPQÒ:Q”r—z‘z &×"5Ñ"5Ô6ÔWY×WbÑWbÓWdˆŒÜ—i‘i × 9Ñ 9¸F¿N¹NÑ JÈF×L^ÑL^Ó_ˆ�r3   r©  c                 ón  — | j                   r|dd…dd…ddd…f   }ng| j                  dk(  r|j                  d¬«      }nE| j                  dk(  r|j                  d¬«      j                  }nt        d| j                  › d�«      ‚| j                  |«      }| j                  | j                  |«      «      }|S )	a[  
        Parameters:
            embedding (`torch.Tensor` of shape `(bs, num_channels, num_patches, d_model)` or
                     `(bs, num_channels, num_patches+1, d_model)` if `cls_token` is set to True, *required*):
                Embedding from the model
        Returns:
            `torch.Tensor` of shape `(bs, num_targets)`

        Nr   r(  r#   r$   r³   úpooling operator ú is not implemented yet)	rñ   rÀ  r(  r³   ÚvaluesrB   rÂ  r¨  r   ©rH   r©  Úpooled_embeddingrv   s       r1   r^   z"PatchTSTClassificationHead.forward7  s®   € ð ×Òà(ªªA¨q²!¨Ñ4ÑØ×Ñ &Ò(à(Ÿ~™~°!˜~Ó4ÑØ×Ñ %Ò'à(Ÿ}™}°˜}Ó3×:Ñ:ÑäÐ0°×1BÑ1BÐ0CÐCZÐ[Ó\Ð\àŸ<™<Ð(8Ó9Ðà—‘˜TŸ\™\Ð*:Ó;Ó<ˆØˆr3   r  ri   s   @r1   r»  r»  .  s!   ø„ ð`˜~õ `ð §¡÷ r3   r»  z0
    The PatchTST for classification model.
    c                   óª   ‡ — e Zd Zdefˆ fd„Ze	 	 	 	 	 ddej                  dej                  dz  dedz  dedz  dedz  d	edz  d
e	e
z  fd„«       Zˆ xZS )ÚPatchTSTForClassificationr;   c                 óÔ   •— t         ‰| �  |«       |j                  rt        j	                  d«       d|_        t        |«      | _        t        |«      | _        | j                  «        y )Nú+Setting `do_mask_input` parameter to False.F)
r?   r@   r“  ÚloggerÚwarningr�  rê   r»  r®  r6  rs   s     €r1   r@   z"PatchTSTForClassification.__init__Y  sT   ø€ Ü‰Ñ˜Ô ð ×ÒÜ�N‰NÐHÔIØ#(ˆFÔ ä" 6Ó*ˆŒ
Ü.¨vÓ6ˆŒ	ð 	�‰Õr3   Nr·   Útarget_valuesr–  r7  rL   r˜  rM   c                 óR  — |�|n| j                   j                  }| j                  ||||d¬«      }| j                  |j                  «      }	d}
|�t        j                  «       } ||	|«      }
|s|	f|dd z   }|
�|
f|z   }|S |}|S t        |
|	|j                  |j                  ¬«      S )ac  
        past_values (`torch.Tensor` of shape `(bs, sequence_length, num_input_channels)`, *required*):
            Input sequence to the model
        target_values (`torch.Tensor`, *optional*):
            Labels associates with the `past_values`
        past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length, num_input_channels)`, *optional*):
            Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected
            in `[0, 1]`:

            - 1 for values that are **observed**,
            - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros).

        Examples:

        ```python
        >>> from transformers import PatchTSTConfig, PatchTSTForClassification

        >>> # classification task with two input channel2 and 3 classes
        >>> config = PatchTSTConfig(
        ...     num_input_channels=2,
        ...     num_targets=3,
        ...     context_length=512,
        ...     patch_length=12,
        ...     stride=12,
        ...     use_cls_token=True,
        ... )
        >>> model = PatchTSTForClassification(config=config)

        >>> # during inference, one only provides past values
        >>> past_values = torch.randn(20, 512, 2)
        >>> outputs = model(past_values=past_values)
        >>> labels = outputs.prediction_logits
        ```NTr°  r   r¼   )rH  rR  rJ   r:  )
r;   r˜  rê   r®  r9  r   ÚCrossEntropyLossrQ  rJ   r:  )rH   r·   rÐ  r–  r7  rL   r˜  r   r¶  Úy_hatr¸  rH  rç   s                r1   r^   z!PatchTSTForClassification.forwardg  s×   € ðZ &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—z‘zØ#Ø1Ø!5Ø/Øð "ó 
ˆð —	‘	˜,×8Ñ8Ó9ˆàˆØÐ$Ü×&Ñ&Ó(ˆDÙ˜E =Ó1ˆHáØ�h ¨a°Ð!3Ñ3ˆGØ/7Ð/C�x�k GÑ+ˆGØˆNð JQˆGØˆNÜ.ØØ#Ø&×4Ñ4Ø#×.Ñ.ô	
ð 	
r3   r¢  )r_   r`   ra   r   r@   r   r)   rf   re   rg   rQ  r^   rh   ri   s   @r1   rË  rË  S  s¤   ø„ ð˜~õ ð ð .2Ø*.Ø,0Ø)-Ø#'ñE
à—\‘\ðE
ð —|‘| dÑ*ðE
ð ! 4™Kð	E
ð
 # T™kðE
ð   $™;ðE
ð ˜D‘[ðE
ð 
Ð0Ñ	0òE
ó ôE
r3   rË  z,
    The PatchTST for regression Model.
    c                   óJ   ‡ — e Zd Zddedefˆ fd„Zdej                  fd„Zˆ xZ	S )ÚPatchTSTPredictionHeadr;   r´   c                 ó  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        |j
                  | _        | j
                  s| j                  r|j                  }n|j                  |z  }| j                  �sVt        j                  «       | _	        t        j                  «       | _
        t        j                  «       | _        t        | j                  «      D ]ò  }| j                  j                  t        j                  d¬«      «       |€:| j                  j                  t        j                  ||j                   «      «       n*| j                  j                  |j#                  |«      «       | j                  j                  |j$                  dkD  rt        j&                  |j$                  «      nt        j(                  «       «       Œô yt        j                  d¬«      | _        |€&t        j                  ||j                   «      | _        n|j#                  |«      | _        |j$                  dkD  rt        j&                  |j$                  «      nt        j(                  «       | _        y)a  
        num_patches (`int`):
            The number of patches in the input sequence.
        distribution_output (`DistributionOutput`, *optional*):
            The distribution output layer for probabilistic forecasting. If None, a linear output layer is used.
        r#   r¾  Nr   )r?   r@   Úshare_projectionrå   rñ   rÀ  rp   r   r  ÚprojectionsÚdropoutsÚflattensr  r�   rÁ  rC   Úprediction_lengthÚget_parameter_projectionr§  rÓ   rÔ   rÂ  Ú
projectionr   )rH   r;   r´   Údistribution_outputrA   r  rI   s         €r1   r@   zPatchTSTPredictionHead.__init__¶  sÕ  ø€ ô 	‰ÑÔà &× 7Ñ 7ˆÔØ"(×";Ñ";ˆÔØ#×1Ñ1ˆÔØ"×/Ñ/ˆÔØ×Ò × 2Ò 2Ø—~‘~‰Hà—~‘~¨Ñ3ˆHà×$Ó$ä!Ÿ}™}›ˆDÔÜŸM™M›OˆDŒMÜŸM™M›OˆDŒMÜ˜4×2Ñ2Ó3ò t�Ø—‘×$Ñ$¤R§Z¡Z¸!Ô%<Ô=Ø&Ð.à×$Ñ$×+Ñ+¬B¯I©I°hÀ×@XÑ@XÓ,YÕZð ×$Ñ$×+Ñ+Ð,?×,XÑ,XÐYaÓ,bÔcØ—‘×$Ñ$È×H[ÑH[Ð^_ÒH_¤R§Z¡Z°×0CÑ0CÔ%DÔeg×epÑepÓerÕsñtô Ÿ:™:°Ô2ˆDŒLØ"Ð*ä"$§)¡)¨H°f×6NÑ6NÓ"O�•ð #6×"NÑ"NÈxÓ"X�”Ø>D×>QÑ>QÐTUÒ>Uœ2Ÿ:™: f×&9Ñ&9Ô:Ô[]×[fÑ[fÓ[hˆD�Lr3   r©  c                 óä  — | j                   r|dd…dd…ddd…f   }nP| j                  dk(  r|j                  d¬«      }n.| j                  dk(  r|j                  d¬«      j                  }n|}| j
                  sŽg }t        | j                  «      D ]\  } | j                  |   |dd…|dd…f   «      } | j                  |   |«      } | j                  |   |«      }|j                  |«       Œ^ t        j                  |d¬«      }n3| j                  |«      }| j                  |«      }| j!                  |«      }t#        |t$        «      rt%        d„ |D «       «      }|S |j'                  dd«      }|S )	aj  
        Parameters:
            embedding (`torch.Tensor` of shape `(bs, num_channels, num_patches, d_model)` or
                     `(bs, num_channels, num_patches+1, d_model)` if `cls_token` is set to True, *required*):
                Embedding from the model
        Returns:
            `torch.Tensor` of shape `(bs, forecast_len, num_channels)`

        Nr   r(  r#   r$   r³   r   c              3   ó@   K  — | ]  }|j                  d d«      –— Œ y­w)r#   r   N)r+   )r›  Úzs     r1   r�  z1PatchTSTPredictionHead.forward.<locals>.<genexpr>  s   è ø€ Ò=°˜1Ÿ;™; q¨!×,Ñ=ùs   ‚)rñ   rÀ  r(  r³   rÇ  r×  r  rå   rÚ  rÙ  rØ  r�   r)   r  rÂ  r   rÝ  r™   rg   r+   )rH   r©  rÉ  rv   r  s        r1   r^   zPatchTSTPredictionHead.forwardá  st  € ð ×Òà(ªªA¨q²!¨Ñ4Ñà× Ñ  FÒ*à#,§>¡>°a >Ó#8Ñ Ø×"Ñ" eÒ+à#,§=¡=°Q =Ó#7×#>Ñ#>Ñ ð $-Ð à×$Ò$ØˆFÜ˜4×2Ñ2Ó3ò 0�à#3 4§=¡=°Ñ#3Ð4DÂQÈÊ1ÀWÑ4MÓ#NÐ Ø#3 4§=¡=°Ñ#3Ð4DÓ#EÐ ð $7 4×#3Ñ#3°AÑ#6Ð7GÓ#HÐ Ø—‘Ð.Õ/ð0ô —[‘[ ¨QÔ/‰Fð  $Ÿ|™|Ð,<Ó=ÐØ#Ÿ|™|Ð,<Ó=Ðð —_‘_Ð%5Ó6ˆFä�fœeÔ$äÑ=°fÔ=Ó=ˆFð ˆð ×%Ñ% a¨Ó+ˆFØˆr3   rÁ   )
r_   r`   ra   r   rc   r@   r)   rf   r^   rh   ri   s   @r1   rÕ  rÕ  °  s*   ø„ ñ)i˜~ð )i¸Cõ )iðV1 §¡÷ 1r3   rÕ  z,
    The PatchTST for prediction model.
    c                   ó"  ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 ddej                  dej                  dz  dej                  dz  dedz  dedz  d	edz  d
ee	z  fd„Z
 ej                  «       	 ddej                  dej                  dz  d
efd„«       Zˆ xZS )ÚPatchTSTForPredictionr;   c                 óŠ  •— t         ‰| �  |«       |j                  rt        j	                  d«       d|_        t        |«      | _        |j                  dk(  rd | _        n™|j                  dk(  rt        |j                  ¬«      | _        nn|j                  dk(  rt        |j                  ¬«      | _        nC|j                  dk(  rt        |j                  ¬«      | _        nt        d|j                  › �«      ‚t        || j                  j                  j                   | j                  ¬	«      | _        | j%                  «        y )
NrÍ  FÚmseÚ	student_tr$   ÚnormalÚnegative_binomialúUnknown distribution output )rÞ  )r?   r@   r“  rÎ  rÏ  r�  rê   rH  rÞ  r   rÛ  r   r   rB   rÕ  r’  r´   r®  r6  rs   s     €r1   r@   zPatchTSTForPrediction.__init__  s  ø€ Ü‰Ñ˜Ô ð ×ÒÜ�N‰NÐHÔIØ#(ˆFÔ ä" 6Ó*ˆŒ
à�;‰;˜%ÒØ'+ˆDÕ$à×)Ñ)¨[Ò8Ü+9¸f×>VÑ>VÔ+W�Õ(Ø×+Ñ+¨xÒ7Ü+7¸F×<TÑ<TÔ+U�Õ(Ø×+Ñ+Ð/BÒBÜ+AÀf×F^ÑF^Ô+_�Õ(ä Ð#?À×@ZÑ@ZÐ?[Ð!\Ó]Ð]ä*Ø�D—J‘J×)Ñ)×5Ñ5È4×KcÑKcô
ˆŒ	ð
 	�‰Õr3   Nr·   r–  r—  r7  rL   r˜  rM   c                 óŒ  — |�|n| j                   j                  }| j                  ||||d¬«      }| j                  |j                  «      }	d}
| j
                  r|	}n|	|j                  z  |j                  z   }|�u| j
                  rJ| j
                  j                  |	|j                  |j                  ¬«      }t        ||«      }
t        |
«      }
nt        j                  d¬«      } |||«      }
|j                  }|j                  }|s|f|dd z   }|
�|
f|z   }|S |}|S t        |
||j                  |j                  ||¬	«      S )
aV	  
        Parameters:
            past_values (`torch.Tensor` of shape `(bs, sequence_length, num_input_channels)`, *required*):
                Input sequence to the model
            past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length, num_input_channels)`, *optional*):
                Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected
                in `[0, 1]`:

                - 1 for values that are **observed**,
                - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros).
            future_values (`torch.Tensor` of shape `(bs, forecast_len, num_input_channels)`, *optional*):
                Future target values associated with the `past_values`
            output_hidden_states (`bool`, *optional*):
                Whether or not to return the hidden states of all layers
            output_attentions (`bool`, *optional*):
                Whether or not to return the output attention of all layers
            return_dict (`bool`, *optional*):
                Whether or not to return a `ModelOutput` instead of a plain tuple.

        Returns:
            `PatchTSTForPredictionOutput` or tuple of `torch.Tensor` (if `return_dict`=False or
            `config.return_dict`=False)

        Examples:

        ```python
        >>> from huggingface_hub import hf_hub_download
        >>> import torch
        >>> from transformers import PatchTSTConfig, PatchTSTForPrediction

        >>> file = hf_hub_download(
        ...     repo_id="hf-internal-testing/etth1-hourly-batch", filename="train-batch.pt", repo_type="dataset"
        ... )
        >>> batch = torch.load(file)

        >>> # Prediction task with 7 input channels and prediction length is 96
        >>> model = PatchTSTForPrediction.from_pretrained("namctin/patchtst_etth1_forecast")

        >>> # during training, one provides both past and future values
        >>> outputs = model(
        ...     past_values=batch["past_values"],
        ...     future_values=batch["future_values"],
        ... )

        >>> loss = outputs.loss
        >>> loss.backward()

        >>> # during inference, one only provides past values, the model outputs future values
        >>> outputs = model(past_values=batch["past_values"])
        >>> prediction_outputs = outputs.prediction_outputs
        ```NTr°  ©rC  rD  r(  r²  r   r!   )rH  rO  rJ   r:  rC  rD  )r;   r˜  rê   r®  r9  rÞ  rD  rC  ÚdistributionrZ  rf  r   rµ  rN  rJ   r:  )rH   r·   r–  r—  r7  rL   r˜  r   r¶  rÓ  r¸  Ú	y_hat_outrì  rH  rC  rD  rç   s                    r1   r^   zPatchTSTForPrediction.forward8  sn  € ð| &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆð —z‘zØ#Ø1Ø!5Ø/Øð "ó 
ˆð —	‘	˜,×8Ñ8Ó9ˆàˆà×#Ò#Ø‰Ià × 2Ñ 2Ñ2°\×5EÑ5EÑEˆIàÐ$Ø×'Ò'Ø#×7Ñ7×DÑDØ˜|×/Ñ/°|×7IÑ7Ið  Eó  �ô ˜|¨]Ó;�ä+¨HÓ5‘ä—z‘z¨FÔ3�Ù 	¨=Ó9�à×ÑˆØ×"Ñ"ˆáØ �l \°!°BÐ%7Ñ7ˆGØ/7Ð/C�x�k GÑ+ˆGØˆNð JQˆGØˆNÜ*ØØ(Ø&×4Ñ4Ø#×.Ñ.ØØô
ð 	
r3   c                 óª  — | j                   j                  } | |d|d¬«      }| j                  rz| j                  j                  |j                  |j
                  |j                  ¬«      }t        |«      D �cg c]  }|j                  «       ‘Œ }}t        j                  |d¬«      }n|j                  j                  d«      }t        |¬«      S c c}w )a   
        Generate sequences of sample predictions from a model with a probability distribution head.

        Parameters:
            past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_input_channels)`):
                Past values of the time series that serves as context in order to predict the future.
            past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length, num_input_channels)`, *optional*):
                Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected
                in `[0, 1]`:

                - 1 for values that are **observed**,
                - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros).

        Return:
            [`SamplePatchTSTOutput`] where the outputs `sequences` tensor will have shape `(batch_size, number of
            samples, prediction_length, 1)` or `(batch_size, number of samples, prediction_length, num_input_channels)`
            for multivariate predictions.
        NF)r·   r—  r–  r7  rë  r   r$   ©rU  )r;   Únum_parallel_samplesrÞ  rì  rO  rC  rD  r  Úsampler)   r  r†   rT  ©rH   r·   r–  rð  rç   rì  r    Úsampless           r1   ÚgeneratezPatchTSTForPrediction.generate¦  sÃ   € ð2  $Ÿ{™{×?Ñ?Ðñ Ø#ØØ1Ø!&ô	
ˆð ×#Ò#à×3Ñ3×@Ñ@Ø×*Ñ*°·±À7Ç=Á=ð Aó ˆLô 7<Ð<PÓ6QÖR°�|×*Ñ*Õ,ÐRˆGÐRä—k‘k '¨qÔ1‰Gà×0Ñ0×:Ñ:¸1Ó=ˆGä#¨gÔ6Ð6ùò Ss   Á8Cr¢  rÁ   )r_   r`   ra   r   r@   r)   rf   re   rg   rN  r^   r  rT  rô  rh   ri   s   @r1   rã  rã    så   ø„ ð˜~õ ð@ 37Ø-1Ø,0Ø)-Ø#'ñl
à—\‘\ðl
ð "ŸL™L¨4Ñ/ðl
ð —|‘| dÑ*ð	l
ð
 # T™kðl
ð   $™;ðl
ð ˜D‘[ðl
ð 
Ð,Ñ	,ól
ð\ €U‡]�]ƒ_ð 37ñ-7à—\‘\ð-7ð "ŸL™L¨4Ñ/ð-7ð 
ò	-7ó ô-7r3   rã  c                   óJ   ‡ — e Zd ZdZddefˆ fd„Zdej                  fd„Zˆ xZ	S )ÚPatchTSTRegressionHeadz
    Regression head
    r;   c                 ó  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        || _        |j                  |j                  z  }t        j                  d¬«      | _        |j                  dkD  rt        j                  |j                  «      nt        j                  «       | _        |€&t        j                   ||j"                  «      | _        y |j'                  |«      | _        y r½  )r?   r@   Úoutput_rangeÚy_rangerñ   rÀ  rÞ  rå   rp   r   rÁ  rÂ  r§  rÓ   rÔ   r   rC   rÃ  rÝ  rÜ  )rH   r;   rÞ  rA   rI   s       €r1   r@   zPatchTSTRegressionHead.__init__Ü  sÃ   ø€ Ü‰ÑÔØ×*Ñ*ˆŒØ#×1Ñ1ˆÔØ"×/Ñ/ˆÔØ#6ˆÔ à×,Ñ,¨v¯~©~Ñ=ˆä—z‘z¨AÔ.ˆŒØ:@×:MÑ:MÐPQÒ:Q”r—z‘z &×"5Ñ"5Ô6ÔWY×WbÑWbÓWdˆŒàÐ&Ü Ÿi™i¨°&×2DÑ2DÓEˆD�Oà1×JÑJÈ8ÓTˆD�Or3   r©  c                 ó2  — | j                   r|dd…dd…ddd…f   }ng| j                  dk(  r|j                  d¬«      }nE| j                  dk(  r|j                  d¬«      j                  }nt        d| j                  › d�«      ‚| j                  | j                  |«      «      }| j                  |«      }| j                  du | j                  duz  rEt        j                  |«      | j                  d	   | j                  d   z
  z  | j                  d   z   }|S )
aY  
        Parameters:
            embedding (`torch.Tensor` of shape `(bs, num_channels, num_patches, d_model)` or
                    `(bs, num_channels, num_patches+1, d_model)` if `cls_token` is set to True, *required*):
                Embedding from the model
        Returns:
            `torch.Tensor` of shape `(bs, output_dim)`

        Nr   r(  r#   r$   r³   rÅ  rÆ  r   )rñ   rÀ  r(  r³   rÇ  rB   r   rÂ  rÝ  rÞ  rù  r)   ÚsigmoidrÈ  s       r1   r^   zPatchTSTRegressionHead.forwardí  s  € ð ×Òà(ªªA¨q²!¨Ñ4ÑØ×Ñ &Ò(à(Ÿ~™~°!˜~Ó4ÑØ×Ñ %Ò'à(Ÿ}™}°˜}Ó3×:Ñ:ÑäÐ0°×1BÑ1BÐ0CÐCZÐ[Ó\Ð\ð  Ÿ<™<¨¯©Ð5EÓ(FÓGÐð —‘Ð!1Ó2ˆà×$Ñ$¨Ð,°·±ÀTÐ1IÒJÜ—]‘] 6Ó*¨d¯l©l¸1©oÀÇÁÈQÁÑ.OÑPÐSW×S_ÑS_Ð`aÑSbÑbˆFØˆr3   rÁ   rw   ri   s   @r1   rö  rö  ×  s&   ø„ ññU˜~õ Uð" §¡÷ r3   rö  z,
    The PatchTST for regression model.
    c                   ó,  ‡ — e Zd Zdefˆ fd„Ze	 	 	 	 	 ddej                  dej                  dz  dej                  dz  dedz  dedz  d	edz  d
e	e
z  fd„«       Z ej                  «       	 ddej                  dej                  dz  d
efd„«       Zˆ xZS )ÚPatchTSTForRegressionr;   c                 óJ  •— t         ‰| �  |«       |j                  rt        j	                  d«       d|_        t        |«      | _        |j                  dk(  rd | _        n™|j                  dk(  rt        |j                  ¬«      | _        nn|j                  dk(  rt        |j                  ¬«      | _        nC|j                  dk(  rt        |j                  ¬«      | _        nt        d|j                  › �«      ‚t        || j                  «      | _        | j!                  «        y )	NrÍ  Frå  ræ  r$   rç  rè  ré  )r?   r@   r“  rÎ  rÏ  r�  rê   rH  rÞ  r   rÃ  r   r   rB   rö  r®  r6  rs   s     €r1   r@   zPatchTSTForRegression.__init__  sî   ø€ Ü‰Ñ˜Ô ð ×ÒÜ�N‰NÐHÔIØ#(ˆFÔ ä" 6Ó*ˆŒ
Ø�;‰;˜%ÒØ'+ˆDÕ$à×)Ñ)¨[Ò8Ü+9¸f×>PÑ>PÔ+Q�Õ(Ø×+Ñ+¨xÒ7Ü+7¸F×<NÑ<NÔ+O�Õ(Ø×+Ñ+Ð/BÒBÜ+AÀf×FXÑFXÔ+Y�Õ(ä Ð#?À×@ZÑ@ZÐ?[Ð!\Ó]Ð]ä*¨6°4×3KÑ3KÓLˆŒ	ð 	�‰Õr3   Nr·   rÐ  r–  r7  rL   r˜  rM   c                 óþ  ‡ — |�|n‰ j                   j                  }‰ j                  ||||d¬«      }‰ j                  |j                  «      }	d}
|�r‰ j
                  rG‰ j
                  j                  |	«      }t        ˆ fd„|	D «       «      }	t        ||«      }
t        |
«      }
nt        j                  d¬«      }
 |
|	|«      }
|s|	f|dd z   }|
�|
f|z   }|S |}|S t        |
|	|j                  |j                  ¬	«      S )
a#  
        past_values (`torch.Tensor` of shape `(bs, sequence_length, num_input_channels)`, *required*):
            Input sequence to the model
        target_values (`torch.Tensor` of shape `(bs, num_input_channels)`):
            Target values associates with the `past_values`
        past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length, num_input_channels)`, *optional*):
            Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected
            in `[0, 1]`:

            - 1 for values that are **observed**,
            - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros).
            Whether or not to return a `ModelOutput` instead of a plain tuple.

        Examples:

        ```python
        >>> from transformers import PatchTSTConfig, PatchTSTForRegression

        >>> # Regression task with 6 input channels and regress 2 targets
        >>> model = PatchTSTForRegression.from_pretrained("namctin/patchtst_etth1_regression")

        >>> # during inference, one only provides past values, the model outputs future values
        >>> past_values = torch.randn(20, 512, 6)
        >>> outputs = model(past_values=past_values)
        >>> regression_outputs = outputs.regression_outputs
        ```NTr°  c              3   ój   •K  — | ]*  }|j                  d ‰j                  j                  «      –— Œ, y­w)r!   N)rQ   r;   rÃ  )r›  ÚitemrH   s     €r1   r�  z0PatchTSTForRegression.forward.<locals>.<genexpr>e  s&   øè ø€ ÒWÈ˜dŸi™i¨¨D¯K©K×,CÑ,C×DÑWùs   ƒ03r(  r²  r   r¼   )rH  rL  rJ   r:  )r;   r˜  rê   r®  r9  rÞ  rì  rg   rZ  rf  r   rµ  rK  rJ   r:  )rH   r·   rÐ  r–  r7  rL   r˜  r   r¶  rÓ  rH  rì  rç   s   `            r1   r^   zPatchTSTForRegression.forward.  s  ø€ ðL &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà—z‘zØ#Ø1Ø!5Ø/Øð "ó 
ˆð —	‘	˜,×8Ñ8Ó9ˆàˆØÐ$Ø×'Ò'Ø#×7Ñ7×DÑDÀUÓK�äÓWÐQVÔWÓW�Ü˜<¨Ó7�ä'¨Ó-‘ä—z‘z¨FÔ3�Ù˜E =Ó1�áà�h ¨a°Ð!3Ñ3ˆGØ+/Ð+;�t�g Ñ'ˆGØˆNð BIˆGØˆNÜ*ØØ$Ø&×4Ñ4Ø#×.Ñ.ô	
ð 	
r3   c                 óv  — | j                   j                  } | |d|d¬«      }| j                  j                  |j                  «      }t        |«      D �cg c]  }|j                  «       ‘Œ }}t        j                  |d¬«      j                  d|| j                   j                  «      }t        |¬«      S c c}w )a¢  
        Generate sequences of sample predictions from a model with a probability distribution head.

        Parameters:
            past_values (`torch.FloatTensor` of shape `(batch_size, sequence_length, num_input_channels)`):
                Past values of the time series that serves as context in order to predict the future.
            past_observed_mask (`torch.BoolTensor` of shape `(batch_size, sequence_length, num_input_channels)`, *optional*):
                Boolean mask to indicate which `past_values` were observed and which were missing. Mask values selected
                in `[0, 1]`:

                - 1 for values that are **observed**,
                - 0 for values that are **missing** (i.e. NaNs that were replaced by zeros).

        Return:
            [`SamplePatchTSTOutput`] where the outputs `sequences` tensor will have shape `(batch_size, number of
            samples, num_targets)`.
        NF)r·   rÐ  r–  r7  r   r$   r!   rï  )r;   rð  rÞ  rì  rL  r  rñ  r)   r  rQ   rÃ  rT  rò  s           r1   rô  zPatchTSTForRegression.generatey  s§   € ð0  $Ÿ{™{×?Ñ?Ðñ Ø#ØØ1Ø!&ô	
ˆð ×/Ñ/×<Ñ<¸W×=WÑ=WÓXˆä27Ð8LÓ2MÖN¨Q�<×&Ñ&Õ(ÐNˆÐNä—+‘+˜g¨1Ô-×2Ñ2°2Ð7KÈTÏ[É[×MdÑMdÓeˆÜ#¨gÔ6Ð6ùò Os   ÁB6r¢  rÁ   )r_   r`   ra   r   r@   r   r)   rf   re   rg   rK  r^   r  rT  rô  rh   ri   s   @r1   rý  rý    sô   ø„ ð˜~õ ð4 ð .2Ø26Ø,0Ø)-Ø#'ñH
à—\‘\ðH
ð —|‘| dÑ*ðH
ð "ŸL™L¨4Ñ/ð	H
ð
 # T™kðH
ð   $™;ðH
ð ˜D‘[ðH
ð 
Ð,Ñ	,òH
ó ðH
ðT €U‡]�]ƒ_ð 37ñ'7à—\‘\ð'7ð "ŸL™L¨4Ñ/ð'7ð 
ò	'7ó ô'7r3   rý  )r�  ré   rã  r¬  rý  rË  )NrO   )NFr   r¦  r?  )Prb   r$  Úcollections.abcr   Údataclassesr   r)   r   Ú r   rò   Úactivationsr   Úintegrations.deepspeedr	   Úmodeling_flash_attention_utilsr
   Úmodeling_outputsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Útime_series_utilsr   r   r   Úutilsr   r   r   r   Úconfiguration_patchtstr   Ú
get_loggerr_   rÎ  r	  rf   rd   r2   r5   rk   Úlistre   rc   r’   r¬   r®   r¿   rË   ré   r  rð   rý   rB  rG  rK  rN  rQ  rT  ÚdistributionsÚDistributionrZ  rf  rh  ry  r‡  r‹  r�  r¤  r¬  r»  rË  rÕ  rã  rö  rý  Ú__all__r–   r3   r1   ú<module>r     sã  ðñ ã Ý $Ý !ã Ý å &Ý "Ý @Ý BÝ /ß FÝ &ß UÑ Uß MÓ MÝ 2ð 
ˆ×	Ñ	˜HÓ	%€ð !Øñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �T‰\ð%ð ð%ð Ð'Ñ(ó%ô:R/˜Ÿ	™	ô R/ôj&˜Ÿ	™	ô &ð2 -1Ø',Øñ7%Ø�L‰Lð7%àð7%ð # T™kð7%ð !%ð	7%ð
 ó7%ðz -1Øñ	A%Ø�L‰LðA%à# c™zðA%ð # T™kðA%ð ó	A%ôH-�r—y‘yô -ô`9"�b—i‘iô 9"ôxH˜2Ÿ9™9ô HðV ô&2˜oó &2ó ð&2ôR!˜Ÿ	™	ô !ôH5 §¡ô 5ôp<xÐ-ô <xñ~ ðôð
 ô1˜+ó 1ó óð1ñ6 ðôð
 ô7 ;ó 7ó óð7ñ ðôð
 ô7 +ó 7ó óð7ñ ðôð
 ô+ +ó +ó óð+ñ4 ðôð
 ô7 kó 7ó óð7ñ ðôð ô/˜;ó /ó óð/ð#ˆu×"Ñ"×/Ñ/ð #¸¿¹ð #È%Ï,É,ó #ñ* 5§<¡<ð *¸%¿,¹,ÈÑ:Mð *Ðch×coÑcoó *ô2 0˜Ÿ	™	ô  0ôH3;˜Ÿ™ô 3;ôn ˜Ÿ	™	ô  ô6 �R—Y‘Yô  ð8 ôn
Ð+ó n
ó ðn
ôb˜rŸy™yô ñ8 ðôô
m
Ð4ó m
óð
m
ô`" §¡ô "ñJ ðôô
U
Ð 7ó U
óð
U
ñp ðôô
]˜RŸY™Yó ]óð
]ñ@ ðôô
z7Ð3ó z7óð
z7ôz4˜RŸY™Yô 4ñn ðôô
N7Ð3ó N7óð
N7òb�r3   