Ë
    ÜÓ^jÒn  ã                   óÚ  — d Z ddlZddlmZ ddlZddlZddlmc m	Z
 ddlmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZ ddlmZmZ ddlmZ ddl m!Z!m"Z" ddl#m$Z$m%Z% ddl&m'Z'm(Z(m)Z) ddl*m+Z+ ddl,m-Z- ddl.m/Z/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5  e)jl                  e7«      Z8e' G d„ de5«      «       Z9 G d„ de!d¬«      Z:e' e-d¬«       G d „ d!e3«      «       «       Z; e'd"¬#«      e G d$„ d%e«      «       «       Z<d&ejz                  d'efd(„Z> G d)„ d*ej~                  «      Z@ G d+„ d,ej~                  «      ZAe' G d-„ d.e«      «       ZBe' G d/„ d0eB«      «       ZC G d1„ d2eB«      ZDg d3¢ZEy)4zTPI0 model: PaliGemma + Action Expert with flow matching for robot action prediction.é    N)ÚCallable)Ústrict)Únné   )Úinitialization)ÚCache)ÚPreTrainedConfig)ÚBatchFeature)Ú
ImageInputÚmake_nested_list_of_images)Úcreate_causal_mask)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚPreTrainedModel)ÚProcessingKwargsÚUnpack)ÚPreTokenizedInputÚ	TextInput)Úauto_docstringÚcan_return_tupleÚlogging)Úmaybe_autocast)Úrequiresé   )ÚCONFIG_MAPPINGÚ
AutoConfigÚ	AutoModel)ÚPaligemmaProcessor)ÚSiglipImageProcessorc                   ó$   — e Zd ZdddœZdddœZdZy)ÚPI0ImageProcessoréà   )Ú
max_heightÚ	max_width)ÚheightÚwidthTN)Ú__name__Ú
__module__Ú__qualname__ÚsizeÚpad_sizeÚdo_pad© ó    ún/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/pi0/modular_pi0.pyr!   r!   .   s   „ à¨CÑ0€DØ¨Ñ,€HØ�Fr.   r!   c                   ó"   — e Zd ZddddœddidœZy)	ÚPI0ProcessorKwargsÚ
max_lengthé0   Úright)Úpaddingr2   Úpadding_sideÚreturn_tensorsÚpt)Útext_kwargsÚcommon_kwargsN)r'   r(   r)   Ú	_defaultsr-   r.   r/   r1   r1   5   s#   „ ð $ØØ#ñ
ð
 +¨DÐ1ñ�Ir.   r1   F)Útotal)ÚvisionÚtorch)Úbackendsc                   ó  ‡ — e Zd Zdˆ fd„	Z	 	 	 ddeee   z  eee      z  dz  deez  ee   z  ee   z  dz  deej                  z  e
j                  z  dz  deej                  z  e
j                  z  dz  dee   defd	„Zeˆ fd
„«       Zˆ xZS )ÚPI0ProcessorNc                 óD  •— |j                   d   |j                   d   c| _        | _        |j                  dg d¢«      }|j                  dg d¢«      }|j                  dg d¢«      }|j                  d	g d
¢«      }t	        j
                  |«      | _        t	        j
                  |«      | _        t	        j
                  |«      | _        t	        j
                  |«      | _	        |j                  dd«      | _
        |j                  dd«      | _        t        ‰	| �5  ||«       y )Nr%   r&   Ú
state_mean)gù g³ês¥¿grŠŽäò¢?g•Ô	h"lê?g‡§WÊ2D@g\ AñcÌá¿gZd;ßOÅ¿gB>èÙ¬úœ?g“©‚QI�¿Ú	state_std)gt$—ÿ�~»?gL¦
F%uÂ?g.ÿ!ýöuÐ?g/n£¼Ö?gëâ6Àó?g¶óýÔxéÖ?gÌ]KÈ=‹?gF%uš‹?Úactions_mean)gÂ&S£’?gâX·Ñ ®?gW[±¿ìž¬¿gHPüs×r?gàœ¥½Ág?gÒ Þ	Š¿gHPüs·¿Úactions_std)g®GázÒ?g`åÐ"ÛùÖ?gÁ9#J{ƒ×?gvOjM£?g>yX¨5Í«?g46<½R¶?gèj+ö—Ýï?Úmax_state_dimé    Ú
chunk_sizeé2   )r*   r%   r&   Úgetr>   ÚtensorrC   rD   rE   rF   rG   rI   ÚsuperÚ__init__)
ÚselfÚimage_processorÚ	tokenizerÚchat_templateÚkwargsrC   rD   rE   rF   Ú	__class__s
            €r/   rN   zPI0Processor.__init__C   sä   ø€ Ø"1×"6Ñ"6°xÑ"@À/×BVÑBVÐW^ÑB_ÐˆŒ�T”ZØ—Z‘Z Ò.rÓsˆ
Ø—J‘J˜{Ò,lÓmˆ	Ø—z‘z .Ò2mÓnˆØ—j‘j Ò0hÓiˆäŸ,™, zÓ2ˆŒÜŸ™ iÓ0ˆŒÜ!ŸL™L¨Ó6ˆÔÜ Ÿ<™<¨Ó4ˆÔØ#ŸZ™Z¨¸Ó<ˆÔØ Ÿ*™* \°2Ó6ˆŒÜ‰Ñ˜¨)Õ4r.   ÚimagesÚtextÚactionsÚstaterS   Úreturnc                 óÚ  —  | j                   t        fd| j                  j                  i|¤Ž}|€t        j                  d«       d}t        |t        «      r|g}t        |«      }t        |«      t        |«      k7  r$t        dt        |«      › dt        |«      › d�«      ‚|d   j                  d	d«      }|d
   j                  d	d«       g }	t        ||«      D ]V  \  }
}| j                  | j                  z  t        |«      z  › | j                  j                  › |
› d�}
|	j!                  |
«       ŒX  | j                  |	fi |d   ¤Ž}t#        d„ |D «       «      }t%        j&                  t        |«      |ft$        j(                  ¬«      }t%        j&                  t        |«      |d| j*                  | j,                  «      }t/        |«      D ]=  \  }} | j0                  |fd	di|d
   ¤Ž}t        |«      }d||d|…f<   |d   ||d|…f<   Œ? i |¥||dœ¥}|�ªt%        j2                  |«      | j4                  z
  | j6                  dz   z  }|j8                  d   | j:                  k  r2t=        j>                  |d| j:                  |j8                  d   z
  f«      }|jA                  d| jB                  | j:                  «      |d<   |�Ÿt%        j2                  |«      | jD                  z
  | jF                  dz   z  }|j8                  d   | j:                  k  r2t=        j>                  |d| j:                  |j8                  d   z
  f«      }|jA                  d| j:                  «      |d<   tI        ||¬«      S )aC  
        actions (`list | np.ndarray | torch.Tensor`, *optional*):
            Actions to be predicted by the model. If provided, padding, mean and std normalization will be applied.
        state (`list | np.ndarray | torch.Tensor`, *optional*):
            Robotic states to be predicted by the model. If provided, padding, mean and std normalization will be applied.

        Returns:
            [`BatchFeature`]: A [`BatchFeature`] with the following fields:

            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`. If `suffix`
              is provided, the `input_ids` will also contain the suffix input ids.
            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
              `None`).
            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
            - **pixel_attention_mask** -- Pixel values padding mask to be fed to a model. Returned when `images` is not `None`.
            - **state** -- Robot state compatible with model if `state` is not None
            - **actions** -- Label-actions compatible with training if `actions` is not None
        Útokenizer_init_kwargsNzPYou are using PI0 without a text prefix. The processor will use an empty prompt.Ú z	Received z image samples for z\ prompts. Each prompt should be associated with one sample (with one or more camera images).r9   r7   Úimages_kwargsú
c              3   ó2   K  — | ]  }t        |«      –— Œ y ­w©N)Úlen)Ú.0Úsample_imagess     r/   ú	<genexpr>z(PI0Processor.__call__.<locals>.<genexpr>Œ   s   è ø€ ÒU°]œc -×0ÑUùs   ‚©Údtyper   r8   TÚpixel_values)rg   Úpixel_attention_maskg:Œ0âŽyE>éÿÿÿÿr   rW   rX   )ÚdataÚtensor_type)%Ú_merge_kwargsr1   rQ   Úinit_kwargsÚloggerÚwarning_onceÚ
isinstanceÚstrr   ra   Ú
ValueErrorÚpopÚzipÚimage_tokenÚimage_seq_lengthÚ	bos_tokenÚappendÚmaxr>   ÚzerosÚboolr%   r&   Ú	enumeraterP   rL   rE   rF   ÚshaperG   ÚFÚpadÚviewrI   rC   rD   r
   )rO   rU   rV   rW   rX   rS   Úoutput_kwargsÚbatched_imagesr7   Úprompt_stringsÚsampleÚ
image_listÚtext_inputsÚmax_num_camerasrh   Úpadded_pixel_valuesÚbatchrc   Ú	processedÚnum_camerasÚreturn_datas                        r/   Ú__call__zPI0Processor.__call__R   s|  € ð6 +˜×*Ñ*Üñ
Ø6:·n±n×6PÑ6Pð
ØTZñ
ˆð ˆ<Ü×ÑÐ rÔsØˆDä�dœCÔ Ø�6ˆDä3°FÓ;ˆÜˆ~Ó¤# d£)Ò+ÜØœC Ó/Ð0Ð0CÄCÈÃIÀ;ð Oeð eóð ð
 ' }Ñ5×9Ñ9Ð:JÈDÓQˆØ�oÑ&×*Ñ*Ð+;¸TÔBàˆÜ"% d¨NÓ";ò 	*ÑˆF�Jà×#Ñ# d×&;Ñ&;Ñ;¼cÀ*»oÑMÐNÈtÏ~É~×OgÑOgÐNhÐioÐhpÐprÐsð ð ×!Ñ! &Õ)ð		*ð %�d—n‘n ^ÑT°}À]Ñ7SÑTˆô ÑUÀnÔUÓUˆÜ$Ÿ{™{¬C°Ó,?ÀÐ+QÔY^×YcÑYcÔdÐÜ#Ÿk™k¬#¨nÓ*=¸ÐPQÐSW×S^ÑS^Ð`d×`jÑ`jÓkÐä$-¨nÓ$=ò 	QÑ ˆE�=Ø,˜×,Ñ,¨]ÑrÈ4ÐrÐS`ÐapÑSqÑrˆIä˜mÓ,ˆKØ8<Ð  ¨¨¨Ð!4Ñ5Ø7@ÀÑ7PÐ  |¨ |Ð 3Ò4ð	Qð
Øð
à/Ø$8ò
ˆð ÐÜ—|‘| GÓ,¨t×/@Ñ/@Ñ@ÀT×EUÑEUÐX]ÑE]Ñ^ˆGØ�}‰}˜RÑ  4×#5Ñ#5Ò5ÜŸ%™% ¨!¨T×-?Ñ-?À'Ç-Á-ÐPRÑBSÑ-SÐ)TÓU�Ø%,§\¡\°"°d·o±oÀt×GYÑGYÓ%ZˆK˜	Ñ"àÐÜ—\‘\ %Ó(¨4¯?©?Ñ:¸t¿~¹~ÐPUÑ?UÑVˆEØ�{‰{˜2‰ ×!3Ñ!3Ò3ÜŸ™˜e a¨×);Ñ);¸e¿k¹kÈ"¹oÑ)MÐ%NÓO�Ø#(§:¡:¨b°$×2DÑ2DÓ#EˆK˜Ñ ä ¸.ÔIÐIr.   c                 ó    •— t         ‰| �  dgz   S )Nrh   )rM   Úmodel_input_names)rO   rT   s    €r/   r�   zPI0Processor.model_input_names«   s   ø€ ä‰wÑ(Ð,BÐ+CÑCÐCr.   )NNN)r'   r(   r)   rN   r   Úlistr   r   ÚnpÚndarrayr>   ÚTensorr   r1   r
   r�   Úpropertyr�   Ú__classcell__©rT   s   @r/   rA   rA   @   së   ø„ õ5ð$ bfØ;?Ø9=ñWJà˜T *Ñ-Ñ-°°T¸*Ñ5EÑ0FÑFÈÑMðWJð Ð+Ñ+¨d°9©oÑ=ÀÐEVÑ@WÑWÐZ^Ñ^ðWJð ˜Ÿ
™
Ñ" U§\¡\Ñ1°DÑ8ð	WJð
 �b—j‘jÑ  5§<¡<Ñ/°$Ñ6ðWJð Ð+Ñ,ðWJð 
óWJðr óDó ôDr.   rA   zlerobot/pi0_base)Ú
checkpointc                   ó  ‡ — e Zd ZU dZdZeedœZdZee	z  dz  e
d<   dZee	z  dz  e
d<   dZee
d<   d	Zee
d
<   d	Zee
d<   dZee
d<   dZee
d<   dZee
d<   dZee
d<   dZee
d<   dZee
d<   dZee
d<   dZee
d<   ˆ fd„Zd„ Zˆ xZS )Ú	PI0ConfigaÜ  
    vlm_config (`dict`, *optional*):
        Configuration for the vlm backbone (PaliGemmaModel).
    dit_config (`dict`, *optional*):
        Configuration for the DiT backbone. Defaults to a Gemma 300M variant.
    chunk_size (`int`, *optional*, defaults to 50):
        Number of action steps to predict per chunk.
    max_state_dim (`int`, *optional*, defaults to 32):
        Maximum state vector dimension (shorter vectors are zero-padded).
    max_action_dim (`int`, *optional*, defaults to 32):
        Maximum action vector dimension (shorter vectors are zero-padded).
    num_inference_steps (`int`, *optional*, defaults to 10):
        Number of denoising steps during inference.
    time_sampling_beta_alpha (`float`, *optional*, defaults to 1.5):
        Alpha parameter for Beta distribution used to sample diffusion time during training.
    time_sampling_beta_beta (`float`, *optional*, defaults to 1.0):
        Beta parameter for Beta distribution used to sample diffusion time during training.
    time_sampling_scale (`float`, *optional*, defaults to 0.999):
        Scale factor for sampled time values.
    time_sampling_offset (`float`, *optional*, defaults to 0.001):
        Offset added to sampled time values.
    min_period (`float`, *optional*, defaults to 0.004):
        Minimum period for sinusoidal time embedding.
    max_period (`float`, *optional*, defaults to 4.0):
        Maximum period for sinusoidal time embedding.
    loss_reduction (`str`, *optional*, defaults to `"mean"`):
        The reduction to use on MSE loss.

    Example:
    ```python
    >>> from transformers import PI0ForConditionalGeneration, PI0Config

    >>> config = PI0Config()
    >>> model = PI0ForConditionalGeneration(config)
    ```
    Úpi0)Ú
vlm_configÚ
dit_configNr›   rœ   rJ   rI   rH   rG   Úmax_action_dimé
   Únum_inference_stepsg      ø?Útime_sampling_beta_alphaç      ð?Útime_sampling_beta_betag+‡ÙÎ÷ï?Útime_sampling_scalegü©ñÒMbP?Útime_sampling_offsetgü©ñÒMbp?Ú
min_periodg      @Ú
max_periodÚmeanÚloss_reductionc                 óà  •— t        | j                  t        «      r:| j                  j                  dd«      }t	        |   di | j                  ¤Ž| _        n5| j                  €)t	        d   ddddddd	d
œdddddddd	ddœ	dd	¬«      | _        t        | j
                  t        «      r:| j
                  j                  dd«      }t	        |   di | j
                  ¤Ž| _        nD| j
                  €8t	        d   dddddd| j                  j                  j                  ¬«      | _        d| j
                  _        d| j
                  _	        d| j                  j                  _	        t        ‰| �,  di |¤Ž y )NÚ
model_typeÚ	paligemmaÚgemmai   é   i @  é   é   i€ì )rª   Úhidden_sizeÚnum_hidden_layersÚintermediate_sizeÚnum_attention_headsÚnum_key_value_headsÚ
vocab_sizeÚsiglip_vision_modeliÐ  i€  é   r"   é   é   F)	rª   r²   r°   Ú
patch_sizeÚ
image_sizer±   r³   rµ   Úvision_use_head)Útext_configÚvision_configÚprojection_dimÚimage_token_idi   i   é   )r°   r±   r²   r³   r´   Úhead_dimrµ   Tr-   )rp   r›   ÚdictrK   r   rœ   r½   rµ   Ú	is_causalÚuse_bidirectional_attentionrM   Ú__post_init__)rO   rS   Úvlm_model_typeÚdit_model_typerT   s       €r/   rÆ   zPI0Config.__post_init__é   sX  ø€ Ü�d—o‘o¤tÔ,Ø!Ÿ_™_×0Ñ0°¸{ÓKˆNÜ,¨^Ñ<ÑO¸t¿¹ÑOˆD�OØ�_‰_Ð$Ü,¨[Ñ9à")Ø#'Ø)+Ø).Ø+,Ø+,Ø"(ñð #8Ø)-Ø#'Ø"$Ø"%Ø)+Ø+-Ø"(Ø',ñ
ð  $Ø%ô-ˆDŒOô2 �d—o‘o¤tÔ,Ø!Ÿ_™_×0Ñ0°¸wÓGˆNÜ,¨^Ñ<ÑO¸t¿¹ÑOˆD�OØ�_‰_Ð$Ü,¨WÑ5Ø Ø"$Ø"&Ø$%Ø$%ØØŸ?™?×6Ñ6×AÑAôˆDŒOð %)ˆ�‰Ô!Ø6:ˆ�‰Ô3ØBFˆ�‰×#Ñ#Ô?Ü‰ÑÑ' Ó'r.   c                 ó–   — | j                   j                  dz  dk7  r-t        d| j                  j                   j                  › d�«      ‚y)zOPart of `@strict`-powered validation. Validates the architecture of the config.r   r   zDiT hidden dim=(z) must be divisible by 2N)rœ   r°   rr   Úconfig©rO   s    r/   Úvalidate_architecturezPI0Config.validate_architecture  sE   € à�?‰?×&Ñ&¨Ñ*¨aÒ/ÜÐ/°·±×0FÑ0F×0RÑ0RÐ/SÐSkÐlÓmÐmð 0r.   )r'   r(   r)   Ú__doc__rª   r   Úsub_configsr›   rÃ   r	   Ú__annotations__rœ   rI   ÚintrG   r�   rŸ   r    Úfloatr¢   r£   r¤   r¥   r¦   r¨   rq   rÆ   rÌ   r•   r–   s   @r/   r™   r™   °   sÃ   ø… ñ#ðJ €JØ!+¸:ÑF€Kà15€J�Ð'Ñ'¨$Ñ.Ó5Ø15€J�Ð'Ñ'¨$Ñ.Ó5Ø€J�ÓØ€M�3ÓØ€N�CÓØ!Ð˜Ó!Ø&)Ð˜eÓ)Ø%(Ð˜UÓ(Ø!&Ð˜Ó&Ø"'Ð˜%Ó'Ø€J�ÓØ€J�ÓØ €N�CÓ ô0(ödnr.   r™   Úblock_boundariesrY   c           
      óP   ‡ — dt         dt         dt         dt         dt        f
ˆ fd„}|S )NÚ	batch_idxÚhead_idxÚq_idxÚkv_idxrY   c                 óf   •— t        j                  |‰«      }t        j                  |‰«      }||k  S r`   )r>   Ú	bucketize)rÔ   rÕ   rÖ   r×   Úq_blockÚkv_blockrÒ   s         €r/   Ú
inner_maskz0blockwise_bidirectional_mask.<locals>.inner_mask"  s0   ø€ Ü—/‘/ %Ð)9Ó:ˆÜ—?‘? 6Ð+;Ó<ˆØ˜7Ñ"Ð"r.   )rÐ   r{   )rÒ   rÜ   s   ` r/   Úblockwise_bidirectional_maskrÝ   !  s3   ø€ ð#œcð #¬Sð #¼ð #Äcð #Ìdõ #ð
 Ðr.   c                   ó4   ‡ — e Zd Zˆ fd„Zed„ «       Zd„ Zˆ xZS )ÚPI0TimestepEmbeddingsc                 óz   •— t         ‰| �  «        || _        | j                  |«      }| j	                  d|d¬«       y )NÚsinusoid_freqF)Ú
persistent)rM   rN   rÊ   Úcompute_freqsÚregister_buffer)rO   rÊ   rá   rT   s      €r/   rN   zPI0TimestepEmbeddings.__init__+  s:   ø€ Ü‰ÑÔØˆŒØ×*Ñ*¨6Ó2ˆØ×Ñ˜_¨mÈÐÕNr.   c                 ó  — t        j                  dd| j                  j                  dz  t         j                  ¬«      }| j
                  | j                  | j
                  z  |z  z  }d|z  dz  t        j                  z  }|S )Nç        r¡   r   re   )	r>   Úlinspacerœ   r°   Úfloat32r¥   r¦   ÚmathÚpi)rÊ   ÚfractionÚperiodrá   s       r/   rã   z#PI0TimestepEmbeddings.compute_freqs1  sr   € ä—>‘> # s¨F×,=Ñ,=×,IÑ,IÈQÑ,NÔV[×VcÑVcÔdˆØ×"Ñ" f×&7Ñ&7¸&×:KÑ:KÑ&KÐPXÑ%XÑXˆØ˜f™ qÑ(¬4¯7©7Ñ2ˆØÐr.   c                 óž  — t        |j                  j                  t        «      r/|j                  j                  dk7  r|j                  j                  nd}t	        |d¬«      5  | j
                  d d d …f   }||d d …d f   z  }t        j                  |j                  «       |j                  «       gd¬«      }d d d «       |S # 1 sw Y   S xY w)NÚmpsÚcpuF)Údevice_typeÚenabledr¯   ©Údim)
rp   ÚdeviceÚtyperq   r   rá   r>   ÚcatÚsinÚcos)rO   Útimerð   rá   ÚembÚtime_embedss         r/   ÚforwardzPI0TimestepEmbeddings.forward8  s°   € Ü*4°T·[±[×5EÑ5EÄsÔ*KÐPT×P[ÑP[×P`ÑP`ÐdiÒPi�d—k‘k×&Ò&ÐotˆÜ¨¸UÔCñ 	CØ ×.Ñ.¨t²Q¨wÑ7ˆMØ $¢q¨$ w¡-Ñ/ˆCÜŸ)™) S§W¡W£Y°·±³	Ð$:ÀÔBˆK÷	Cð Ð÷		Cð Ðús   Á#ACÃC)r'   r(   r)   rN   Ústaticmethodrã   rü   r•   r–   s   @r/   rß   rß   *  s"   ø„ ôOð ñó ðör.   rß   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚPI0ActionTimeEmbeddingc                 ó8  •— t         ‰| �  «        t        |«      | _        t	        j
                  |j                  |j                  j                  «      | _	        t	        j
                  |j                  |j                  j                  «      | _        t	        j
                  d|j                  j                  z  |j                  j                  «      | _        t	        j
                  |j                  j                  |j                  j                  «      | _        y )Nr   )rM   rN   rß   Úsinusoid_embedsr   ÚLinearr�   rœ   r°   Úaction_in_projrG   Ú
state_projÚaction_time_mlp_inÚaction_time_mlp_out©rO   rÊ   rT   s     €r/   rN   zPI0ActionTimeEmbedding.__init__B  s¿   ø€ Ü‰ÑÔÜ4°VÓ<ˆÔÜ Ÿi™i¨×(=Ñ(=¸v×?PÑ?P×?\Ñ?\Ó]ˆÔÜŸ)™) F×$8Ñ$8¸&×:KÑ:K×:WÑ:WÓXˆŒÜ"$§)¡)¨A°×0AÑ0A×0MÑ0MÑ,MÈv×O`ÑO`×OlÑOlÓ"mˆÔÜ#%§9¡9¨V×->Ñ->×-JÑ-JÈF×L]ÑL]×LiÑLiÓ#jˆÕ r.   c                 ó´  — | j                  |«      }| j                  |«      }| j                  |«      }|d d …d d d …f   j                  |«      j	                  |j
                  ¬«      }t        j                  ||gd¬«      }| j                  t        j                  | j                  |«      «      «      }t        j                  |d d …d d d …f   |gd¬«      }|S )Nre   r   rò   r¯   )r  r  r  Ú	expand_asÚtorf   r>   rö   r  r~   Úsilur  )	rO   rX   ÚnoiseÚtimestepÚstate_embedsÚaction_embedsrû   Úaction_time_embedsÚaction_embeds_mergeds	            r/   rü   zPI0ActionTimeEmbedding.forwardJ  sÅ   € Ø—‘ uÓ-ˆØ×+Ñ+¨EÓ2ˆà×*Ñ*¨8Ó4ˆØ!¢! Tª1 *Ñ-×7Ñ7¸ÓF×IÑIÐP]×PcÑPcÐIÓdˆä"ŸY™Y¨°{Ð'CÈÔKÐØ!×5Ñ5´a·f±f¸T×=TÑ=TÐUgÓ=hÓ6iÓjÐÜ$Ÿy™y¨,²q¸$Â°zÑ*BÐDVÐ)WÐ]^Ô_ÐØ#Ð#r.   )r'   r(   r)   rN   rü   r•   r–   s   @r/   rÿ   rÿ   A  s   ø„ ôkö
$r.   rÿ   c                   óT   ‡ — e Zd ZU eed<   dZdZdZdgZdZ	dZ
dZdZdZdZˆ fd„Zˆ xZS )ÚPI0PreTrainedModelrÊ   ÚmodelrX   TÚpast_key_values)ÚimagerV   c                 ó¸   •— t         ‰| �  |«       t        |t        «      r:t	        j
                  |j                  |j                  |j                  «      «       y y r`   )	rM   Ú_init_weightsrp   rß   ÚinitÚcopy_rá   rã   rÊ   )rO   ÚmodulerT   s     €r/   r  z PI0PreTrainedModel._init_weightse  sC   ø€ Ü‰Ñ˜fÔ%Ü�fÔ3Ô4Ü�J‰J�v×+Ñ+¨V×-AÑ-AÀ&Ç-Á-Ó-PÕQð 5r.   )r'   r(   r)   r™   rÏ   Úbase_model_prefixÚmain_input_nameÚsupports_gradient_checkpointingÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendÚinput_modalitiesr  r•   r–   s   @r/   r  r  W  sR   ø… àÓØÐØ€OØ&*Ð#Ø#4Ð"5ÐØÐØ€NØÐØ!ÐØ"&ÐØ(Ð÷Rð Rr.   r  c                   ó>  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zdd„Zee		 	 	 	 	 	 	 dde
j                  de
j                  dz  d	e
j                  dz  d
e
j                  dz  de
j                  dz  de
j                  dz  de
j                  dz  dedz  defd„«       «       Zˆ xZS )ÚPI0ModelrÊ   c                 óÔ   •— t         ‰| �  |«       t        j                  |j                  «      | _        t        j                  |j                  «      | _        | j                  «        y r`   )	rM   rN   r   Úfrom_configrœ   Úditr›   ÚvlmÚ	post_initr  s     €r/   rN   zPI0Model.__init__m  sJ   ø€ Ü‰Ñ˜Ô Ü×(Ñ(¨×):Ñ):Ó;ˆŒÜ×(Ñ(¨×):Ñ):Ó;ˆŒØ�‰Õr.   c                 ó6   — | j                   j                  «       S r`   )r+  Úget_input_embeddingsrË   s    r/   r.  zPI0Model.get_input_embeddingss  s   € Ø�x‰x×,Ñ,Ó.Ð.r.   c                 ó:   — | j                   j                  |«       y r`   )r+  Úset_input_embeddings)rO   Úvalues     r/   r0  zPI0Model.set_input_embeddingsv  s   € Ø�‰×%Ñ% eÕ,r.   Nc                 óÞ  — |j                   d   }|j                  dd«      }| j                  j                  |«      j                  }|j                  d||j                   d   |j                   d   «      }g }t        |«      D ]  \  }}	||   |	   }
|j                  |
«       Œ  t        j                  |d¬«      }|j                  «       }d||| j                  j                  j                  k(  <    | j                  j                  «       |«      }|| j                  j                  j                  k(  j                  d«      j!                  |j"                  «      }|j%                  ||«      }|S )Nr¯   r   ri   r   rò   )r}   Úflattenr+  Úget_image_featuresÚpooler_outputÚreshaper|   rx   r>   rö   ÚclonerÊ   r›   rÀ   r.  Ú	unsqueezer
  rô   Úmasked_scatter)rO   Ú	input_idsrg   rh   Úattention_maskr‡   Úimage_featuresÚtotal_image_featuresrÔ   ÚmaskÚunpadded_image_featuresÚllm_input_idsÚinputs_embedsÚspecial_image_masks                 r/   Úembed_prefixzPI0Model.embed_prefixy  s[  € Ø.×4Ñ4°QÑ7ˆØ#×+Ñ+¨A¨qÓ1ˆØŸ™×4Ñ4°\ÓB×PÑPˆØ'×/Ñ/°°OÀ^×EYÑEYÐZ[ÑE\Ð^l×^rÑ^rÐstÑ^uÓvˆà!ÐÜ(Ð)=Ó>ò 	A‰OˆI�tØ&4°YÑ&?ÀÑ&EÐ#Ø ×'Ñ'Ð(?Õ@ð	Aô  %Ÿy™yÐ)=À1ÔEÐà!Ÿ™Ó)ˆØLMˆ�i 4§;¡;×#9Ñ#9×#HÑ#HÑHÑIØ7˜Ÿ™×5Ñ5Ó7¸ÓFˆà˜$Ÿ+™+×0Ñ0×?Ñ?Ñ?×JÑJÈ2ÓN×QÑQÐR_×RfÑRfÓgð 	ð &×4Ñ4Ð5GÐI]Ó^ˆàÐr.   r  r:  rg   r;  rh   Úposition_idsrA  r  rY   c	                 ó:  — |�n|€l|�|€|j                  d«      dz
  }|€| j                  |||«      }t        j                  |«      dd…dd…df   }
| j	                  ||||
d¬«      j
                  }|�|j                  dk7  rt        d«      ‚dx}}|�‘t        j                  |j                  d   |j                  d   |j                  |j                  ¬	«      }t        j                  ||gd¬
«      }t        j                   |d¬
«      dz
  dd…|j                  d    d…f   }|j                  «       }t        j                  t        j                  |dz   |j                  t        j                  ¬«      t        j                  |j                  d   dz
  |j                  t        j                  ¬«      g«      }|ddd…f   j!                  |j                  d   d«      }t#        | j$                  j&                  ||||¬«      } | j(                  d||||dœ|	¤Ž}|S )zû
        action_embeds (`torch.Tensor`, *optional*):
            The embeddings of input actions and robot states.
        pixel_attention_mask (`torch.Tensor`, *optional*):
            The mask indicating padded positions in the input image.
        Nri   r¯   r   T)rA  r;  rD  Útoken_type_idsÚ	use_cacher   z:Only two-dimensional attention masks are accepted for now!©rf   rô   rò   ©rô   rf   )rÊ   rA  r;  r  Úblock_sequence_ids)rA  r;  rD  r  r-   )ÚcumsumrC  r>   Ú
zeros_liker+  r  Úndimrr   Úonesr}   rf   rô   rö   Úget_seq_lengthrz   ÚlongÚrepeatr   rÊ   rœ   r*  )rO   r  r:  rg   r;  rh   rD  rA  r  rS   rF  Údit_position_idsÚdit_attention_maskÚ
noise_maskÚvlm_input_lengthrJ  Úbidirectional_maskÚ
dit_outputs                     r/   rü   zPI0Model.forward�  sI  € ð( Ð#¨Ð(?ØÐ)¨lÐ.BØ-×4Ñ4°RÓ8¸1Ñ<�àÐ$Ø $× 1Ñ 1°)¸\ÐK_Ó `�ô #×-Ñ-¨mÓ<ºQÂÀ1¸WÑEˆNØ"Ÿh™hØ+Ø-Ø)Ø-Øð 'ó ÷ ‰oð ð Ð%¨.×*=Ñ*=ÀÒ*BÜÐYÓZÐZð 15Ð4ÐÐ-ØÐ%ÜŸ™Ø×#Ñ# AÑ&Ø×#Ñ# AÑ&Ø$×*Ñ*Ø%×,Ñ,ô	ˆJô "'§¡¨N¸JÐ+GÈQÔ!OÐÜ %§¡Ð-?ÀQÔ GÈ!Ñ KÊQÐQ^×QdÑQdÐefÑQgÐPgÑPiÐMiÑjÐð +×9Ñ9Ó;ÐÜ"ŸY™Yä—‘Ð,¨qÑ0¸×9MÑ9MÔUZ×U_ÑU_Ô`Ü—
‘
˜=×.Ñ.¨qÑ1°AÑ5¸m×>RÑ>RÔZ_×ZdÑZdÔeðó
Ðð 0°²a°Ñ8×?Ñ?À×@SÑ@SÐTUÑ@VÐXYÓZÐÜ/Ø—;‘;×)Ñ)Ø'Ø-Ø+Ø1ô
Ðð �T—X‘Xð 
Ø'Ø-Ø)Ø+ñ	
ð
 ñ
ˆ
ð Ðr.   r`   )NNNNNNN)r'   r(   r)   r™   rN   r.  r0  rC  r   r   r>   r“   Ú
LongTensorr   r   rü   r•   r–   s   @r/   r'  r'  k  sô   ø„ ð˜yõ ò/ò-óð, Øð *.Ø,0Ø.2Ø48Ø04Ø-1Ø(,ñKà—|‘|ðKð —<‘< $Ñ&ðKð —l‘l TÑ)ð	Kð
 Ÿ™ tÑ+ðKð $Ÿl™l¨TÑ1ðKð ×&Ñ&¨Ñ-ðKð —|‘| dÑ*ðKð  ™ðKð 
!òKó ó ôKr.   r'  c                   óŽ  ‡ — e Zd ZdZddiZdefˆ fd„Zee	 	 	 	 	 	 	 	 	 	 dde	j                  de	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  dedz  de	j                  defd„«       «       Z e	j"                  «       	 	 	 	 dde	j                  d
e	j                  de	j                  de	j                  dz  de	j                  dz  de	j                  dz  dedz  de	j                  fd„«       Zˆ xZS )ÚPI0ForConditionalGenerationz9PI0 model with action projection heads and flow matching.Úaction_out_projÚcolwise_gather_outputrÊ   c                 ó  •— t         ‰| �  |«       t        |«      | _        |j                  j
                  | _        t        |«      | _        t        j                  | j                  |j                  «      | _        | j                  «        y r`   )rM   rN   r'  r  rœ   r°   Úexpert_hidden_sizerÿ   Úembed_action_timer   r  r�   r[  r,  r  s     €r/   rN   z$PI0ForConditionalGeneration.__init__ä  sf   ø€ Ü‰Ñ˜Ô Ü˜fÓ%ˆŒ
Ø"(×"3Ñ"3×"?Ñ"?ˆÔÜ!7¸Ó!?ˆÔÜ!Ÿy™y¨×)@Ñ)@À&×BWÑBWÓXˆÔØ�‰Õr.   NrX   r  r  r:  rg   rh   r;  rD  rA  r  rW   rY   c                 óÎ  — |j                   d   }|€ût        j                  | j                  j                  t        j
                  ¬«      }t        j                  | j                  j                  t        j
                  ¬«      }t        j                  j                  ||«      }|j                  |f«      j                  |j                  «      }|| j                  j                  z  | j                  j                  z   j                  «       }|€Vt        j                  || j                  j                   | j                  j"                  |j                  |j$                  ¬«      }|�7|dd…ddf   }||z  d|z
  |z  z   j                  |j$                  «      }||z
  }n|}| j'                  |||«      } | j(                  d	||||||	||
dœ|¤Ž}|j*                  dd…| j                  j                    d…f   }| j-                  |«      }d}|�,t/        j0                  || j                  j2                  ¬«      }t5        |||j6                  |j8                  |j:                  ¬«      S )
a-  
        state (`torch.Tensor`, *optional*):
            Current robot state.
        noise (`torch.Tensor`, *optional*):
            Random noise at current timestep that needs to be denoised
        timestep (`torch.Tensor`, *optional*):
            Current denoising timestep.
        pixel_attention_mask (`torch.Tensor`, *optional*):
            The mask indicating padded positions in the input image.
        actions (`torch.Tensor`, *optional*):
            Input actions that need to be predicted. Used only when training to compiute loss.
        r   Nre   rI  r¯   )r:  rg   r;  rh   rD  rA  r  r  )Ú	reduction)ÚlossÚlogitsr  Úhidden_statesÚ
attentionsr-   )r}   r>   rL   rÊ   r    rè   r¢   ÚdistributionsÚBetar„   r
  rô   r£   r¤   rÑ   ÚrandnrI   r�   rf   r_  r  Úlast_hidden_stater[  r~   Úmse_lossr¨   r   r  rd  re  )rO   rX   r  r  r:  rg   rh   r;  rD  rA  r  rW   rS   Ú
batch_sizeÚalpha_tÚbeta_tÚdistÚ	time_betaÚtime_expandedÚnoisy_actionsÚtarget_velocityr  ÚoutputsÚlast_hidden_statesÚpredicted_velocityrb  s                             r/   rü   z#PI0ForConditionalGeneration.forwardì  s'  € ð: —[‘[ ‘^ˆ
ð ÐÜ—l‘l 4§;¡;×#GÑ#GÌuÏ}É}Ô]ˆGÜ—\‘\ $§+¡+×"EÑ"EÌUÏ]É]Ô[ˆFÜ×&Ñ&×+Ñ+¨G°VÓ<ˆDØŸ™ Z MÓ2×5Ñ5°e·l±lÓCˆIØ! D§K¡K×$CÑ$CÑCÀdÇkÁk×FfÑFfÑf×mÑmÓoˆHð ˆ=Ü—K‘KØØ—‘×&Ñ&Ø—‘×*Ñ*Ø—|‘|Ø—k‘kôˆEð ÐØ$¢Q¨¨d ]Ñ3ˆMØ*¨UÑ2°a¸-Ñ6GÈ7Ñ5RÑR×VÑVÐW^×WdÑWdÓeˆMØ# g™o‰Oà!ˆMð "×3Ñ3°E¸=È(ÓSÐà�$—*‘*ð 

ØØ%Ø)Ø!5Ø%Ø'Ø,Ø+ñ

ð ñ

ˆð %×6Ñ6²q¸4¿;¹;×;QÑ;QÐ:QÑ:SÐ7SÑTÐØ!×1Ñ1Ð2DÓEÐàˆØÐä—:‘:˜oÐ/AÈTÏ[É[×MgÑMgÔhˆDä%ØØ%Ø#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r.   Ú	num_stepsc                 ó8  — |xs | j                   j                  }|j                  d   }	|j                  }
|€Ot	        j
                  dd|	| j                   j                  | j                   j                  f|j                  |
¬«      }d}|�|j                  d«      dz
  }| j                  j                  |||«      }t	        j                  |«      dd…dd…df   }| j                  j                  ||||dd¬	«      j                  }|j                  «       }d
|z  }t!        |«      D ]q  }d||z  z   }t	        j"                  |t        j$                  |
¬«      j'                  |	«      } | d||||||dœ|¤Ž}|j)                  |«       |||j*                  z  z   }Œs |S )z0Run flow matching inference to generate actions.r   Nræ   r¡   )r§   Ústdr*   rf   rô   ri   r¯   T)rA  r;  rD  rF  rG  Úreturn_dictg      ð¿rH  )rX   r  r  rh   r;  r  r-   )rÊ   rŸ   r}   rô   r>   ÚnormalrI   r�   rf   rK  r  rC  rL  r+  r  rO  ÚrangerL   rè   ÚexpandÚcroprc  )rO   rX   r:  rg   r  r;  rh   rv  rS   rk  rô   rD  rA  rF  r  Úprefix_lengthÚdtÚsteprù   Útime_tensorÚoutputs                        r/   Úsample_actionsz*PI0ForConditionalGeneration.sample_actionsD  s¹  € ð Ò@ §¡×!@Ñ!@ˆ	Ø—_‘_ QÑ'ˆ
Ø×!Ñ!ˆð ˆ=Ü—L‘LØØàØ—K‘K×*Ñ*Ø—K‘K×.Ñ.ðð
 #×(Ñ(Øô
ˆEð ˆØÐ%Ø)×0Ñ0°Ó4°qÑ8ˆLØŸ
™
×/Ñ/°	¸<ÐI]Ó^ˆÜ×)Ñ)¨-Ó8ººA¸q¸ÑAˆØŸ*™*Ÿ.™.Ø'Ø)Ø%Ø)ØØð )ó 
÷ ‰/ð 	ð (×6Ñ6Ó8ˆð �IÑˆÜ˜)Ó$ò 	/ˆDØ˜ ™‘?ˆDÜŸ,™, t´5·=±=ÈÔP×WÑWÐXbÓcˆKÙð ØØØ$Ø%9Ø-Ø /ñð ñˆFð × Ñ  Ô/Ø˜B §¡Ñ.Ñ.‰Eð	/ð  ˆr.   )
NNNNNNNNNN)NNNN)r'   r(   r)   rÍ   Ú_tp_planr™   rN   r   r   r>   ÚFloatTensorr“   Ú
BoolTensorrX  r   r   rü   Úno_gradrÐ   rƒ  r•   r–   s   @r/   rZ  rZ  ß  sû  ø„ ÙCà!Ð#:Ð;€Hð˜yõ ð Øð +/Ø-1Ø)-Ø,0Ø8<Ø.2Ø04Ø-1Ø(,Ø%)ñT
à× Ñ ðT
ð × Ñ  4Ñ'ðT
ð ×#Ñ# dÑ*ð	T
ð
 —<‘< $Ñ&ðT
ð —l‘l TÑ)ðT
ð $×.Ñ.°Ñ5ðT
ð Ÿ™ tÑ+ðT
ð ×&Ñ&¨Ñ-ðT
ð —|‘| dÑ*ðT
ð  ™ðT
ð ×"Ñ"ðT
ð 
 òT
ó ó ðT
ðl €U‡]�]ƒ_ð +/Ø.2Ø8<Ø $ñAà× Ñ ðAð ×#Ñ#ðAð ×'Ñ'ð	Að
 × Ñ  4Ñ'ðAð Ÿ™ tÑ+ðAð $×.Ñ.°Ñ5ðAð ˜‘:ðAð 
×	Ñ	òAó ôAr.   rZ  )r™   r  r'  rZ  rA   r!   )FrÍ   ré   Úcollections.abcr   Únumpyr‘   r>   Útorch.nn.functionalr   Ú
functionalr~   Úhuggingface_hub.dataclassesr   r\   r   r  Úcache_utilsr   Úconfiguration_utilsr	   Úfeature_extraction_utilsr
   Úimage_utilsr   r   Úmasking_utilsr   Úmodeling_outputsr   r   Úmodeling_utilsr   Úprocessing_utilsr   r   Útokenization_utils_baser   r   Úutilsr   r   r   Úutils.genericr   Úutils.import_utilsr   Úautor   r   r   Úpaligemma.processing_paligemmar   Úsiglip.image_processing_siglipr   Ú
get_loggerr'   rn   r!   r1   rA   r™   r“   rÝ   ÚModulerß   rÿ   r  r'  rZ  Ú__all__r-   r.   r/   ú<module>rŸ     s†  ðñ [ã Ý $ã Û ß Ð Ý .Ý å &Ý  Ý 3Ý 4ß AÝ /ß OÝ -ß 8ß Cß >Ñ >Ý +Ý *ß 8Ñ 8Ý ?Ý Að 
ˆ×	Ñ	˜HÓ	%€ð ôÐ,ó ó ðôÐ)°õ ð Ù	Ð&Ô'ôkDÐ%ó kDó (ó ðkDñ\ Ð-Ô.ØôlnÐ ó lnó ó /ðlnð^°5·<±<ð ÀHó ô˜BŸI™Iô ô.$˜RŸY™Yô $ð, ôR˜ó Ró ðRð& ôpÐ!ó pó ðpôfgÐ"4ô gòT�r.   