
    ^jA                     X   d dl Z d dlZd dlmc mZ d dlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZ  G d dej6                        Z G d dej6                        Ze G d de             Ze G d de             Z G d de      Z g dZ!y)    N)nn   )initialization)Cache)create_causal_mask)BaseModelOutputWithPastCausalLMOutputWithPast)PreTrainedModel)auto_docstringcan_return_tuple)maybe_autocast   )	AutoModel   )	PI0Configc                   4     e Zd Z fdZed        Zd Z xZS )PI0TimestepEmbeddingsc                 z    t         |           || _        | j                  |      }| j	                  d|d       y )Nsinusoid_freqF)
persistent)super__init__configcompute_freqsregister_buffer)selfr   r   	__class__s      o/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/pi0/modeling_pi0.pyr   zPI0TimestepEmbeddings.__init__'   s:    **62_mN    c                    t        j                  dd| j                  j                  dz  t         j                        }| j
                  | j                  | j
                  z  |z  z  }d|z  dz  t        j                  z  }|S )N              ?r   dtype)	torchlinspace
dit_confighidden_sizefloat32
min_period
max_periodmathpi)r   fractionperiodr   s       r   r   z#PI0TimestepEmbeddings.compute_freqs-   sr    >>#sF,=,=,I,IQ,NV[VcVcd""f&7&7&:K:K&KPX%XXfq(4772r   c                    t        |j                  j                  t              r/|j                  j                  dk7  r|j                  j                  nd}t	        |d      5  | j
                  d d d f   }||d d d f   z  }t        j                  |j                         |j                         gd      }d d d        |S # 1 sw Y   S xY w)NmpscpuF)device_typeenabledr   dim)

isinstancedevicetypestrr   r   r%   catsincos)r   timer3   r   embtime_embedss         r   forwardzPI0TimestepEmbeddings.forward4   s    *4T[[5E5Es*KPTP[P[P`P`diPidkk&&otUC 	C ..tQw7M$q$w-/C))SWWY	$:BK	C 		C s   #ACC)__name__
__module____qualname__r   staticmethodr   rA   __classcell__r   s   @r   r   r   &   s"    O  r   r   c                   $     e Zd Z fdZd Z xZS )PI0ActionTimeEmbeddingc                 8   t         |           t        |      | _        t	        j
                  |j                  |j                  j                        | _	        t	        j
                  |j                  |j                  j                        | _        t	        j
                  d|j                  j                  z  |j                  j                        | _        t	        j
                  |j                  j                  |j                  j                        | _        y )Nr   )r   r   r   sinusoid_embedsr   Linearmax_action_dimr'   r(   action_in_projmax_state_dim
state_projaction_time_mlp_inaction_time_mlp_outr   r   r   s     r   r   zPI0ActionTimeEmbedding.__init__>   s    4V< ii(=(=v?P?P?\?\]))F$8$8&:K:K:W:WX"$))A0A0A0M0M,MvO`O`OlOl"m#%99V->->-J-JFL]L]LiLi#j r   c                    | j                  |      }| j                  |      }| j                  |      }|d d d d d f   j                  |      j	                  |j
                        }t        j                  ||gd      }| j                  t        j                  | j                  |                  }t        j                  |d d d d d f   |gd      }|S )Nr#   r   r5   r   )rP   rN   rK   	expand_astor$   r%   r;   rR   FsilurQ   )	r   statenoisetimestepstate_embedsaction_embedsr@   action_time_embedsaction_embeds_mergeds	            r   rA   zPI0ActionTimeEmbedding.forwardF   s    u-++E2**84!!T1*-77FIIP]PcPcId"YY{'CK!55affT=T=TUg=h6ij$yy,q$z*BDV)W]^_##r   )rB   rC   rD   r   rA   rF   rG   s   @r   rI   rI   =   s    k
$r   rI   c                   T     e Zd ZU eed<   dZdZdZdgZdZ	dZ
dZdZdZdZ fdZ xZS )PI0PreTrainedModelr   modelrY   Tpast_key_values)imagetextc                     t         |   |       t        |t              r:t	        j
                  |j                  |j                  |j                               y y N)	r   _init_weightsr7   r   initcopy_r   r   r   )r   moduler   s     r   rh   z PI0PreTrainedModel._init_weightsa   sC    f%f34JJv++V-A-A&---PQ 5r   )rB   rC   rD   r   __annotations__base_model_prefixmain_input_namesupports_gradient_checkpointing_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraph_supports_attention_backendinput_modalitiesrh   rF   rG   s   @r   ra   ra   S   sR    O&*##4"5N!"&(R Rr   ra   c                   >    e Zd Zdef fdZd Zd ZddZee		 	 	 	 	 	 	 dde
j                  de
j                  dz  d	e
j                  dz  d
e
j                  dz  de
j                  dz  de
j                  dz  de
j                  dz  dedz  defd              Z xZS )PI0Modelr   c                     t         |   |       t        j                  |j                        | _        t        j                  |j                        | _        | j                          y rg   )	r   r   r   from_configr'   dit
vlm_configvlm	post_initrS   s     r   r   zPI0Model.__init__i   sJ     (():):;(():):;r   c                 6    | j                   j                         S rg   )r}   get_input_embeddings)r   s    r   r   zPI0Model.get_input_embeddingso   s    xx,,..r   c                 :    | j                   j                  |       y rg   )r}   set_input_embeddings)r   values     r   r   zPI0Model.set_input_embeddingsr   s    %%e,r   Nc                    |j                   d   }|j                  dd      }| j                  j                  |      j                  }|j                  d||j                   d   |j                   d         }g }t        |      D ]  \  }}	||   |	   }
|j                  |
         t        j                  |d      }|j                         }d||| j                  j                  j                  k(  <    | j                  j                         |      }|| j                  j                  j                  k(  j                  d      j!                  |j"                        }|j%                  ||      }|S )Nr   r   r   r5   )shapeflattenr}   get_image_featurespooler_outputreshape	enumerateappendr%   r;   cloner   r|   image_token_idr   	unsqueezerV   r8   masked_scatter)r   	input_idspixel_valuespixel_attention_maskattention_maskmax_num_camerasimage_featurestotal_image_features	batch_idxmaskunpadded_image_featuresllm_input_idsinputs_embedsspecial_image_masks                 r   embed_prefixzPI0Model.embed_prefixu   s[   .44Q7#++Aq144\BPP'//O^EYEYZ[E\^l^r^rst^uv!()=> 	AOIt&4Y&?&E# ''(?@	A  %yy)=1E!)LMi4;;#9#9#H#HHI7557F$++00???JJ2NQQR_RfRfg 	 &445GI]^r   r]   r   r   r   r   position_idsr   rc   returnc	                 :   |n|l|||j                  d      dz
  }|| j                  |||      }t        j                  |      dddddf   }
| j	                  ||||
d      j
                  }||j                  dk7  rt        d      dx}}|t        j                  |j                  d   |j                  d   |j                  |j                  	      }t        j                  ||gd
      }t        j                   |d
      dz
  dd|j                  d    df   }|j                         }t        j                  t        j                  |dz   |j                  t        j                        t        j                  |j                  d   dz
  |j                  t        j                        g      }|dddf   j!                  |j                  d   d      }t#        | j$                  j&                  ||||      } | j(                  d||||d|	}|S )z
        action_embeds (`torch.Tensor`, *optional*):
            The embeddings of input actions and robot states.
        pixel_attention_mask (`torch.Tensor`, *optional*):
            The mask indicating padded positions in the input image.
        Nr   r   r   T)r   r   r   token_type_ids	use_cacher   z:Only two-dimensional attention masks are accepted for now!r$   r8   r5   r8   r$   )r   r   r   rc   block_sequence_ids)r   r   r   rc    )cumsumr   r%   
zeros_liker}   rc   ndim
ValueErroronesr   r$   r8   r;   get_seq_lengthzeroslongrepeatr   r   r'   r{   )r   r]   r   r   r   r   r   r   rc   kwargsr   dit_position_idsdit_attention_mask
noise_maskvlm_input_lengthr   bidirectional_mask
dit_outputs                     r   rA   zPI0Model.forward   sI   ( #(?)l.B-44R81<$ $ 1 1)\K_ ` #--m<Q1WEN"hh+-)- '  o  %.*=*=*BYZZ 154-%##A&##A&$**%,,	J "'NJ+GQ!O %-?Q G! KQQ^QdQdefQgPgPiMij +99;"YY,q09M9MUZU_U_`

=..q1A5m>R>RZ_ZdZde
 0a8??@S@STU@VXYZ/;;))'-+1
 TXX 
'-)+	

 

 r   rg   )NNNNNNN)rB   rC   rD   r   r   r   r   r   r   r   r%   Tensor
LongTensorr   r   rA   rF   rG   s   @r   rx   rx   g   s    y /-,  *.,0.24804-1(,K||K <<$&K llT)	K
 t+K $llT1K &&-K ||d*K K 
!K  Kr   rx   c                       e Zd ZdZddiZdef fdZee	 	 	 	 	 	 	 	 	 	 dde	j                  de	j                  dz  d	e	j                  dz  d
e	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  de	j                  dz  dedz  de	j                  defd              Z e	j"                         	 	 	 	 dde	j                  d
e	j                  de	j                  de	j                  dz  de	j                  dz  de	j                  dz  dedz  de	j                  fd       Z xZS )PI0ForConditionalGenerationz9PI0 model with action projection heads and flow matching.action_out_projcolwise_gather_outputr   c                    t         |   |       t        |      | _        |j                  j
                  | _        t        |      | _        t        j                  | j                  |j                        | _        | j                          y rg   )r   r   rx   rb   r'   r(   expert_hidden_sizerI   embed_action_timer   rL   rM   r   r~   rS   s     r   r   z$PI0ForConditionalGeneration.__init__   sf     f%
"("3"3"?"?!7!?!yy)@)@&BWBWXr   NrY   rZ   r[   r   r   r   r   r   r   rc   actionsr   c                    |j                   d   }|t        j                  | j                  j                  t        j
                        }t        j                  | j                  j                  t        j
                        }t        j                  j                  ||      }|j                  |f      j                  |j                        }|| j                  j                  z  | j                  j                  z   j                         }|Vt        j                  || j                  j                   | j                  j"                  |j                  |j$                        }|7|ddddf   }||z  d|z
  |z  z   j                  |j$                        }||z
  }n|}| j'                  |||      } | j(                  d	||||||	||
d|}|j*                  dd| j                  j                    df   }| j-                  |      }d}|,t/        j0                  || j                  j2                        }t5        |||j6                  |j8                  |j:                        S )
a-  
        state (`torch.Tensor`, *optional*):
            Current robot state.
        noise (`torch.Tensor`, *optional*):
            Random noise at current timestep that needs to be denoised
        timestep (`torch.Tensor`, *optional*):
            Current denoising timestep.
        pixel_attention_mask (`torch.Tensor`, *optional*):
            The mask indicating padded positions in the input image.
        actions (`torch.Tensor`, *optional*):
            Input actions that need to be predicted. Used only when training to compiute loss.
        r   Nr#   r   r   )r   r   r   r   r   r   r]   rc   )	reduction)losslogitsrc   hidden_states
attentionsr   )r   r%   tensorr   time_sampling_beta_alphar)   time_sampling_beta_betadistributionsBetasamplerV   r8   time_sampling_scaletime_sampling_offsetfloatrandn
chunk_sizerM   r$   r   rb   last_hidden_stater   rW   mse_lossloss_reductionr	   rc   r   r   )r   rY   rZ   r[   r   r   r   r   r   r   rc   r   r   
batch_sizealpha_tbeta_tdist	time_betatime_expandednoisy_actionstarget_velocityr^   outputslast_hidden_statespredicted_velocityr   s                             r   rA   z#PI0ForConditionalGeneration.forward   s'   : [[^
 ll4;;#G#Gu}}]G\\$++"E"EU]][F&&++GV<DZM255ellCI!DKK$C$CCdkkFfFffmmoH =KK&&**||kkE $Qd]3M*U2a-6G75RRVVW^WdWdeM#goO!M "33E=(S$** 

%)!5%',+

 

 %66q4;;;Q;Q:Q:S7ST!112DE::o/AT[[MgMghD%%#33!//))
 	
r   	num_stepsc                 8   |xs | j                   j                  }|j                  d   }	|j                  }
|Ot	        j
                  dd|	| j                   j                  | j                   j                  f|j                  |
      }d}||j                  d      dz
  }| j                  j                  |||      }t	        j                  |      dddddf   }| j                  j                  ||||dd	      j                  }|j                         }d
|z  }t!        |      D ]q  }d||z  z   }t	        j"                  |t        j$                  |
      j'                  |	      } | d||||||d|}|j)                  |       |||j*                  z  z   }s |S )z0Run flow matching inference to generate actions.r   Nr!   r"   )meanstdsizer$   r8   r   r   T)r   r   r   r   r   return_dictg      r   )rY   rZ   r[   r   r   rc   r   )r   num_inference_stepsr   r8   r%   normalr   rM   r$   r   rb   r   r   r}   rc   r   ranger   r)   expandcropr   )r   rY   r   r   rZ   r   r   r   r   r   r8   r   r   r   rc   prefix_lengthdtstepr>   time_tensoroutputs                        r   sample_actionsz*PI0ForConditionalGeneration.sample_actions@  s    @!@!@	__Q'
!! =LLKK**KK..
 #((
E %)004q8L

//	<I]^))-8AqA**..')%) ) 
 / 	 (668 I)$ 	/D?D,,t5==PWWXbcK $%9- / F   /B..E	/  r   )
NNNNNNNNNN)NNNN)rB   rC   rD   __doc___tp_planr   r   r   r   r%   FloatTensorr   
BoolTensorr   r   r	   rA   no_gradintr   rF   rG   s   @r   r   r      s   C!#:;Hy   +/-1)-,08<.204-1(,%)T
  T
   4'T
 ##d*	T

 <<$&T
 llT)T
 $..5T
 t+T
 &&-T
 ||d*T
 T
 ""T
 
 T
  T
l U]]_ +/.28< $A  A ##A ''	A
   4'A t+A $..5A :A 
		A Ar   r   )ra   rx   r   )"r,   r%   torch.nn.functionalr   
functionalrW    r   ri   cache_utilsr   masking_utilsr   modeling_outputsr   r	   modeling_utilsr
   utilsr   r   utils.genericr   autor   configuration_pi0r   Moduler   rI   ra   rx   r   __all__r   r   r   <module>r     s   *      &   / O - 5 +  (BII .$RYY $, R R R& p! p pfg"4 gT Lr   