
    ^j                     R   d dl mZmZ d dlmZ d dlmZ d dlZd dlm	Z	 d dl
m	c mZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZ ddlmZmZ ddlmZ ddlm Z m!Z!m"Z"m#Z#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z,  e"d      e G d de                    Z- e"d      e G d de                     Z. G d de	j^                        Z0 G d de	j^                        Z1 G d de	j^                        Z2de3d e3d!ejh                  fd"Z5 G d# d$e	j^                        Z6	 	 	 dQd%e	j^                  d&ejh                  d'ejh                  d(ejh                  d)ejh                  dz  d*e7e3z  d+e7dz  d,e7dz  d!e8ejh                  ejh                  f   fd-Z9d.ejh                  d/e3d!ejh                  fd0Z: G d1 d2e	j^                        Z; G d3 d4e	jx                        Z= G d5 d6e	j^                        Z> G d7 d8e      Z?e" G d9 d:e             Z@ e"d;       G d< d=e@             ZAd>ZB G d? d@e	j^                        ZCdRdAej                  d e3dBe7fdCZE e"dD       G dE dFe@             ZF e"dG       G dH dIe@             ZG e"dJ       G dK dLe@             ZH e"dM       G dN dOe@             ZIg dPZJy)S    )CallableIterable)	dataclass)AnyN   )initialization)ACT2FN)create_causal_mask)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPoolingImageClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)ModelOutputTransformersKwargsauto_docstringcan_return_tuple	torch_int)merge_with_config_defaults)capture_outputs   )VideoPrismConfigVideoPrismTextConfigVideoPrismVisionConfigzFBase class for model outputs that include spatial and temporal states.)custom_introc                   b    e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   y)+BaseModelOutputWithSpatialAndTemporalStatesa  
    last_temporal_hidden_state (`torch.FloatTensor`, *optional*):
        The last hidden state of the temporal encoder, typically of shape
        `(batch_size * num_patches, num_frames, hidden_size)`.
    last_spatial_hidden_state (`torch.FloatTensor`, *optional*):
        The last hidden state of the spatial encoder, typically of shape
        `(batch_size * num_frames, num_patches, hidden_size)`.
    Nlast_temporal_hidden_statelast_spatial_hidden_state)	__name__
__module____qualname____doc__r    torchFloatTensor__annotations__r!        }/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/videoprism/modeling_videoprism.pyr   r   +   s6     <@ 1 1D 8?:>u0047>r*   r   z+Base class for VideoPrismClipModel outputs.c                      e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
ej                  dz  ed<   dZeed<   dZeed<   dZej                  dz  ed	<   d
ee   fdZy)VideoPrismClipOutputa  
    logits_per_video (`torch.FloatTensor` of shape `(video_batch_size, text_batch_size)`):
        The scaled dot product scores between `video_embeds` and `text_embeds`. This represents the video-text
        similarity scores.
    logits_per_text (`torch.FloatTensor` of shape `(text_batch_size, video_batch_size)`):
        The scaled dot product scores between `text_embeds` and `video_embeds`. This represents the text-video
        similarity scores.
    video_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)`):
        The video embeddings obtained by applying the projection layer to the pooled output of [`VideoPrismVideoModel`].
    text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)`):
        The text embeddings obtained by applying the projection layer to the pooled output of [`VideoPrismTextModel`].
    video_model_output (`BaseModelOutputWithPooling`):
        The output of [`VideoPrismVideoModel`].
    text_model_output (`BaseModelOutputWithPooling`):
        The output of the [`VideoPrismTextModel`].
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
        Contrastive loss for video-text similarity.
    Nlogits_per_videologits_per_textvideo_embedstext_embedsvideo_model_outputtext_model_outputlossreturnc                 H     t         fd j                         D              S )Nc              3   d   K   | ]'  }|d vr|   nt        |      j                          ) yw))r3   r2   N)getattrto_tuple).0kselfs     r+   	<genexpr>z0VideoPrismClipOutput.to_tuple.<locals>.<genexpr>\   s=      
  KKDGQXY]_`QaQjQjQll
s   -0)tuplekeysr<   s   `r+   r9   zVideoPrismClipOutput.to_tuple[   s#     
YY[
 
 	
r*   )r"   r#   r$   r%   r.   r&   r'   r(   r/   r0   r1   r2   r   r3   r4   r>   r   r9   r)   r*   r+   r-   r-   ;   s    
& 26e''$.504OU&&-4-1L%##d*1,0K""T)059294818%)D%

d
")
%* 
r*   r-   c                   f     e Zd ZdZdef fdZddej                  dedej                  fdZ	 xZ
S )	VideoPrismTubeletEmbeddingsa  
    VideoPrism Tubelet Embeddings.

    The authors of Videoprism use the Factorized Encoder architecture, i.e. "Model 2", introduced in the VIVIT paper (https://huggingface.co/papers/2103.15691).
    This differs from Vivit by using a convolution of `tubelet_size=(1, 18, 18)`, which is essentially a 2d convolution in the spatial dimension.
    The temporal dimension is also merged with the `batch_size` in order to make sure the image embeddings have no temporal component, unlike Vivit.
    configc                    t         |           |j                  }|j                  }t	        |t
              r|n||f}|| _        t        j                  |j                  |j                  ||      | _
        | j                  d   |d   z  | j                  d   |d   z  g| _        | j                  d   | j                  d   z  | _        y )N)kernel_sizestrider   r      )super__init__tubelet_size
image_size
isinstancer   nnConv3dnum_channelshidden_size
projectionpos_emb_shapenum_patches)r<   rC   rJ   rK   	__class__s       r+   rI   z$VideoPrismTubeletEmbeddings.__init__k   s    **&&
#-j(#CZ*V`Ia
$))!3!3Vb
 #ooa0LOCT__UVEW[ghi[jEjk--a043E3Ea3HHr*   pixel_values_videosinterpolate_pos_encodingr5   c                    |j                   \  }}}}}|sV|| j                  d   k7  s|| j                  d   k7  r2t        d| d| d| j                  d    d| j                  d    d	      |j                  dd      }| j	                  |      }|j                  d      j                  dddd      }|j                   \  }}}	}
|j                  ||z  |	|
      }|S )	Nr   r   zImage size (*z) doesn't match model (z[). Set interpolate_pos_encoding=True to automatically resize the model position embeddings.rG   r   )shaperK   
ValueError	transposerQ   flattenpermutereshape)r<   rU   rV   
batch_size
num_framesrO   heightwidthhidden_statesrS   rP   s              r+   forwardz#VideoPrismTubeletEmbeddings.forwardw   s   >Q>W>W;
Jfe'Vtq7I-IUVZVeVefgVhMhvhaw.EdooVWFXEYYZ[_[j[jkl[mZn  oJ  K  2;;AqA(;<%--a088Aq!D;H;N;N8
J[%--j:.E{T_`r*   F)r"   r#   r$   r%   r   rI   r&   Tensorboolrd   __classcell__rT   s   @r+   rB   rB   b   s=    
I5 
I5<< SW didpdp r*   rB   c                        e Zd ZdZdef fdZdej                  dededej                  fdZ		 dd	ej                  d
e
dz  dej                  fdZ xZS )VideoPrismSpatialEmbeddingszY
    Construct the CLS token, position and tubelet patch embeddings for video input.
    rC   c                 P   t         |           t        |      | _        | j                  j                  }t        j                  t        j                  d||j                              | _
        t        j                  |j                        | _        |j                  dd  | _        y Nr   )rH   rI   rB   patch_embeddingsrS   rM   	Parameterr&   zerosrP   position_embeddingsDropouthidden_dropout_probdropoutrJ   
patch_size)r<   rC   rS   rT   s      r+   rI   z$VideoPrismSpatialEmbeddings.__init__   sz     ;F C++77#%<<A{FL^L^0_#` zz&"<"<= --ab1r*   
embeddingsra   rb   r5   c                 8   |j                   d   }| j                  j                   d   }t        j                  j	                         s||k(  r||k(  r| j                  S |j                   d   }|| j
                  d   z  }|| j
                  d   z  }t        |dz        }	| j                  j                  d|	|	|      }
|
j                  dddd      }
t        j                  j                  |
||fdd	      }
|
j                  dddd      j                  dd|      }
|
S )
   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   r         ?r   rG   bilinearTsizemode	antialias)rY   rq   r&   jit
is_tracingru   r   r^   r]   rM   
functionalinterpolateview)r<   rv   ra   rb   rS   num_positionsdimnum_row_patchesnum_col_patchessqrt_num_positionspatch_pos_embeds              r+   rV   z4VideoPrismSpatialEmbeddings.interpolate_pos_encoding   s*    !&&q)0066q9 yy##%+*F6UZ?+++r" DOOA$664??1#55&}c'9:22::1>PRdfij)11!Q1= --33!?3	 4 
 *11!Q1=BB1b#Nr*   rU   rV   Nc                     |j                   \  }}}}}| j                  ||      }|r|| j                  |||      z   }n|| j                  z   }| j	                  |      }|S N)rY   rn   rV   rq   rt   )	r<   rU   rV   batchframeschannelra   rb   rv   s	            r+   rd   z#VideoPrismSpatialEmbeddings.forward   ss    
 1D0I0I-vw**+>@XY
 $#d&C&CJPVX]&^^J#d&>&>>J\\*-
r*   re   )r"   r#   r$   r%   r   rI   r&   rf   intrV   rg   rd   rh   ri   s   @r+   rk   rk      su    25 2$5<< $ $UX $]b]i]i $R 16"\\ #'+ 
	r*   rk   c            	            e Zd ZdZdef fdZdej                  dej                  fdZ	 ddej                  dej                  d	e
d
z  dej                  fdZ xZS )VideoPrismTemporalEmbeddingsz
    VideoPrism Temporal Embeddings.

    Receives embeddings from spatial encoder, reshapes the hidden state to
    (batch_size * num_patches, num_frames, hidden_size) and adds positional embeddings.
    rC   c                     t         |           t        j                  t	        j
                  d|j                  |j                              | _        t        j                  |j                        | _        y rm   )rH   rI   rM   ro   r&   rp   r`   rP   rq   rr   rs   rt   r<   rC   rT   s     r+   rI   z%VideoPrismTemporalEmbeddings.__init__   sO    #%<<Av?P?PRXRdRd0e#f zz&"<"<=r*   rv   r5   c                 t   |j                   d   }| j                  j                   d   }t        j                  j	                         s||k(  r| j                  S | j                  }|j                   d   }|j                  d      }t        j                  j                  |||fdd      }|j                  d      S )rx   r   ry   r{   Tr|   )
rY   rq   r&   r   r   	unsqueezerM   r   r   squeeze)r<   rv   target_emb_lengthsource_emb_length
source_embr   s         r+   rV   z5VideoPrismTemporalEmbeddings.interpolate_pos_encoding   s     ',,Q/ 44::1= yy##%*;?P*P+++--
r"))!,
]]..#S)	 / 

 !!!$$r*   rU   input_shaperV   Nc                     ||\  }}}}}|j                   \  }	}
}|j                  |
|      }|j                  dd      }|j                  ||
z  ||      }|r|| j	                  |      z   }n|| j
                  z   }| j                  |      }|S )NrG   r   )rY   r   r[   r^   rV   rq   rt   )r<   rU   r   rV   r   r   r   ra   rb   _featuresr   rc   rv   s                 r+   rd   z$VideoPrismTemporalEmbeddings.forward   s     "4?1E67FE.448S+00#N%//15"**58+;VSI
 $#d&C&CJ&OOJ#d&>&>>J\\*-
r*   re   )r"   r#   r$   r%   r   rI   r&   rf   rV   Sizerg   rd   rh   ri   s   @r+   r   r      sq    >5 >
%5<< %ELL %@ 16	"\\ ZZ #'+	
 
r*   r   num_posr   r5   c                    ddt        j                  d|dt         j                        |z  z  z  }t        j                  dt        j                  | t         j                        j	                         |      j	                         }t        j
                  t        j                  |      t        j                  |      fd      S )	N      ?i'  r   rG   dtypezi , j -> i jr   r   )r&   arangeint64einsumfloatcatsincos)r   r   inv_freqsinusoid_inps       r+   create_sinusoidal_positionsr     s    eQQekk JS PQRH<<WEKK0X0^0^0`bjkqqsL99eii-uyy/FGQOOr*   c            	            e Zd Zdef fdZ	 	 	 d	dej                  dz  dej                  dz  dej                  dz  dej                  fdZ	 xZ
S )
VideoPrismTextEmbeddingsrC   c                    t         |           || _        |j                  }t	        j
                  |j                  |      | _        | j                  dt        |j                  |j                               | j                  dt        j                  |j                        j                  d             t	        j                  t        j                  dd|j                              | _        |j                  dz  | _        y )Nposition_embeddingposition_idsr   ry   r   rz   )rH   rI   rC   rP   rM   	Embedding
vocab_sizetoken_embeddingregister_bufferr   max_position_embeddingsr&   r   expandro   rp   cls_embscaling)r<   rC   	embed_dimrT   s      r+   rI   z!VideoPrismTextEmbeddings.__init__  s    &&	!||F,=,=yI "=f>\>\^d^p^p"q	
 	^U\\&:X:X-Y-`-`ah-ij||EKK1f6H6H$IJ))3.r*   N	input_idsr   inputs_embedsr5   c                    || j                  |      }|"| j                  d d d |j                  d   f   }|| j                  z  }| j                  |   j                  |j                        }||z   }| j                  | j                  z  }|j                  |j                  d   dd      }t        j                  ||fd      }|S )Nr   r   r   ry   r   )r   r   rY   r   r   tor   r   r   r&   r   )r<   r   r   r   rq   rv   r   s          r+   rd   z VideoPrismTextEmbeddings.forward"  s       00;M,,Q0H-2E2Ea2H0H-HIL%4"55lCFF]M`M`Fa"%88
,,-..!1!1!!4b"=YY
G4!<
r*   )NNN)r"   r#   r$   r   rI   r&   
LongTensorr'   rf   rd   rh   ri   s   @r+   r   r     sk    
/3 
/ .20426	##d* &&- ((4/	
 
r*   r   modulequerykeyvalueattention_maskrt   r   softcapc                 |   || j                   dz  }t        || j                        }	t        || j                        }
t        j                  ||	j                  dd            |z  }|||z  }t        j                  |      }||z  }|||z   }t        j                  j                  |dt        j                        j                  |j                        }t        j                  j                  ||| j                        }t        j                  ||
      }|j                  dd      j                         }||fS )N      rG   r   ry   )r   r   )ptrainingr   )head_dim	repeat_kvnum_key_value_groupsr&   matmulr[   tanhrM   r   softmaxfloat32r   r   rt   r   
contiguous)r   r   r   r   r   rt   r   r   kwargs
key_statesvalue_statesattn_weightsattn_outputs                r+   eager_attention_forwardr   8  s    //4'3 ; ;<JUF$?$?@L<<z';';Aq'ABWLL#g-zz,/#g-!#n4 ==((2U]](SVVW\WbWbcL==((6??([L,,|\:K''1-88:K$$r*   rc   n_repc                     | j                   \  }}}}|dk(  r| S | dddddddddf   j                  |||||      } | j                  |||z  ||      S )z
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)rY   r   r^   )rc   r   r   num_key_value_headsslenr   s         r+   r   r   Z  so    
 2?1D1D.Ehz!!Qa"23::5BUW\^bdlmM  (;e(CT8TTr*   c                        e Zd Zdeez  f fdZ	 d	dej                  dej                  dz  dee	   de
ej                  ej                  f   fdZ xZS )
VideoPrismAttentionrC   c                 6   t         |           || _        t        |d|j                  |j
                  z        | _        |j                  | _        | j                  dz  | _	        d| _
        t        j                  |j                  |j
                  | j                  z  |j                        | _        t        j                  |j                  |j
                  | j                  z  |j                        | _        t        j                  |j                  |j
                  | j                  z  |j                        | _        t        j                  |j
                  | j                  z  |j                  d      | _        d| _        |j&                  | _        y )Nr   r   FbiasTr   )rH   rI   rC   r8   rP   num_attention_headsr   attention_probs_dropout_probattention_dropoutr   	is_causalrM   Linearqkv_biasq_projk_projv_projo_projr   attn_logit_softcappingr   s     r+   rI   zVideoPrismAttention.__init__g  s4   
F4F4F&JdJd4de!'!D!D}}d*ii 2 2F4N4NQUQ^Q^4^eketetuii 2 2F4N4NQUQ^Q^4^eketetuii 2 2F4N4NQUQ^Q^4^eketetuii : :T]] JFL^L^eij$'!&,&C&C#r*   Nrc   r   r   r5   c                    |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }t        j                  | j                  j                  t              }	 |	| ||||f| j                  sdn| j                  | j                  | j                  d|\  }
} |
j                   g |d j#                         }
| j%                  |
      }
|
|fS )Nry   r   rG           )rt   r   r   )rY   r   r   r   r[   r   r   r   get_interfacerC   _attn_implementationr   r   r   r   r   r^   r   r   )r<   rc   r   r   r   hidden_shapequery_statesr   r   attention_interfacer   r   s               r+   rd   zVideoPrismAttention.forwardv  sT    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST(?(M(MKK,,.E)
 %8
%
  $}}C$2H2HLL//
%
 
%
!\ *k));;;;FFHkk+.L((r*   r   )r"   r#   r$   r   r   rI   r&   rf   r   r   r>   rd   rh   ri   s   @r+   r   r   f  sm    D58LL D$ /3)||) t+) +,	)
 
u||U\\)	*)r*   r   c                   D    e Zd Zdej                  dej                  fdZy)VideoPrismLayerNormrc   r5   c                     t        j                  || j                  | j                  dz   | j                  | j
                        S )Nr   )F
layer_normnormalized_shapeweightr   epsr<   rc   s     r+   rd   zVideoPrismLayerNorm.forward  s9     ||M4+@+@$++PSBSUYU^U^`d`h`hiir*   N)r"   r#   r$   r&   rf   rd   r)   r*   r+   r   r     s     jU\\ jell jr*   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )VideoPrismMLPrC   c                    t         |           || _        t        |j                     | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        y r   )rH   rI   rC   r	   
hidden_actactivation_fnrM   r   rP   intermediate_sizefc1fc2r   s     r+   rI   zVideoPrismMLP.__init__  sd    #F$5$5699V//1I1IJ99V55v7I7IJr*   rc   r5   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )r  r  r  r   s     r+   rd   zVideoPrismMLP.forward  s4    /**=9/r*   )	r"   r#   r$   r   rI   r&   rf   rd   rh   ri   s   @r+   r  r    s-    K/ KU\\ ell r*   r  c            	            e Zd Zdeez  f fdZ	 d	dej                  dej                  dz  dee	   dej                  fdZ
 xZS )
VideoPrismLayerrC   c                 B   t         |           t        |      | _        t	        |j
                  |j                        | _        t	        |j
                  |j                        | _        t        |      | _
        t        j                  |j                        | _        y N)r   )rH   rI   r   	attentionr   rP   layer_norm_epslayernorm_beforelayernorm_afterr  mlprM   rr   rs   rt   r   s     r+   rI   zVideoPrismLayer.__init__  sr    ,V4 3F4F4FFLaLa b263E3E6K`K`a (zz&"<"<=r*   Nrc   r   r   r5   c                     |}| j                  |      } | j                  ||fi |\  }}| j                  |      }||z   }|}| j                  |      }| j	                  |      }| j                  |      }||z   }|S r   )r  r  rt   r  r  )r<   rc   r   r   residualr   s         r+   rd   zVideoPrismLayer.forward  s     !--m<)4>>-R6Rq]3%0 !,,];/]3%0r*   r   )r"   r#   r$   r   r   rI   r&   rf   r   r   rd   rh   ri   s   @r+   r
  r
    s]    >58LL > /3|| t+ +,	
 
r*   r
  c                        e Zd ZU eed<   dZdZdZdZg dZ	dZ
dZdZdZdZeedZ ej&                          fd	       Z xZS )
VideoPrismPreTrainedModelrC   modelrU   )videotextT)rk   r   r
  r   'VideoPrismMultiheadAttentionPoolingHeadF)rc   
attentionsc                    t         |   |       t        |t        j                  t        j
                  f      r t        j                  |j                         yt        |t              r t        j                  |j                         yt        |t              r t        j                  |j                         yt        |t              r?t        j                  |j                         t        j                  |j                         yt        |t               r t        j                  |j                         yt        |t"              rt%        |j&                  j(                  |j&                  j*                        j-                  |j.                  j0                  |j.                  j2                        }t        j4                  |j.                  |       t        j4                  |j6                  t9        j:                  |j6                  j<                  d         j?                  d             yt        |t@              rt        jB                  |jD                  jF                  j                  |j&                  j*                  dz         t        jB                  |jD                  jH                  |j&                  j*                  dz         yy)zInitialize the weightsdevicer   ry   r   r   )stdN)%rH   _init_weightsrL   rM   r   rN   initlecun_normal_r   rk   rq   r   r  zeros_per_dim_scalepooling_attention_queryr   r   r   rC   r   rP   r   r   r  r   copy_r   r&   r   rY   r   VideoPrismTextModelnormal_rv   r   r   )r<   r   r   rT   s      r+   r  z'VideoPrismPreTrainedModel._init_weights  s    	f%fryy"))45v}}- ;<v99: <=v99: GHKK,,-v==> 34KK& 89!<55v}}7P7P"b1188@Y@Y@_@_b`  JJv002DEJJv**ELL9L9L9R9RSU9V,W,^,^_f,gh 34LL**::AAv}}G`G`bfGfgLL**228Q8QSW8WX 5r*   )r"   r#   r$   r   r(   base_model_prefixmain_input_nameinput_modalitiessupports_gradient_checkpointing_no_split_modules_supports_sdpa_supports_flash_attn_supports_flex_attn_supports_attention_backend_can_compile_fullgraphr
  r   _can_record_outputsr&   no_gradr  rh   ri   s   @r+   r  r    ss    +O(&*# N"&!()
 U]]_Y Yr*   r  z
    The bare VideoPrism vision encoder outputting raw hidden-states without any specific head on top. This model is the backbone encoder used in VideoPrismVideoModel.
    c                        e Zd ZU eed<   dZdZdef fdZdej                  fdZ
dej                  fdZeee	 	 dd
ej                   d	z  ded	z  dee   defd                     Z xZS )VideoPrismVisionModelrC   r  r  c                 @   t         |   |       t        |j                  |j                        | _        t        |j                  |j                        | _        t        |      | _        t        |      | _
        t        j                  t        |j                        D cg c]  }t        |       c}      | _        t        j                  t        |j"                        D cg c]  }t        |       c}      | _        | j'                          y c c}w c c}w r  )rH   rI   r   rP   r  
layernorm1
layernorm2rk   spatial_embeddingsr   temporal_embeddingsrM   
ModuleListrangenum_spatial_layersr
  spatial_layersnum_temporal_layerstemporal_layers	post_initr<   rC   r   rT   s      r+   rI   zVideoPrismVisionModel.__init__  s     -f.@.@fF[F[\-f.@.@fF[F[\"=f"E#?#G  mmeTZTmTmNn,o_V-D,op!}}uU[UoUoOp-q!of.E-qr -p-qs   #D'Dr5   c                 .    | j                   j                  S r   r:  rn   r@   s    r+   get_input_embeddingsz*VideoPrismVisionModel.get_input_embeddings  s    &&777r*   r   c                 &    || j                   _        y r   rE  r<   r   s     r+   set_input_embeddingsz*VideoPrismVisionModel.set_input_embeddings!  s    380r*   NrU   rV   r   c                 ,   |t        d      |j                  }| j                  ||      }|}| j                  D ]  } ||fi |} | j	                  |      }| j                  |||      }	|	}
| j                  D ]  } ||
fi |}
 | j                  |
      }|j                  \  }}}|j                  |d   d||      j                  dd      j                         }|j                  \  }}}}|j                  |d   ||z  d      }t        ||
|      S )Nz'You have to specify pixel_values_videosr   ry   r   rG   )last_hidden_stater    r!   )rZ   rY   r:  r?  r8  r;  rA  r9  r   r[   r   r   )r<   rU   rV   r   r   spatial_embedsspatial_hidden_statesspatial_layerr   temporal_embedstemporal_hidden_statestemporal_layerr   r`   r   rS   s                   r+   rd   zVideoPrismVisionModel.forward$  sM    &FGG)// 001DF^_ .!00 	SM$12G$R6$R!	S??#89 228[Jbc!0"22 	VN%34J%Uf%U"	V??#9: &^^:s==QZEOOPQSTU``b*2..':{C==Qk1I2N:&'=&;
 	
r*   NF)r"   r#   r$   r   r(   r*  r(  rI   rM   ModulerF  rI  r   r   r   r&   r'   rg   r   r   r   rd   rh   ri   s   @r+   r5  r5  
  s     #"!5 8bii 89")) 9   9=05#
"..5#
 #'+#
 +,	#

 
5#
    #
r*   r5  g^$3eG?c                        e Zd Zdef fdZ	 d	dej                  dej                  dz  dee	   de
ej                  ej                  f   fdZ xZS )
r  rC   c                    t         |           || _        |j                  |j                  z  | _        |j                  | _        t        | j
                  dz  z  | _	        d| _
        t        j                  |j                  |j                  | j
                  z  |j                        | _        t        j                  |j                  |j                  | j
                  z  |j                        | _        t        j                  |j                  |j                  | j
                  z  |j                        | _        t        j                  |j                  | j
                  z  |j                  d      | _        d| _        t        j(                  t+        j,                  | j
                              | _        t        j(                  t+        j,                  dd|j                              | _        y )Nrz   Fr   Tr   r   )rH   rI   rC   r  r   r   r   r   _R_SOFTPLUS_0r   r   rM   r   rP   r   r   r   r   r   r   ro   r&   rp   r#  r$  r   s     r+   rI   z0VideoPrismMultiheadAttentionPoolingHead.__init__Q  sd   00F4N4NN!'!D!D$s(:;ii 2 2F4N4NQUQ^Q^4^eketetuii 2 2F4N4NQUQ^Q^4^eketetuii 2 2F4N4NQUQ^Q^4^eketetuii : :T]] JFL^L^eij$'!\\%++dmm*DE')||EKK1fFXFX4Y'Z$r*   Nrc   r   r   r5   c                    |j                   d d }g |d| j                  }| j                  j                  |d   dd      } | j	                  |      j
                  g |j                   d d d| j                   j                  dd      }|| j                  z  t        j                  j                  | j                        z  }| j                  |      j                  |      j                  dd      }	| j                  |      j                  |      j                  dd      }
t        j                  | j                   j"                  t$              } || ||	|
|fd| j&                  sdn| j(                  d d|\  }} |j*                  g |j                   d d d j-                         }| j/                  |      }||fS )Nry   r   r   rG   r   r   )r   rt   r   )rY   r   r$  r   r   r   r[   r   rM   r   softplusr#  r   r   r   r   rC   r   r   r   r   r^   r   r   )r<   rc   r   r   r   r   r   query_layerr   r   r   r   r   r   s                 r+   rd   z/VideoPrismMultiheadAttentionPoolingHead.forwarda  s    $))#2.88b8$--8,,33KNBK-dkk%(--Su{{3B/?SST]]S]]^_abc"T\\1BMM4J4J4K]K]4^^[[/44\BLLQPQR
{{=166|DNNqRST(?(M(MKK,,.E)
 %8
%
 #}}C$2H2H
%
 
%
!\ *k))@5;;s+;@R@KKMkk+.L((r*   r   )r"   r#   r$   r   rI   r&   r'   r   r   r   r>   rd   rh   ri   s   @r+   r  r  P  so    [5 [& 37")((") ((4/") +,	")
 
u  %"3"33	4")r*   r  xr   c                 d    t        j                  | | z  j                  |d      |z         }| |z  S )zUThis function is intended to align with the l2norm implementation in the FLA library.T)r   keepdim)r&   rsqrtsum)rZ  r   r   inv_norms       r+   l2normr`    s0    {{AE;;3;=CDHx<r*   z
    The bare VideoPrism text encoder outputting last hidden states without any specific head on top. This model is used in VideoPrismClipModel.
    c                        e Zd ZU eed<   dZdZdZddgZdZ	def fdZ
eee	 	 	 	 ddej                  d	z  d
ej                   d	z  dej                   d	z  dej                   d	z  dee   defd                     Z xZS )r&  rC   )r  r  r   r   r
  r   c                 B   t         |   |       t        |      | _        t	        j
                  t        |j                        D cg c]  }t        |       c}      | _	        t        |j                  |j                        | _        | j                          y c c}w r  )rH   rI   r   rv   rM   r<  r=  num_hidden_layersr
  layersr   rP   r  	layernormrB  rC  s      r+   rI   zVideoPrismTextModel.__init__  sq     26:mmeFLdLdFe$f_V%<$fg,V-?-?VEZEZ[ %gs   BNr   r   r   r   r5   c                    |d u |d uz  rt        d      | j                  |||      }|lt        j                  |j                  d   d|j
                  |j                        }t        j                  ||fd      }t        | j                  ||d       }| j                  D ]  } |||fi |} | j                  |      }|d d df   }	| j                  j                  rt        |	d      }	t        ||		      S )
Nz:You must specify exactly one of input_ids or inputs_embeds)r   r   r   r   r   r  r   )rC   r   r   past_key_valuesry   )rK  pooler_output)rZ   rv   r&   onesrY   r  r   r   r
   rC   rd  re  apply_l2normr`  r   )
r<   r   r   r   r   r   rc   cls_paddinglayertext_embeddingss
             r+   rd   zVideoPrismTextModel.forward  s    -t";<YZZ),fst%**##A&.2G2G~OcOcK #YY'D!LN/{{+- $	N [[ 	KE!-J6JM	K}5'2.;;##$_"=O)MYhiir*   )NNNN)r"   r#   r$   r   r(   r*  r(  r)  r,  _input_embed_layerrI   r   r   r   r&   r   rf   r   r   r   rd   rh   ri   s   @r+   r&  r&    s     !  !O35FG*3    .2.2-1,0!j##d*!j t+!j ||d*	!j
 llT)!j +,!j 
$!j    !jr*   r&  z
    VideoPrism video model consisting of the vision encoder backbone with auxiliary encoder layers and an attention pooling head on top. This model is used in VideoPrismClipModel.
    c                        e Zd ZU eed<   def fdZdej                  fdZdej                  fdZ	e
e	 ddej                  ded	z  d
ee   defd              Z xZS )VideoPrismVideoModelrC   c                 v   t         |   |       t        j                  |      | _        t        j                  t        |j                        D cg c]  }t        |       c}      | _
        t        |      | _        t        |j                  |j                        | _        | j#                          y c c}w r  )rH   rI   r5  _from_configvision_modelrM   r<  r=  num_auxiliary_layersr
  auxiliary_layersr  headr   rP   r  head_layernormrB  rC  s      r+   rI   zVideoPrismVideoModel.__init__  s     1>>vF "PUV\VqVqPr.s1v/F.s t;FC	1&2D2D&J_J_` /ts   B6r5   c                 6    | j                   j                         S r   rs  rF  r@   s    r+   rF  z)VideoPrismVideoModel.get_input_embeddings        5577r*   r   c                 :    | j                   j                  |       y r   rs  rI  rH  s     r+   rI  z)VideoPrismVideoModel.set_input_embeddings      ..u5r*   rU   rV   Nr   c                 R    | j                   d||d|}|j                  }| j                  D ]  } ||fi |}  | j                  |fi |}| j	                  |d         }| j
                  j                  rt        |d      }t        |||j                  |j                        S )NrU   rV   r   ry   r   )rK  rh  rc   r  r)   )rs  rK  ru  rv  rw  rC   rj  r`  r   rc   r  )	r<   rU   rV   r   vision_model_outputsauxiliary_hidden_statesrl  head_outputvideo_embeddingss	            r+   rd   zVideoPrismVideoModel.forward  s      1t00  
 3Nf 
jp 
 #7"H"H** 	OE&+,C&Nv&N#	O  dii 7B6B..{1~>;;##%&6B?)5*.<<+66	
 	
r*   re   )r"   r#   r$   r   r(   rI   rM   rS  rF  rI  r   r   r&   r'   rg   r   r   r   rd   rh   ri   s   @r+   rp  rp    s     #"5 8bii 86")) 6  16
"..
 #'+
 +,	

 
$
  
r*   rp  z
    VideoPrism model for video-text contrastive learning. This model consists of a VideoPrismVideoModel and a VideoPrismTextModel, and computes similarity scores between video and text inputs.
    c                       e Zd Zdef fdZdej                  fdZdej                  fdZe	e
	 ddej                  d	ej                  dz  d
ee   deez  fd              Ze	e
	 ddej$                  dedz  d
ee   deez  fd              Ze	e
	 	 	 	 ddej$                  dej                  d	ej                  dz  dedz  dedz  dedz  d
ee   defd              Z xZS )VideoPrismClipModelrC   c                     t         |   |       t        j                  |j                        | _        t        j                  |j                        | _        | j                          y r   )
rH   rI   rp  rr  vision_configvideo_modelr&  text_config
text_modelrB  r   s     r+   rI   zVideoPrismClipModel.__init__  sK     /<<V=Q=QR-::6;M;MNr*   r5   c                 6    | j                   j                         S r   )r  rF  r@   s    r+   rF  z(VideoPrismClipModel.get_input_embeddings  s    3355r*   r   c                 :    | j                   j                  |       y r   )r  rI  rH  s     r+   rI  z(VideoPrismClipModel.set_input_embeddings  s    ,,U3r*   Nr   r   r   c                 ,     | j                   d||d|S )a  
        Examples:

        ```python
        >>> from transformers import AutoTokenizer, VideoPrismClipModel

        >>> model = VideoPrismClipModel.from_pretrained("google/videoprism-lvt-base-f16r288")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/videoprism-lvt-base-f16r288")

        >>> inputs = tokenizer(["a video of a cat.", "a video of a dog."], padding="max_length", return_tensors="pt")
        >>> with torch.no_grad():
        ...     text_features = model.get_text_features(**inputs)
        ```r   r   r)   )r  )r<   r   r   r   s       r+   get_text_featuresz%VideoPrismClipModel.get_text_features  s     * t\>\U[\\r*   rU   rV   c                 ,     | j                   d||d|S )a  
        Examples:

        ```python
        >>> from transformers import VideoPrismProcessor, VideoPrismClipModel

        >>> model = VideoPrismClipModel.from_pretrained("google/videoprism-lvt-base-f16r288")
        >>> processor = VideoPrismProcessor.from_pretrained("google/videoprism-lvt-base-f16r288")

        >>> inputs = processor(videos="path/to/video.mp4", return_tensors="pt")
        >>> with torch.no_grad():
        ...     video_features = model.get_video_features(**inputs)
        ```r  r)   )r  )r<   rU   rV   r   s       r+   get_video_featuresz&VideoPrismClipModel.get_video_features"  s.    *  t 
 3%=
 
 	
r*   temperaturereturn_lossc           	      r    | j                   d||d|} | j                  d||d|}	|j                  }
|	j                  }|
j                  d   }|j                  d   }|
j	                  d|      }|j	                  d|      }t        j                  ||j                        }|||z  }t        j                  |      }|j                  }|t        j                  |dd      z  }|t        j                  |dd      z  }d}|rt        j                  |j                  d      |j                        }t        j                  |       d	|z  z   }t
        j                  j                  j!                  ||z        }t        j                  |d
       }|j#                         }t%        ||||||	|      S )a  
        temperature (`float`, *optional*):
            A temperature scalar to scale the similarity scores. If not provided, no scaling is applied.
        return_loss (`bool`, *optional*):
            Whether or not to return the contrastive loss.
        r  r  ry   Nr   T)r   keepdims)r  rG   r   )r.   r/   r0   r1   r2   r3   r4   r)   )r  r  rh  rY   r^   r&   r   Texpr^  eyer}   r  	ones_likerM   r   
logsigmoidmeanr-   )r<   rU   r   r   rV   r  r  r   video_model_outputstext_model_outputsr  rm  video_emb_dimtext_emb_dimr0   r1   similarity_matrixr.   r/   r4   r  m1_diag1logliknlls                           r+   rd   zVideoPrismClipModel.forward=  s   & 6d55 
 3Nf
jp
 4T33qiXfqjpq.<<,::(..r2&,,R0'//MB%--b,?!LL{}}E", 99%67*,,+eii8HaZ^.__)EIIo1W[,\\ ))O003O<R<RSC881s7BHXX((33H4NOF99V,,C88:D#-+%#20
 	
r*   r   re   )NFNN)r"   r#   r$   r   rI   rM   rS  rF  rI  r   r   r&   rf   r   r   r>   r   r  r'   rg   r  r   r-   rd   rh   ri   s   @r+   r  r    s   / 6bii 64")) 4  /3]<<] t+] +,	]
 
+	+]  ]*  16
"..
 #'+
 +,	

 
+	+
  
2 
 /305$(#'9
"..9
 <<9
 t+	9

 #'+9
 T\9
 D[9
 +,9
 
9
  9
r*   r  z
    VideoPrism Model transformer with a video classification head on top (a linear layer on top of the attention pooler).
    c                        e Zd ZU eed<   dZdZdef fdZdej                  fdZ
dej                  fdZee	 	 dd
ej                  dej                   d	z  ded	z  dee   def
d              Z xZS ) VideoPrismForVideoClassificationrC   r6  r  c                 B   t         |   |       t        j                  |      | _        t        |      | _        t        |j                  |j                        | _
        t        j                  |j                  |j                        | _        | j                          y r  )rH   rI   r5  rr  rs  r  rv  r   rP   r  rw  rM   r   
num_labels
classifierrB  r   s     r+   rI   z)VideoPrismForVideoClassification.__init__  sr     1>>vF;FC	1&2D2D&J_J_`))F$6$68I8IJr*   r5   c                 6    | j                   j                         S r   ry  r@   s    r+   rF  z5VideoPrismForVideoClassification.get_input_embeddings  rz  r*   r   c                 :    | j                   j                  |       y r   r|  rH  s     r+   rI  z5VideoPrismForVideoClassification.set_input_embeddings  r}  r*   NrU   labelsrV   r   c                 <    | j                   d||d|}|j                  }| j                   | j                  |fi |d         }| j	                  |      }d }	| | j
                  ||| j                  fi |}	t        |	||j                  |j                        S )Nr  r   )r4   logitsrc   r  r)   )
rs  rK  rw  rv  r  loss_functionrC   r   rc   r  )
r<   rU   r  rV   r   r  sequence_outputpooled_outputr  r4   s
             r+   rd   z(VideoPrismForVideoClassification.forward  s      1t00  
 3Nf 
jp 
 /@@++IDIIo,P,PQR,ST/%4%%ffdkkLVLD$.<<+66	
 	
r*   rR  )r"   r#   r$   r   r(   r*  r(  rI   rM   rS  rF  rI  r   r   r&   r'   r   rg   r   r   r   rd   rh   ri   s   @r+   r  r  {  s     #"!5 8bii 86")) 6  +/05	
"..
   4'
 #'+	

 +,
 

  
r*   r  )r5  r  rp  r&  r  r  )r   NN)ry   gư>)Kcollections.abcr   r   dataclassesr   typingr   r&   torch.nnrM   torch.nn.functionalr   r    r   r   activationsr	   masking_utilsr
   modeling_layersr   modeling_outputsr   r   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   r   utils.genericr   utils.output_capturingr   configuration_videoprismr   r   r   r   r-   rS  rB   rk   r   r   rf   r   r   r   r>   r   r   r   	LayerNormr   r  r
  r  r5  rV  r  r'   r`  r&  rp  r  r  __all__r)   r*   r+   <module>r     s2  , / !      & ! / 9 b b F & a a 7 5 d d ij
?/ ?  k? B  
;  
  
F$")) $ND")) DN<299 <~P P3 P5<< P ryy  R   %II%<<% 
% <<	%
 LL4'% S[% T\% T\% 5<<%&%D	UU\\ 	U# 	U%,, 	U/)")) /)dj",, jBII  0 @ 6Y 6Y 6Yr 
;
5 ;

;
| 3)bii 3)le c U  
3j3 3j
3jl 
*
4 *

*
Z 
z
3 z

z
z 
+
'@ +

+
\r*   