
    ^j]                        d dl mZ d dlmZ d dlmZ d dlZd dlmZ d dl	mc m
Z d dlmZ ddlmZ ddlmZ dd	lmZmZmZ dd
lmZmZ ddlmZmZmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5 ddl6m7Z7m8Z8m9Z9m:Z:m;Z;  e#jx                  e=      Z>dZ? e!d      e G d de5                    Z@ e!d      e G d de1                    ZA e!d      e G d d e0                    ZB G d! d"e3      ZC G d# d$ed%&      ZDe! G d' d(e             ZE e!d)*      e G d+ d,e                    ZF e!d-*      e G d. d/e                    ZG G d0 d1e;      ZH G d2 d3e8      ZI G d4 d5e8      ZJ G d6 d7ej                        ZL G d8 d9e7      ZM G d: d;ej                        ZO G d< d=e9      ZPe! G d> d?e:             ZQ e!d@*       G dA dBeQ             ZR G dC dDe7      ZS e!dE*       G dF dGeQ             ZT e!dH*       G dI dJeQ             ZU e!dK*       G dL dMeQ             ZV e!dN*       G dO dPeQ             ZWg dQZXy)R    )Callable)	dataclass)AnyN)strict   )initialization)create_causal_mask)BaseModelOutputBaseModelOutputWithPoolingImageClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)ProcessingKwargsProcessorMixinUnpack)ModelOutputTransformersKwargsauto_docstringcan_return_tuplelogging	torch_int)merge_with_config_defaults)capture_outputs   )create_sinusoidal_positions)eager_attention_forward)l2norm)SiglipConfigSiglipTextConfig)T5Tokenizer)VivitConfig)VivitAttentionVivitEmbeddings
VivitLayerVivitPreTrainedModelVivitTubeletEmbeddingsg^$3eG?zgoogle/videoprism-base-f16r288)
checkpointc                       e Zd ZU dZdZdZdZeee   z  e	eef   z  e
d<   dZee
d<   dZee   e	ed	f   z  e
d
<   dZee
d<   dZee
d<   dZee
d<   dZee
d<   dZee
d<   dZee
d<    e       Z e       Z e       Zd Zy)VideoPrismVisionConfiga  
    num_frames (`int`, *optional*, defaults to 16):
        The number of frames in the input video.
    tubelet_size (`List[int]`, *optional*, defaults to `[1, 18, 18]`):
        The size of the tubelet patch.
    num_spatial_layers (`int`, *optional*, defaults to 12):
        Number of spatial transformer blocks.
    num_temporal_layers (`int`, *optional*, defaults to 4):
        Number of temporal transformer blocks.
    attn_logit_softcapping (`float`, *optional*, defaults to 50.0):
        Softcapping constant for attention logits.
    num_auxiliary_layers (`int`, *optional*, defaults to 2):
        Number of auxiliary layers. This is used in the VideoPrismVideoModel that is a part of VideoPrismClipModel.
    apply_l2norm (`bool`, *optional*, defaults to `True`):
        Whether to apply L2 normalization to the output. This is used in the VideoPrismVideoModel that is a part of VideoPrismClipModel.
    videoprism_vision_modelvision_config   
image_size   
num_frames)      r1   .tubelet_size   num_spatial_layers   num_temporal_layersgelu_python
hidden_act      I@attn_logit_softcappingr   num_auxiliary_layersTapply_l2normc                     t        d      NzNot used hereAttributeErrorselfkwargss     |/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/videoprism/modular_videoprism.py__post_init__z$VideoPrismVisionConfig.__post_init__X       _--    N)__name__
__module____qualname____doc__
model_typebase_config_keyr-   intlisttuple__annotations__r/   r2   r4   r6   r8   strr:   floatr;   r<   boolr@   num_hidden_layers
pooler_actpooler_output_sizerE    rG   rD   r)   r)   5   s    " +J%O47Jd3i%S/17J0;L$s)eCHo-;    #J#$(E( !#!L$&(!J').rG   r)   z"google/videoprism-lvt-base-f16r288c                       e Zd ZU dZdZeed<   dZedz  ed<   dZ	edz  ed<   dZ
eee   z  dz  ed<   d	Zeez  ed
<   dZeed<   dZeed<   d	Zeed<   dZeed<   dZeed<    e       Z e       Zd Zy)VideoPrismTextConfiga	  
    apply_l2norm (`bool`, *optional*, defaults to `True`):
        Whether to apply L2 normalization to the output of VideoPrismTextEncoder.
    attn_logit_softcapping (`float`, *optional*, defaults to 50.0):
        Softcapping constant for attention logits.
    relur8   r   Npad_token_idbos_token_ideos_token_id        attention_probs_dropout_probTr<   qkv_biashidden_dropout_probg{Gz?initializer_ranger9   r:   c                     t        d      r>   r?   rA   s     rD   rE   z"VideoPrismTextConfig.__post_init__s   rF   rG   )rH   rI   rJ   rK   r8   rR   rQ   r\   rN   r]   r^   rO   r`   rS   r<   rT   ra   rb   rc   r:   r@   attention_dropoutprojection_sizerE   rX   rG   rD   rZ   rZ   \   s     J L#* #L#*#+/L#S	/D(/03 %#+3L$Hd!$$#u#$(E(&($&O.rG   rZ   c                       e Zd ZdZ e       Zy)VideoPrismConfiga  
    Example:

    ```python
    >>> from transformers import VideoPrismClipModel, VideoPrismConfig

    >>> # Initializing a VideoPrismConfig with default values
    >>> configuration = VideoPrismConfig()

    >>> # Initializing a VideoPrismClipModel with the configuration
    >>> model = VideoPrismClipModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    N)rH   rI   rJ   rK   r@   initializer_factorrX   rG   rD   rh   rh   w   s    " ()rG   rh   c                   T     e Zd ZdZ	 	 	 	 	 	 	 ddeeeeef      z  dz  f fdZ xZ	S )VideoPrismTokenizeraV  
    Constructs a VideoPrism tokenizer, which is essentially a T5 tokenizer without its postprocessor
    (appending an EOS token at the end of the sequence).

    This tokenizer inherits from [`T5Tokenizer`] which contains most of the main methods. Users should refer to this
    superclass for more information regarding those methods.
    Nvocabc                 L    t        	|   d|||||||d| | j                  `y )N)rl   	eos_token	unk_token	pad_token_spm_precompiled_charsmap	extra_idsadditional_special_tokensrX   )super__init__
_tokenizerpost_processor)
rB   rl   rn   ro   rp   rq   rr   rs   rC   	__class__s
            rD   ru   zVideoPrismTokenizer.__init__   sB     	 		
&?&?		
 		
 OO*rG   )Nz</s>z<unk>z<pad>Nd   N)
rH   rI   rJ   rK   rR   rO   rP   rS   ru   __classcell__rx   s   @rD   rk   rk      sI     7;"&"&+T%U
+,,t3+ +rG   rk   c                   ,    e Zd Zddddddddddd	Zy
)VideoPrismProcessorKwargs
max_lengthT@   )padding
truncationr~   r,   )heightwidthF)sizedo_normalizedo_sample_frames)text_kwargsvideo_kwargsN)rH   rI   rJ   	_defaultsrX   rG   rD   r}   r}      s.     $
  #S1! $
IrG   r}   F)totalc                   $     e Zd ZeZd fd	Z xZS )VideoPrismProcessorc                 &    t         |   ||       y N)rt   ru   )rB   video_processor	tokenizerrx   s      rD   ru   zVideoPrismProcessor.__init__   s    )4rG   )NN)rH   rI   rJ   r}   valid_processor_kwargsru   rz   r{   s   @rD   r   r      s    65 5rG   r   zFBase class for model outputs that include spatial and temporal states.)custom_introc                   b    e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   y)+BaseModelOutputWithSpatialAndTemporalStatesa  
    last_temporal_hidden_state (`torch.FloatTensor`, *optional*):
        The last hidden state of the temporal encoder, typically of shape
        `(batch_size * num_patches, num_frames, hidden_size)`.
    last_spatial_hidden_state (`torch.FloatTensor`, *optional*):
        The last hidden state of the spatial encoder, typically of shape
        `(batch_size * num_frames, num_patches, hidden_size)`.
    Nlast_temporal_hidden_statelast_spatial_hidden_state)	rH   rI   rJ   rK   r   torchFloatTensorrQ   r   rX   rG   rD   r   r      s6     <@ 1 1D 8?:>u0047>rG   r   z+Base class for VideoPrismClipModel outputs.c                      e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
ej                  dz  ed<   dZeed<   dZeed<   dZej                  dz  ed	<   d
ee   fdZy)VideoPrismClipOutputa  
    logits_per_video (`torch.FloatTensor` of shape `(video_batch_size, text_batch_size)`):
        The scaled dot product scores between `video_embeds` and `text_embeds`. This represents the video-text
        similarity scores.
    logits_per_text (`torch.FloatTensor` of shape `(text_batch_size, video_batch_size)`):
        The scaled dot product scores between `text_embeds` and `video_embeds`. This represents the text-video
        similarity scores.
    video_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)`):
        The video embeddings obtained by applying the projection layer to the pooled output of [`VideoPrismVideoModel`].
    text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)`):
        The text embeddings obtained by applying the projection layer to the pooled output of [`VideoPrismTextModel`].
    video_model_output (`BaseModelOutputWithPooling`):
        The output of [`VideoPrismVideoModel`].
    text_model_output (`BaseModelOutputWithPooling`):
        The output of the [`VideoPrismTextModel`].
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
        Contrastive loss for video-text similarity.
    Nlogits_per_videologits_per_textvideo_embedstext_embedsvideo_model_outputtext_model_outputlossreturnc                 H     t         fd j                         D              S )Nc              3   d   K   | ]'  }|d vr|   nt        |      j                          ) yw))r   r   N)getattrto_tuple).0krB   s     rD   	<genexpr>z0VideoPrismClipOutput.to_tuple.<locals>.<genexpr>   s=      
  KKDGQXY]_`QaQjQjQll
s   -0)rP   keysrB   s   `rD   r   zVideoPrismClipOutput.to_tuple   s#     
YY[
 
 	
rG   )rH   rI   rJ   rK   r   r   r   rQ   r   r   r   r   r   r   r   rP   r   r   rX   rG   rD   r   r      s    
& 26e''$.504OU&&-4-1L%##d*1,0K""T)059294818%)D%

d
")
%* 
rG   r   c                   f     e Zd ZdZdef fdZddej                  dedej                  fdZ	 xZ
S )	VideoPrismTubeletEmbeddingsa  
    VideoPrism Tubelet Embeddings.

    The authors of Videoprism use the Factorized Encoder architecture, i.e. "Model 2", introduced in the VIVIT paper (https://huggingface.co/papers/2103.15691).
    This differs from Vivit by using a convolution of `tubelet_size=(1, 18, 18)`, which is essentially a 2d convolution in the spatial dimension.
    The temporal dimension is also merged with the `batch_size` in order to make sure the image embeddings have no temporal component, unlike Vivit.
    configc                     t         |   |       | `| j                  d   t        d   z  | j                  d   t        d   z  g| _        | j
                  d   | j
                  d   z  | _        y )Nr   r0   r   )rt   ru   num_patchesr-   r2   pos_emb_shaperB   r   rx   s     rD   ru   z$VideoPrismTubeletEmbeddings.__init__  sn     "ooa0LOCT__UVEW[ghi[jEjk--a043E3Ea3HHrG   pixel_values_videosinterpolate_pos_encodingr   c                    |j                   \  }}}}}|sV|| j                  d   k7  s|| j                  d   k7  r2t        d| d| d| j                  d    d| j                  d    d	      |j                  dd      }| j	                  |      }|j                  d      j                  dddd      }|j                   \  }}}	}
|j                  ||z  |	|
      }|S )	Nr   r0   zImage size (*z) doesn't match model (z[). Set interpolate_pos_encoding=True to automatically resize the model position embeddings.r   r   )shaper-   
ValueError	transpose
projectionflattenpermutereshape)rB   r   r   
batch_sizer/   num_channelsr   r   hidden_statesr   hidden_sizes              rD   forwardz#VideoPrismTubeletEmbeddings.forward  s   >Q>W>W;
Jfe'Vtq7I-IUVZVeVefgVhMhvhaw.EdooVWFXEYYZ[_[j[jkl[mZn  oJ  K  2;;AqA(;<%--a088Aq!D;H;N;N8
J[%--j:.E{T_`rG   F)rH   rI   rJ   rK   r)   ru   r   TensorrT   r   rz   r{   s   @rD   r   r      s=    I5 I5<< SW didpdp rG   r   c                        e Zd Zdef fdZdej                  dededej                  fdZ	 ddej                  d	e	d
z  dej                  fdZ
 xZS )VideoPrismSpatialEmbeddingsr   c                     t         |   |       | `| `t	        j
                  t        j                  dt        |j                              | _
        y Nr0   )rt   ru   	cls_tokenr-   nn	Parameterr   zerosr   r   position_embeddingsr   s     rD   ru   z$VideoPrismSpatialEmbeddings.__init__   s>     NO#%<<A{FL^L^0_#` rG   
embeddingsr   r   r   c                 8   |j                   d   }| j                  j                   d   }t        j                  j	                         s||k(  r||k(  r| j                  S |j                   d   }|| j
                  d   z  }|| j
                  d   z  }t        |dz        }	| j                  j                  d|	|	|      }
|
j                  dddd      }
t        j                  j                  |
||fdd	      }
|
j                  dddd      j                  dd|      }
|
S )
a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r0   r         ?r   r   bilinearTr   mode	antialias)r   r   r   jit
is_tracing
patch_sizer   r   r   r   
functionalinterpolateview)rB   r   r   r   r   num_positionsdimnum_row_patchesnum_col_patchessqrt_num_positionspatch_pos_embeds              rD   r   z4VideoPrismSpatialEmbeddings.interpolate_pos_encoding&  s*    !&&q)0066q9 yy##%+*F6UZ?+++r" DOOA$664??1#55&}c'9:22::1>PRdfij)11!Q1= --33!?3	 4 
 *11!Q1=BB1b#NrG   r   r   Nc                     |j                   \  }}}}}| j                  ||      }|r|| j                  |||      z   }n|| j                  z   }| j	                  |      }|S r   )r   patch_embeddingsr   r   dropout)	rB   r   r   batchframeschannelr   r   r   s	            rD   r   z#VideoPrismSpatialEmbeddings.forwardL  ss    
 1D0I0I-vw**+>@XY
 $#d&C&CJPVX]&^^J#d&>&>>J\\*-
rG   r   )rH   rI   rJ   r)   ru   r   r   rN   r   rT   r   rz   r{   s   @rD   r   r     sr    a5 a$5<< $ $UX $]b]i]i $R 16"\\ #'+ 
	rG   r   c            	            e Zd ZdZdef fdZdej                  dej                  fdZ	 ddej                  dej                  d	e
d
z  dej                  fdZ xZS )VideoPrismTemporalEmbeddingsz
    VideoPrism Temporal Embeddings.

    Receives embeddings from spatial encoder, reshapes the hidden state to
    (batch_size * num_patches, num_frames, hidden_size) and adds positional embeddings.
    r   c                     t         |   |       | `| `| `~| `t        j                  t        j                  d|j                  |j                              | _        y r   )rt   ru   r   r   r   r-   r   r   r   r   r/   r   r   )rB   r   r   rx   s      rD   ru   z%VideoPrismTemporalEmbeddings.__init__g  sS     N!OO#%<<Av?P?PRXRdRd0e#f rG   r   r   c                 t   |j                   d   }| j                  j                   d   }t        j                  j	                         s||k(  r| j                  S | j                  }|j                   d   }|j                  d      }t        j                  j                  |||fdd      }|j                  d      S )Nr0   r   r   Tr   )
r   r   r   r   r   	unsqueezer   r   r   squeeze)rB   r   target_emb_lengthsource_emb_length
source_embr   s         rD   r   z5VideoPrismTemporalEmbeddings.interpolate_pos_encodingp  s    &,,Q/ 44::1= yy##%*;?P*P+++--
r"))!,
]]..#S)	 / 

 !!!$$rG   r   input_shaper   Nc                     ||\  }}}}}|j                   \  }	}
}|j                  |
|      }|j                  dd      }|j                  ||
z  ||      }|r|| j	                  |      z   }n|| j
                  z   }| j                  |      }|S )Nr   r0   )r   r   r   r   r   r   r   )rB   r   r   r   r   r   r   r   r   _featuresr   r   r   s                 rD   r   z$VideoPrismTemporalEmbeddings.forward  s     "4?1E67FE.448S+00#N%//15"**58+;VSI
 $#d&C&CJ&OOJ#d&>&>>J\\*-
rG   r   )rH   rI   rJ   rK   r)   ru   r   r   r   SizerT   r   rz   r{   s   @rD   r   r   _  sr    g5 g%5<< %ELL %0 16	"\\ ZZ #'+	
 
rG   r   c            	            e Zd Zdef fdZ	 	 	 d	dej                  dz  dej                  dz  dej                  dz  dej                  fdZ	 xZ
S )
VideoPrismTextEmbeddingsr   c                    t         |           || _        |j                  }t	        j
                  |j                  |      | _        | j                  dt        |j                  |j                               | j                  dt        j                  |j                        j                  d             t	        j                  t        j                  dd|j                              | _        |j                  dz  | _        y )Nposition_embeddingposition_idsr0   r   r0   r   )rt   ru   r   r   r   	Embedding
vocab_sizetoken_embeddingregister_bufferr   max_position_embeddingsr   arangeexpandr   r   cls_embscaling)rB   r   	embed_dimrx   s      rD   ru   z!VideoPrismTextEmbeddings.__init__  s    &&	!||F,=,=yI "=f>\>\^d^p^p"q	
 	^U\\&:X:X-Y-`-`ah-ij||EKK1f6H6H$IJ))3.rG   N	input_idsr   inputs_embedsr   c                    || j                  |      }|"| j                  d d d |j                  d   f   }|| j                  z  }| j                  |   j                  |j                        }||z   }| j                  | j                  z  }|j                  |j                  d   dd      }t        j                  ||fd      }|S )Nr0   )dtyper   r   r   )r   r   r   r  r   tor	  r  r  r   cat)rB   r  r   r  r   r   r  s          rD   r   z VideoPrismTextEmbeddings.forward  s       00;M,,Q0H-2E2Ea2H0H-HIL%4"55lCFF]M`M`Fa"%88
,,-..!1!1!!4b"=YY
G4!<
rG   )NNN)rH   rI   rJ   rZ   ru   r   
LongTensorr   r   r   rz   r{   s   @rD   r   r     sk    
/3 
/ .20426	##d* &&- ((4/	
 
rG   r   c                        e Zd Zdeez  f fdZ	 d	dej                  dej                  dz  dee	   de
ej                  ej                  f   fdZ xZS )
VideoPrismAttentionr   c                 X    t         |   |       | `d| _        |j                  | _        y N      ?)rt   ru   num_attention_headsnum_key_value_groupsr:   r   s     rD   ru   zVideoPrismAttention.__init__  s,     $$'!&,&C&C#rG   Nr   attention_maskrC   r   c                    |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }t        j                  | j                  j                  t              }	 |	| ||||f| j                  sdn| j                  | j                  | j                  d|\  }
} |
j                   g |d j#                         }
| j%                  |
      }
|
|fS )Nr   r0   r   r_   )r   r  softcap)r   head_dimq_projr   r   k_projv_projr   get_interfacer   _attn_implementationr   trainingre   r  r:   r   
contiguouso_proj)rB   r   r  rC   r   hidden_shapequery_states
key_statesvalue_statesattention_interfaceattn_outputattn_weightss               rD   r   zVideoPrismAttention.forward  sT    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST(?(M(MKK,,.E)
 %8
%
  $}}C$2H2HLL//
%
 
%
!\ *k));;;;FFHkk+.L((rG   r   )rH   rI   rJ   r)   rZ   ru   r   r   r   r   rP   r   rz   r{   s   @rD   r  r    sm    D58LL D /3)||) t+) +,	)
 
u||U\\)	*)rG   r  c                   D    e Zd Zdej                  dej                  fdZy)VideoPrismLayerNormr   r   c                     t        j                  || j                  | j                  dz   | j                  | j
                        S r  )F
layer_normnormalized_shapeweightbiaseps)rB   r   s     rD   r   zVideoPrismLayerNorm.forward  s9     ||M4+@+@$++PSBSUYU^U^`d`h`hiirG   N)rH   rI   rJ   r   r   r   rX   rG   rD   r)  r)    s     jU\\ jell jrG   r)  c                   *     e Zd Zdeez  f fdZ xZS )VideoPrismLayerr   c                     t         |   |       t        |      | _        t	        |j
                  |j                        | _        t	        |j
                  |j                        | _        y N)r0  )	rt   ru   r  	attentionr)  r   layer_norm_epslayernorm_beforelayernorm_afterr   s     rD   ru   zVideoPrismLayer.__init__  sR     ,V4 3F4F4FFLaLa b263E3E6K`K`arG   )rH   rI   rJ   r)   rZ   ru   rz   r{   s   @rD   r2  r2    s    b58LL b brG   r2  c                   l    e Zd ZU eed<   dZdZdZg dZdZ	 e
       Z ej                         d        Zy)	VideoPrismPreTrainedModelr   modelr   )videotext)r   r   r2  r   'VideoPrismMultiheadAttentionPoolingHeadFc                    t        j                  | |       t        |t        j                  t        j
                  f      r t        j                  |j                         y t        |t              r t        j                  |j                         y t        |t              r t        j                  |j                         y t        |t              r?t        j                  |j                         t        j                  |j                         y t        |t               r t        j                  |j                         y t        |t"              rt%        |j&                  j(                  |j&                  j*                        j-                  |j.                  j0                  |j.                  j2                        }t        j4                  |j.                  |       t        j4                  |j6                  t9        j:                  |j6                  j<                  d         j?                  d             y t        |t@              rt        jB                  |jD                  jF                  j                  |j&                  j*                  dz         t        jB                  |jD                  jH                  |j&                  j*                  dz         y y )Ndevicer	  r   r   g      )std)%r   _init_weights
isinstancer   LinearConv3dinitlecun_normal_r.  r   r   r   r>  zeros_per_dim_scalepooling_attention_queryr)  r   r   r   r   r   r  r   rA  r	  copy_r   r   r  r   r  VideoPrismTextModelnormal_r   r   r  )rB   moduler   s      rD   rC  z'VideoPrismPreTrainedModel._init_weights  s   %%dF3fryy"))45v}}- ;<v99: <=v99: GHKK,,-v==> 34KK& 89!<55v}}7P7P"b1188@Y@Y@_@_b`  JJv002DEJJv**ELL9L9L9R9RSU9V,W,^,^_f,gh 34LL**::AAv}}G`G`bfGfgLL**228Q8QSW8WX 5rG   N)rH   rI   rJ   rh   rQ   base_model_prefixmain_input_nameinput_modalities_no_split_modules_supports_sdpar@   _input_embed_layerr   no_gradrC  rX   rG   rD   r:  r:    sM    +O( N')U]]_Y YrG   r:  z
    The bare VideoPrism vision encoder outputting raw hidden-states without any specific head on top. This model is the backbone encoder used in VideoPrismVideoModel.
    c                        e Zd ZU eed<   dZdZdef fdZdej                  fdZ
dej                  fdZeee	 	 dd
ej                   d	z  ded	z  dee   defd                     Z xZS )VideoPrismVisionModelr   r<  r;  c                 @   t         |   |       t        |j                  |j                        | _        t        |j                  |j                        | _        t        |      | _        t        |      | _
        t        j                  t        |j                        D cg c]  }t        |       c}      | _        t        j                  t        |j"                        D cg c]  }t        |       c}      | _        | j'                          y c c}w c c}w r4  )rt   ru   r)  r   r6  
layernorm1
layernorm2r   spatial_embeddingsr   temporal_embeddingsr   
ModuleListranger4   r2  spatial_layersr6   temporal_layers	post_initrB   r   r   rx   s      rD   ru   zVideoPrismVisionModel.__init__1  s     -f.@.@fF[F[\-f.@.@fF[F[\"=f"E#?#G  mmeTZTmTmNn,o_V-D,op!}}uU[UoUoOp-q!of.E-qr -p-qs   #D'Dr   c                 .    | j                   j                  S r   r]  r   r   s    rD   get_input_embeddingsz*VideoPrismVisionModel.get_input_embeddings;  s    &&777rG   valuec                 &    || j                   _        y r   rf  rB   rh  s     rD   set_input_embeddingsz*VideoPrismVisionModel.set_input_embeddings>  s    380rG   Nr   r   rC   c                 ,   |t        d      |j                  }| j                  ||      }|}| j                  D ]  } ||fi |} | j	                  |      }| j                  |||      }	|	}
| j                  D ]  } ||
fi |}
 | j                  |
      }|j                  \  }}}|j                  |d   d||      j                  dd      j                         }|j                  \  }}}}|j                  |d   ||z  d      }t        ||
|      S )Nz'You have to specify pixel_values_videosr   r   r0   r   )last_hidden_stater   r   )r   r   r]  ra  r[  r^  rb  r\  r   r   r  r   )rB   r   r   rC   r   spatial_embedsspatial_hidden_statesspatial_layerr   temporal_embedstemporal_hidden_statestemporal_layerr   r/   r   r   s                   rD   r   zVideoPrismVisionModel.forwardA  sM    &FGG)// 001DF^_ .!00 	SM$12G$R6$R!	S??#89 228[Jbc!0"22 	VN%34J%Uf%U"	V??#9: &^^:s==QZEOOPQSTU``b*2..':{C==Qk1I2N:&'=&;
 	
rG   NF)rH   rI   rJ   r)   rQ   rR  rP  ru   r   Modulerg  rk  r   r   r   r   r   rT   r   r   r   r   rz   r{   s   @rD   rX  rX  '  s     #"!5 8bii 89")) 9   9=05#
"..5#
 #'+#
 +,	#

 
5#
    #
rG   rX  c                        e Zd Zdef fdZ	 d	dej                  dej                  dz  dee	   de
ej                  ej                  f   fdZ xZS )
r>  r   c                    t         |   |       | `|j                  |j                  z  | _        d| _        t        j                  t        j                  | j                              | _
        t        | j                  dz  z  | _        t        j                  t        j                  dd|j                              | _        y )Nr  r   r0   )rt   ru   r  intermediate_sizer  r  r   r   r   r   rJ  _R_SOFTPLUS_0r  r   rK  r   s     rD   ru   z0VideoPrismMultiheadAttentionPoolingHead.__init__k  s     $00F4N4NN$'!\\%++dmm*DE$s(:;')||EKK1fFXFX4Y'Z$rG   Nr   r  rC   r   c                    |j                   d d }g |d| j                  }| j                  j                  |d   dd      } | j	                  |      j
                  g |j                   d d d| j                   j                  dd      }|| j                  z  t        j                  j                  | j                        z  }| j                  |      j                  |      j                  dd      }	| j                  |      j                  |      j                  dd      }
t        j                  | j                   j"                  t$              } || ||	|
|fd| j&                  sdn| j(                  d d|\  }} |j*                  g |j                   d d d j-                         }| j/                  |      }||fS )Nr   r   r0   r   r  r_   )r  r   r  )r   r  rK  r  r  r   r   r  r   r   softplusrJ  r  r  r   r  r   r  r   r  re   r   r  r   )rB   r   r  rC   r   r!  queryquery_layerr"  r#  r$  r%  r&  r'  s                 rD   r   z/VideoPrismMultiheadAttentionPoolingHead.forwardt  s    $))#2.88b8$--8,,33KNBK-dkk%(--Su{{3B/?SST]]S]]^_abc"T\\1BMM4J4J4K]K]4^^[[/44\BLLQPQR
{{=166|DNNqRST(?(M(MKK,,.E)
 %8
%
 #}}C$2H2H
%
 
%
!\ *k))@5;;s+;@R@KKMkk+.L((rG   r   )rH   rI   rJ   r)   ru   r   r   r  r   r   rP   r   rz   r{   s   @rD   r>  r>  j  so    [5 [ 37")((") ((4/") +,	")
 
u  %"3"33	4")rG   r>  z
    The bare VideoPrism text encoder outputting last hidden states without any specific head on top. This model is used in VideoPrismClipModel.
    c                        e Zd ZU eed<   dZdZdZddgZdZ	def fdZ
eee	 	 	 	 ddej                  d	z  d
ej                   d	z  dej                   d	z  dej                   d	z  dee   defd                     Z xZS )rM  r   )r=  r;  r  r   r2  r   c                 B   t         |   |       t        |      | _        t	        j
                  t        |j                        D cg c]  }t        |       c}      | _	        t        |j                  |j                        | _        | j                          y c c}w r4  )rt   ru   r   r   r   r_  r`  rU   r2  layersr)  r   r6  	layernormrc  rd  s      rD   ru   zVideoPrismTextModel.__init__  sq     26:mmeFLdLdFe$f_V%<$fg,V-?-?VEZEZ[ %gs   BNr  r  r   rC   r   c                    |d u |d uz  rt        d      | j                  |||      }|lt        j                  |j                  d   d|j
                  |j                        }t        j                  ||fd      }t        | j                  ||d       }| j                  D ]  } |||fi |} | j                  |      }|d d df   }	| j                  j                  rt        |	d      }	t        ||		      S )
Nz:You must specify exactly one of input_ids or inputs_embeds)r  r   r  r   r0   r@  r
  )r   r  r  past_key_valuesr   )rm  pooler_output)r   r   r   onesr   rA  r	  r  r	   r   r  r  r<   r   r   )
rB   r  r  r  r   rC   r   cls_paddinglayertext_embeddingss
             rD   r   zVideoPrismTextModel.forward  s    -t";<YZZ),fst%**##A&.2G2G~OcOcK #YY'D!LN/{{+- $	N [[ 	KE!-J6JM	K}5'2.;;##$_"=O)MYhiirG   )NNNN)rH   rI   rJ   rZ   rQ   rR  rP  rQ  rS  rU  ru   r   r   r   r   r  r   r   r   r   r   rz   r{   s   @rD   rM  rM    s     !  !O35FG*3    .2.2-1,0!j##d*!j t+!j ||d*	!j
 llT)!j +,!j 
$!j    !jrG   rM  z
    VideoPrism video model consisting of the vision encoder backbone with auxiliary encoder layers and an attention pooling head on top. This model is used in VideoPrismClipModel.
    c                        e Zd ZU eed<   def fdZdej                  fdZdej                  fdZ	e
e	 ddej                  ded	z  d
ee   defd              Z xZS )VideoPrismVideoModelr   c                 v   t         |   |       t        j                  |      | _        t        j                  t        |j                        D cg c]  }t        |       c}      | _
        t        |      | _        t        |j                  |j                        | _        | j#                          y c c}w r4  )rt   ru   rX  _from_configvision_modelr   r_  r`  r;   r2  auxiliary_layersr>  headr)  r   r6  head_layernormrc  rd  s      rD   ru   zVideoPrismVideoModel.__init__  s     1>>vF "PUV\VqVqPr.s1v/F.s t;FC	1&2D2D&J_J_` /ts   B6r   c                 6    | j                   j                         S r   r  rg  r   s    rD   rg  z)VideoPrismVideoModel.get_input_embeddings        5577rG   rh  c                 :    | j                   j                  |       y r   r  rk  rj  s     rD   rk  z)VideoPrismVideoModel.set_input_embeddings      ..u5rG   r   r   NrC   c                 R    | j                   d||d|}|j                  }| j                  D ]  } ||fi |}  | j                  |fi |}| j	                  |d         }| j
                  j                  rt        |d      }t        |||j                  |j                        S )Nr   r   r   r   r
  )rm  r  r   
attentionsrX   )r  rm  r  r  r  r   r<   r   r   r   r  )	rB   r   r   rC   vision_model_outputsauxiliary_hidden_statesr  head_outputvideo_embeddingss	            rD   r   zVideoPrismVideoModel.forward  s      1t00  
 3Nf 
jp 
 #7"H"H** 	OE&+,C&Nv&N#	O  dii 7B6B..{1~>;;##%&6B?)5*.<<+66	
 	
rG   r   )rH   rI   rJ   r)   rQ   ru   r   ru  rg  rk  r   r   r   r   rT   r   r   r   r   rz   r{   s   @rD   r  r    s     #"5 8bii 86")) 6  16
"..
 #'+
 +,	

 
$
  
rG   r  z
    VideoPrism model for video-text contrastive learning. This model consists of a VideoPrismVideoModel and a VideoPrismTextModel, and computes similarity scores between video and text inputs.
    c                       e Zd Zdef fdZdej                  fdZdej                  fdZe	e
	 ddej                  d	ej                  dz  d
ee   deez  fd              Ze	e
	 ddej$                  dedz  d
ee   deez  fd              Ze	e
	 	 	 	 ddej$                  dej                  d	ej                  dz  dedz  dedz  dedz  d
ee   defd              Z xZS )VideoPrismClipModelr   c                     t         |   |       t        j                  |j                        | _        t        j                  |j                        | _        | j                          y r   )
rt   ru   r  r  r+   video_modelrM  text_config
text_modelrc  r   s     rD   ru   zVideoPrismClipModel.__init__  sK     /<<V=Q=QR-::6;M;MNrG   r   c                 6    | j                   j                         S r   )r  rg  r   s    rD   rg  z(VideoPrismClipModel.get_input_embeddings  s    3355rG   rh  c                 :    | j                   j                  |       y r   )r  rk  rj  s     rD   rk  z(VideoPrismClipModel.set_input_embeddings  s    ,,U3rG   Nr  r  rC   c                 ,     | j                   d||d|S )a  
        Examples:

        ```python
        >>> from transformers import AutoTokenizer, VideoPrismClipModel

        >>> model = VideoPrismClipModel.from_pretrained("google/videoprism-lvt-base-f16r288")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/videoprism-lvt-base-f16r288")

        >>> inputs = tokenizer(["a video of a cat.", "a video of a dog."], padding="max_length", return_tensors="pt")
        >>> with torch.no_grad():
        ...     text_features = model.get_text_features(**inputs)
        ```r  r  rX   )r  )rB   r  r  rC   s       rD   get_text_featuresz%VideoPrismClipModel.get_text_features  s     * t\>\U[\\rG   r   r   c                 ,     | j                   d||d|S )a  
        Examples:

        ```python
        >>> from transformers import VideoPrismProcessor, VideoPrismClipModel

        >>> model = VideoPrismClipModel.from_pretrained("google/videoprism-lvt-base-f16r288")
        >>> processor = VideoPrismProcessor.from_pretrained("google/videoprism-lvt-base-f16r288")

        >>> inputs = processor(videos="path/to/video.mp4", return_tensors="pt")
        >>> with torch.no_grad():
        ...     video_features = model.get_video_features(**inputs)
        ```r  rX   )r  )rB   r   r   rC   s       rD   get_video_featuresz&VideoPrismClipModel.get_video_features/  s.    *  t 
 3%=
 
 	
rG   temperaturereturn_lossc           	      r    | j                   d||d|} | j                  d||d|}	|j                  }
|	j                  }|
j                  d   }|j                  d   }|
j	                  d|      }|j	                  d|      }t        j                  ||j                        }|||z  }t        j                  |      }|j                  }|t        j                  |dd      z  }|t        j                  |dd      z  }d}|rt        j                  |j                  d      |j                        }t        j                  |       d	|z  z   }t
        j                  j                  j!                  ||z        }t        j                  |d
       }|j#                         }t%        ||||||	|      S )a  
        temperature (`float`, *optional*):
            A temperature scalar to scale the similarity scores. If not provided, no scaling is applied.
        return_loss (`bool`, *optional*):
            Whether or not to return the contrastive loss.
        r  r  r   Nr   T)r   keepdims)rA  r   r
  )r   r   r   r   r   r   r   rX   )r  r  r  r   r   r   matmulTexpsumeyer   rA  	ones_liker   r   
logsigmoidmeanr   )rB   r   r  r  r   r  r  rC   video_model_outputstext_model_outputsr  r  video_emb_dimtext_emb_dimr   r   similarity_matrixr   r   r   r  m1_diag1logliknlls                           rD   r   zVideoPrismClipModel.forwardJ  s   & 6d55 
 3Nf
jp
 4T33qiXfqjpq.<<,::(..r2&,,R0'//MB%--b,?!LL{}}E", 99%67*,,+eii8HaZ^.__)EIIo1W[,\\ ))O003O<R<RSC881s7BHXX((33H4NOF99V,,C88:D#-+%#20
 	
rG   r   r   )NFNN)rH   rI   rJ   rh   ru   r   ru  rg  rk  r   r   r   r   r   r   rP   r   r  r   rT   r  rS   r   r   rz   r{   s   @rD   r  r    s   / 6bii 64")) 4  /3]<<] t+] +,	]
 
+	+]  ]*  16
"..
 #'+
 +,	

 
+	+
  
2 
 /305$(#'9
"..9
 <<9
 t+	9

 #'+9
 T\9
 D[9
 +,9
 
9
  9
rG   r  z
    VideoPrism Model transformer with a video classification head on top (a linear layer on top of the attention pooler).
    c                        e Zd ZU eed<   dZdZdef fdZdej                  fdZ
dej                  fdZee	 	 dd
ej                  dej                   d	z  ded	z  dee   def
d              Z xZS ) VideoPrismForVideoClassificationr   rY  r;  c                 B   t         |   |       t        j                  |      | _        t        |      | _        t        |j                  |j                        | _
        t        j                  |j                  |j                        | _        | j                          y r4  )rt   ru   rX  r  r  r>  r  r)  r   r6  r  r   rE  
num_labels
classifierrc  r   s     rD   ru   z)VideoPrismForVideoClassification.__init__  sr     1>>vF;FC	1&2D2D&J_J_`))F$6$68I8IJrG   r   c                 6    | j                   j                         S r   r  r   s    rD   rg  z5VideoPrismForVideoClassification.get_input_embeddings  r  rG   rh  c                 :    | j                   j                  |       y r   r  rj  s     rD   rk  z5VideoPrismForVideoClassification.set_input_embeddings  r  rG   Nr   labelsr   rC   c                 <    | j                   d||d|}|j                  }| j                   | j                  |fi |d         }| j	                  |      }d }	| | j
                  ||| j                  fi |}	t        |	||j                  |j                        S )Nr  r   )r   logitsr   r  rX   )
r  rm  r  r  r  loss_functionr   r   r   r  )
rB   r   r  r   rC   r  sequence_outputpooled_outputr  r   s
             rD   r   z(VideoPrismForVideoClassification.forward  s      1t00  
 3Nf 
jp 
 /@@++IDIIo,P,PQR,ST/%4%%ffdkkLVLD$.<<+66	
 	
rG   rt  )rH   rI   rJ   r)   rQ   rR  rP  ru   r   ru  rg  rk  r   r   r   r   r  rT   r   r   r   r   rz   r{   s   @rD   r  r    s     #"!5 8bii 86")) 6  +/05	
"..
   4'
 #'+	

 +,
 

  
rG   r  )r)   rZ   rh   rX  r:  r  rM  r  r  rk   r   )Ycollections.abcr   dataclassesr   typingr   r   torch.nnr   torch.nn.functionalr   r+  huggingface_hub.dataclassesr    r   rG  masking_utilsr	   modeling_outputsr
   r   r   modeling_utilsr   r   processing_utilsr   r   r   utilsr   r   r   r   r   r   utils.genericr   utils.output_capturingr   codegen.modeling_codegenr   gemma2.modeling_gemma2r   qwen3_next.modeling_qwen3_nextr   siglip.configuration_siglipr   r   t5.tokenization_t5r    vivit.configuration_vivitr!   vivit.modeling_vivitr"   r#   r$   r%   r&   
get_loggerrH   loggerry  r)   rZ   rh   rk   r}   r   r   r   r   r   r   ru  r   r  	LayerNormr)  r2  r:  rX  r>  rM  r  r  r  __all__rX   rG   rD   <module>r     s    % !      . & / b b F H H j j 7 5 B < 3 H , 3  
		H	% ;<".[ ".  =".J ?@.+ .  A.2 ?@*| *  A**++ +D 0  5. 5 5 ij
?/ ?  k? B  
;  
  
F"8 B=/ =@8? 8v ryy  F&). &)Rj",, jbj b -Y 4 -Y -Y` 
;
5 ;

;
|,)n ,)^ 
3j3 3j
3jl 
*
4 *

*
Z 
z
3 z

z
z 
+
'@ +

+
\rG   