Ë
    ÜÓ^j]œ  ã                   óÀ  — d dl mZ d dlmZ d dlmZ d dlZd dlmZ d dl	mc m
Z d dlmZ ddlmZ ddlmZ dd	lmZmZmZ dd
lmZmZ ddlmZmZmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z* ddl+m,Z, ddl-m.Z. ddl/m0Z0m1Z1 ddl2m3Z3 ddl4m5Z5 ddl6m7Z7m8Z8m9Z9m:Z:m;Z;  e#jx                  e=«      Z>dZ? e!d¬«      e G d„ de5«      «       «       Z@ e!d¬«      e G d„ de1«      «       «       ZA e!d¬«      e G d„ d e0«      «       «       ZB G d!„ d"e3«      ZC G d#„ d$ed%¬&«      ZDe! G d'„ d(e«      «       ZE e!d)¬*«      e G d+„ d,e«      «       «       ZF e!d-¬*«      e G d.„ d/e«      «       «       ZG G d0„ d1e;«      ZH G d2„ d3e8«      ZI G d4„ d5e8«      ZJ G d6„ d7ej–                  «      ZL G d8„ d9e7«      ZM G d:„ d;ejœ                  «      ZO G d<„ d=e9«      ZPe! G d>„ d?e:«      «       ZQ e!d@¬*«       G dA„ dBeQ«      «       ZR G dC„ dDe7«      ZS e!dE¬*«       G dF„ dGeQ«      «       ZT e!dH¬*«       G dI„ dJeQ«      «       ZU e!dK¬*«       G dL„ dMeQ«      «       ZV e!dN¬*«       G dO„ dPeQ«      «       ZWg dQ¢ZXy)Ré    )ÚCallable)Ú	dataclass)ÚAnyN)Ústricté   )Úinitialization)Úcreate_causal_mask)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚImageClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚProcessingKwargsÚProcessorMixinÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚloggingÚ	torch_int)Úmerge_with_config_defaults)Úcapture_outputsé   )Úcreate_sinusoidal_positions)Úeager_attention_forward)Úl2norm)ÚSiglipConfigÚSiglipTextConfig)ÚT5Tokenizer)ÚVivitConfig)ÚVivitAttentionÚVivitEmbeddingsÚ
VivitLayerÚVivitPreTrainedModelÚVivitTubeletEmbeddingsg^$3eG÷?zgoogle/videoprism-base-f16r288)Ú
checkpointc                   óú   — e Zd ZU dZdZdZdZeee   z  e	eef   z  e
d<   dZee
d<   dZee   e	ed	f   z  e
d
<   dZee
d<   dZee
d<   dZee
d<   dZee
d<   dZee
d<   dZee
d<    e«       Z e«       Z e«       Zd„ Zy)ÚVideoPrismVisionConfiga™  
    num_frames (`int`, *optional*, defaults to 16):
        The number of frames in the input video.
    tubelet_size (`List[int]`, *optional*, defaults to `[1, 18, 18]`):
        The size of the tubelet patch.
    num_spatial_layers (`int`, *optional*, defaults to 12):
        Number of spatial transformer blocks.
    num_temporal_layers (`int`, *optional*, defaults to 4):
        Number of temporal transformer blocks.
    attn_logit_softcapping (`float`, *optional*, defaults to 50.0):
        Softcapping constant for attention logits.
    num_auxiliary_layers (`int`, *optional*, defaults to 2):
        Number of auxiliary layers. This is used in the VideoPrismVideoModel that is a part of VideoPrismClipModel.
    apply_l2norm (`bool`, *optional*, defaults to `True`):
        Whether to apply L2 normalization to the output. This is used in the VideoPrismVideoModel that is a part of VideoPrismClipModel.
    Úvideoprism_vision_modelÚvision_configé   Ú
image_sizeé   Ú
num_frames)é   é   r1   .Útubelet_sizeé   Únum_spatial_layersé   Únum_temporal_layersÚgelu_pythonÚ
hidden_actç      I@Úattn_logit_softcappingr   Únum_auxiliary_layersTÚapply_l2normc                 ó   — t        d«      ‚©NzNot used here©ÚAttributeError©ÚselfÚkwargss     ú|/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/videoprism/modular_videoprism.pyÚ__post_init__z$VideoPrismVisionConfig.__post_init__X   ó   € Ü˜_Ó-Ð-ó    N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚbase_config_keyr-   ÚintÚlistÚtupleÚ__annotations__r/   r2   r4   r6   r8   Ústrr:   Úfloatr;   r<   Úboolr@   Únum_hidden_layersÚ
pooler_actÚpooler_output_sizerE   © rG   rD   r)   r)   5   s¶   … ñð" +€JØ%€OØ47€J��d˜3‘i‘ %¨¨S¨¡/Ñ1Ó7Ø€J�ÓØ0;€L�$�s‘)˜e C¨ H™oÑ-Ó;Ø Ð˜Ó Ø Ð˜Ó Ø#€J�Ó#Ø$(Ð˜EÓ(Ø !Ð˜#Ó!Ø€L�$ÓÙ&Ó(ÐÙÓ!€JÙ'Ó)Ðó.rG   r)   z"google/videoprism-lvt-base-f16r288c                   óä   — e Zd ZU dZdZeed<   dZedz  ed<   dZ	edz  ed<   dZ
eee   z  dz  ed<   d	Zeez  ed
<   dZeed<   dZeed<   d	Zeed<   dZeed<   dZeed<    e«       Z e«       Zd„ Zy)ÚVideoPrismTextConfiga	  
    apply_l2norm (`bool`, *optional*, defaults to `True`):
        Whether to apply L2 normalization to the output of VideoPrismTextEncoder.
    attn_logit_softcapping (`float`, *optional*, defaults to 50.0):
        Softcapping constant for attention logits.
    Úrelur8   r   NÚpad_token_idÚbos_token_idÚeos_token_idç        Úattention_probs_dropout_probTr<   Úqkv_biasÚhidden_dropout_probg{®Gáz”?Úinitializer_ranger9   r:   c                 ó   — t        d«      ‚r>   r?   rA   s     rD   rE   z"VideoPrismTextConfig.__post_init__s   rF   rG   )rH   rI   rJ   rK   r8   rR   rQ   r\   rN   r]   r^   rO   r`   rS   r<   rT   ra   rb   rc   r:   r@   Úattention_dropoutÚprojection_sizerE   rX   rG   rD   rZ   rZ   \   s    … ñð €J�ÓØ €L�#˜‘*Ó Ø#€L�#˜‘*Ó#Ø+/€L�#˜˜S™	‘/ DÑ(Ó/Ø03Ð  %¨#¡+Ó3Ø€L�$ÓØ€HˆdÓØ!$Ð˜Ó$Ø#Ð�uÓ#Ø$(Ð˜EÓ(Ù&Ó(ÐÙ$Ó&€Oó.rG   rZ   c                   ó   — e Zd ZdZ e«       Zy)ÚVideoPrismConfiga¤  
    Example:

    ```python
    >>> from transformers import VideoPrismClipModel, VideoPrismConfig

    >>> # Initializing a VideoPrismConfig with default values
    >>> configuration = VideoPrismConfig()

    >>> # Initializing a VideoPrismClipModel with the configuration
    >>> model = VideoPrismClipModel(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    N)rH   rI   rJ   rK   r@   Úinitializer_factorrX   rG   rD   rh   rh   w   s   „ ññ" (Ó)ÑrG   rh   c                   óT   ‡ — e Zd ZdZ	 	 	 	 	 	 	 ddeeeeef      z  dz  fˆ fd„Zˆ xZ	S )ÚVideoPrismTokenizeraV  
    Constructs a VideoPrism tokenizer, which is essentially a T5 tokenizer without its postprocessor
    (appending an EOS token at the end of the sequence).

    This tokenizer inherits from [`T5Tokenizer`] which contains most of the main methods. Users should refer to this
    superclass for more information regarding those methods.
    NÚvocabc                 óL   •— t        ‰	| �  d|||||||dœ|¤Ž | j                  `y )N)rl   Ú	eos_tokenÚ	unk_tokenÚ	pad_tokenÚ_spm_precompiled_charsmapÚ	extra_idsÚadditional_special_tokensrX   )ÚsuperÚ__init__Ú
_tokenizerÚpost_processor)
rB   rl   rn   ro   rp   rq   rr   rs   rC   Ú	__class__s
            €rD   ru   zVideoPrismTokenizer.__init__—   sB   ø€ ô 	‰Ñð 		
ØØØØØ&?ØØ&?ñ		
ð ò		
ð �O‰OÑ*rG   )Nz</s>z<unk>z<pad>Néd   N)
rH   rI   rJ   rK   rR   rO   rP   rS   ru   Ú__classcell__©rx   s   @rD   rk   rk   Ž   sI   ø„ ñð 7;ØØØØ"&ØØ"&ñ+à�T˜%  U 
Ñ+Ñ,Ñ,¨tÑ3÷+ñ +rG   rk   c                   ó,   — e Zd Zddddœdddœdddœd	œZy
)ÚVideoPrismProcessorKwargsÚ
max_lengthTé@   )ÚpaddingÚ
truncationr~   r,   )ÚheightÚwidthF)ÚsizeÚdo_normalizeÚdo_sample_frames)Útext_kwargsÚvideo_kwargsN)rH   rI   rJ   Ú	_defaultsrX   rG   rD   r}   r}   °   s.   „ ð $ØØñ
ð  #¨SÑ1Ø!Ø $ñ
ñ�IrG   r}   F)Útotalc                   ó$   ‡ — e Zd ZeZdˆ fd„	Zˆ xZS )ÚVideoPrismProcessorc                 ó&   •— t         ‰| �  ||«       y ©N)rt   ru   )rB   Úvideo_processorÚ	tokenizerrx   s      €rD   ru   zVideoPrismProcessor.__init__Ã   s   ø€ Ü‰Ñ˜¨)Õ4rG   )NN)rH   rI   rJ   r}   Úvalid_processor_kwargsru   rz   r{   s   @rD   rŒ   rŒ   ¿   s   ø„ à6Ð÷5ñ 5rG   rŒ   zFBase class for model outputs that include spatial and temporal states.)Úcustom_introc                   ób   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   y)Ú+BaseModelOutputWithSpatialAndTemporalStatesa™  
    last_temporal_hidden_state (`torch.FloatTensor`, *optional*):
        The last hidden state of the temporal encoder, typically of shape
        `(batch_size * num_patches, num_frames, hidden_size)`.
    last_spatial_hidden_state (`torch.FloatTensor`, *optional*):
        The last hidden state of the spatial encoder, typically of shape
        `(batch_size * num_frames, num_patches, hidden_size)`.
    NÚlast_temporal_hidden_stateÚlast_spatial_hidden_state)	rH   rI   rJ   rK   r•   ÚtorchÚFloatTensorrQ   r–   rX   rG   rD   r”   r”   Ç   s6   … ñð <@Ð × 1Ñ 1°DÑ 8Ó?Ø:>Ð˜u×0Ñ0°4Ñ7Ô>rG   r”   z+Base class for VideoPrismClipModel outputs.c                   ó  — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
ej                  dz  ed<   dZeed<   dZeed<   dZej                  dz  ed	<   d
ee   fd„Zy)ÚVideoPrismClipOutputa¼  
    logits_per_video (`torch.FloatTensor` of shape `(video_batch_size, text_batch_size)`):
        The scaled dot product scores between `video_embeds` and `text_embeds`. This represents the video-text
        similarity scores.
    logits_per_text (`torch.FloatTensor` of shape `(text_batch_size, video_batch_size)`):
        The scaled dot product scores between `text_embeds` and `video_embeds`. This represents the text-video
        similarity scores.
    video_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)`):
        The video embeddings obtained by applying the projection layer to the pooled output of [`VideoPrismVideoModel`].
    text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)`):
        The text embeddings obtained by applying the projection layer to the pooled output of [`VideoPrismTextModel`].
    video_model_output (`BaseModelOutputWithPooling`):
        The output of [`VideoPrismVideoModel`].
    text_model_output (`BaseModelOutputWithPooling`):
        The output of the [`VideoPrismTextModel`].
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
        Contrastive loss for video-text similarity.
    NÚlogits_per_videoÚlogits_per_textÚvideo_embedsÚtext_embedsÚvideo_model_outputÚtext_model_outputÚlossÚreturnc                 óH   ‡ — t        ˆ fd„‰ j                  «       D «       «      S )Nc              3   ód   •K  — | ]'  }|d vr‰|   nt        ‰|«      j                  «       –— Œ) y­w))r    rŸ   N)ÚgetattrÚto_tuple)Ú.0ÚkrB   s     €rD   ú	<genexpr>z0VideoPrismClipOutput.to_tuple.<locals>.<genexpr>ø   s=   øè ø€ ò 
àð Ð KÑKˆD�ŠGÔQXÐY]Ð_`ÓQa×QjÑQjÓQlÓlñ
ùs   ƒ-0)rP   Úkeys©rB   s   `rD   r¦   zVideoPrismClipOutput.to_tuple÷   s#   ø€ Üó 
à—Y‘Y“[ô
ó 
ð 	
rG   )rH   rI   rJ   rK   r›   r—   r˜   rQ   rœ   r�   rž   rŸ   r   r    r¡   rP   r   r¦   rX   rG   rD   rš   rš   ×   s›   … ñ
ð& 26Ð�e×'Ñ'¨$Ñ.Ó5Ø04€O�U×&Ñ&¨Ñ-Ó4Ø-1€L�%×#Ñ# dÑ*Ó1Ø,0€K�×"Ñ" TÑ)Ó0Ø59ÐÐ2Ó9Ø48ÐÐ1Ó8Ø%)€Dˆ%×
Ñ
˜dÑ
"Ó)ð
˜% ™*ô 
rG   rš   c                   óf   ‡ — e Zd ZdZdefˆ fd„Zddej                  dedej                  fd„Z	ˆ xZ
S )	ÚVideoPrismTubeletEmbeddingsañ  
    VideoPrism Tubelet Embeddings.

    The authors of Videoprism use the Factorized Encoder architecture, i.e. "Model 2", introduced in the VIVIT paper (https://huggingface.co/papers/2103.15691).
    This differs from Vivit by using a convolution of `tubelet_size=(1, 18, 18)`, which is essentially a 2d convolution in the spatial dimension.
    The temporal dimension is also merged with the `batch_size` in order to make sure the image embeddings have no temporal component, unlike Vivit.
    Úconfigc                 óÞ   •— t         ‰| �  |«       | `| j                  d   t        d   z  | j                  d   t        d   z  g| _        | j
                  d   | j
                  d   z  | _        y )Nr   r0   r   )rt   ru   Únum_patchesr-   r2   Úpos_emb_shape©rB   r®   rx   s     €rD   ru   z$VideoPrismTubeletEmbeddings.__init__  sn   ø€ Ü‰Ñ˜Ô ØÐØ"Ÿo™o¨aÑ0´LÀ±OÑCÀTÇ_Á_ÐUVÑEWÔ[gÐhiÑ[jÑEjÐkˆÔØ×-Ñ-¨aÑ0°4×3EÑ3EÀaÑ3HÑHˆÕrG   Úpixel_values_videosÚinterpolate_pos_encodingr¢   c                 ó´  — |j                   \  }}}}}|sV|| j                  d   k7  s|| j                  d   k7  r2t        d|› d|› d| j                  d   › d| j                  d   › d�	«      ‚|j                  dd«      }| j	                  |«      }|j                  d«      j                  dddd«      }|j                   \  }}}	}
|j                  ||z  |	|
«      }|S )	Nr   r0   zImage size (Ú*z) doesn't match model (z[). Set interpolate_pos_encoding=True to automatically resize the model position embeddings.r   r   )Úshaper-   Ú
ValueErrorÚ	transposeÚ
projectionÚflattenÚpermuteÚreshape)rB   r³   r´   Ú
batch_sizer/   Únum_channelsr‚   rƒ   Úhidden_statesr°   Úhidden_sizes              rD   Úforwardz#VideoPrismTubeletEmbeddings.forward  s  € Ø>Q×>WÑ>WÑ;ˆ
�J ¨f°eÙ'¨V°t·±ÀqÑ7IÒ-IÈUÐVZ×VeÑVeÐfgÑVhÒMhÜØ˜v˜h a¨ wÐ.EÀdÇoÁoÐVWÑFXÐEYÐYZÐ[_×[jÑ[jÐklÑ[mÐZnð  oJð  Kóð ð 2×;Ñ;¸A¸qÓAÐØŸ™Ð(;Ó<ˆà%×-Ñ-¨aÓ0×8Ñ8¸¸A¸qÀ!ÓDˆà;H×;NÑ;NÑ8ˆ
�J ¨[Ø%×-Ñ-¨j¸:Ñ.EÀ{ÐT_Ó`ˆàÐrG   ©F)rH   rI   rJ   rK   r)   ru   r—   ÚTensorrT   rÂ   rz   r{   s   @rD   r­   r­   þ   s=   ø„ ñðIÐ5õ Iñ¨5¯<©<ð ÐSWð Ðdi×dpÑdp÷ rG   r­   c                   óª   ‡ — e Zd Zdefˆ fd„Zdej                  dededej                  fd„Z	 ddej                  d	e	d
z  dej                  fd„Z
ˆ xZS )ÚVideoPrismSpatialEmbeddingsr®   c                 ó¦   •— t         ‰| �  |«       | `| `t	        j
                  t        j                  dt        |j                  «      «      | _
        y ©Nr0   )rt   ru   Ú	cls_tokenr-   ÚnnÚ	Parameterr—   Úzerosr°   rÁ   Úposition_embeddingsr²   s     €rD   ru   z$VideoPrismSpatialEmbeddings.__init__   s>   ø€ Ü‰Ñ˜Ô ØˆNØˆOÜ#%§<¡<´·±¸A¼{ÈF×L^ÑL^Ó0_Ó#`ˆÕ rG   Ú
embeddingsr‚   rƒ   r¢   c                 ó8  — |j                   d   }| j                  j                   d   }t        j                  j	                  «       s||k(  r||k(  r| j                  S |j                   d   }|| j
                  d   z  }|| j
                  d   z  }t        |dz  «      }	| j                  j                  d|	|	|«      }
|
j                  dddd«      }
t        j                  j                  |
||fdd¬	«      }
|
j                  dddd«      j                  dd|«      }
|
S )
a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r0   éÿÿÿÿr   ç      à?r   r   ÚbilinearT©r„   ÚmodeÚ	antialias)r·   rÍ   r—   ÚjitÚ
is_tracingÚ
patch_sizer   r½   r¼   rÊ   Ú
functionalÚinterpolateÚview)rB   rÎ   r‚   rƒ   r°   Únum_positionsÚdimÚnum_row_patchesÚnum_col_patchesÚsqrt_num_positionsÚpatch_pos_embeds              rD   r´   z4VideoPrismSpatialEmbeddings.interpolate_pos_encoding&  s*  € ð !×&Ñ& qÑ)ˆØ×0Ñ0×6Ñ6°qÑ9ˆô �y‰y×#Ñ#Ô%¨+¸Ò*FÈ6ÐUZÊ?Ø×+Ñ+Ð+à×Ñ˜rÑ"ˆà  D§O¡O°AÑ$6Ñ6ˆØ 4§?¡?°1Ñ#5Ñ5ˆä& }°cÑ'9Ó:ÐØ×2Ñ2×:Ñ:¸1Ð>PÐRdÐfiÓjˆØ)×1Ñ1°!°Q¸¸1Ó=ˆô Ÿ-™-×3Ñ3ØØ! ?Ð3ØØð	 4ó 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆØÐrG   r³   r´   Nc                 óÀ   — |j                   \  }}}}}| j                  ||«      }|r|| j                  |||«      z   }n|| j                  z   }| j	                  |«      }|S rŽ   )r·   Úpatch_embeddingsr´   rÍ   Údropout)	rB   r³   r´   ÚbatchÚframesÚchannelr‚   rƒ   rÎ   s	            rD   rÂ   z#VideoPrismSpatialEmbeddings.forwardL  ss   € ð
 1D×0IÑ0IÑ-ˆˆv�w ¨Ø×*Ñ*Ð+>Ð@XÓYˆ
ñ $Ø# d×&CÑ&CÀJÐPVÐX]Ó&^Ñ^‰Jà# d×&>Ñ&>Ñ>ˆJà—\‘\ *Ó-ˆ
àÐrG   rÃ   )rH   rI   rJ   r)   ru   r—   rÄ   rN   r´   rT   rÂ   rz   r{   s   @rD   rÆ   rÆ     sr   ø„ ðaÐ5õ að$°5·<±<ð $Èð $ÐUXð $Ð]b×]iÑ]ió $ðR 16ñà"Ÿ\™\ðð #'¨¡+ðð 
�‰÷	rG   rÆ   c            	       ó¾   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Z	 ddej                  dej                  d	e
d
z  dej                  fd„Zˆ xZS )ÚVideoPrismTemporalEmbeddingszÍ
    VideoPrism Temporal Embeddings.

    Receives embeddings from spatial encoder, reshapes the hidden state to
    (batch_size * num_patches, num_frames, hidden_size) and adds positional embeddings.
    r®   c                 ó¼   •— t         ‰| �  |«       | `| `| `~| `t        j                  t        j                  d|j                  |j                  «      «      | _        y rÈ   )rt   ru   rÉ   rã   rØ   r-   rÊ   rË   r—   rÌ   r/   rÁ   rÍ   )rB   r®   r°   rx   s      €rD   ru   z%VideoPrismTemporalEmbeddings.__init__g  sS   ø€ Ü‰Ñ˜Ô ØˆNØÐ!ØˆOØØˆOÜ#%§<¡<´·±¸A¸v×?PÑ?PÐRX×RdÑRdÓ0eÓ#fˆÕ rG   rÎ   r¢   c                 ót  — |j                   d   }| j                  j                   d   }t        j                  j	                  «       s||k(  r| j                  S | j                  }|j                   d   }|j                  d«      }t        j                  j                  |||fdd¬«      }|j                  d«      S )Nr0   rÐ   rÒ   TrÓ   )
r·   rÍ   r—   rÖ   r×   Ú	unsqueezerÊ   rÙ   rÚ   Úsqueeze)rB   rÎ   Útarget_emb_lengthÚsource_emb_lengthÚ
source_embrÝ   s         rD   r´   z5VideoPrismTemporalEmbeddings.interpolate_pos_encodingp  s¹   € Ø&×,Ñ,¨QÑ/ÐØ ×4Ñ4×:Ñ:¸1Ñ=Ðô �y‰y×#Ñ#Ô%Ð*;Ð?PÒ*PØ×+Ñ+Ð+à×-Ñ-ˆ
Ø×Ñ˜rÑ"ˆØ×)Ñ)¨!Ó,ˆ
Ü—]‘]×.Ñ.ØØ# SÐ)ØØð	 /ó 
ˆ
ð ×!Ñ! !Ó$Ð$rG   r³   Úinput_shaper´   Nc                 ó   — |�|\  }}}}}|j                   \  }	}
}|j                  |
|«      }|j                  dd«      }|j                  ||
z  ||«      }|r|| j	                  |«      z   }n|| j
                  z   }| j                  |«      }|S )Nr   r0   )r·   rÛ   r¹   r½   r´   rÍ   rä   )rB   r³   rñ   r´   rå   ræ   rç   r‚   rƒ   Ú_ÚfeaturesrÝ   rÀ   rÎ   s                 rD   rÂ   z$VideoPrismTemporalEmbeddings.forward„  s«   € ð Ð"Ø4?Ñ1ˆE�6˜7 F¨EØ.×4Ñ4Ñˆˆ8�SØ+×0Ñ0°¸ÀÈ#ÓNˆØ%×/Ñ/°°1Ó5ˆØ"×*Ñ*¨5°8Ñ+;¸VÀSÓIˆ
ñ $Ø# d×&CÑ&CÀJÓ&OÑO‰Jà# d×&>Ñ&>Ñ>ˆJØ—\‘\ *Ó-ˆ
ØÐrG   rÃ   )rH   rI   rJ   rK   r)   ru   r—   rÄ   r´   ÚSizerT   rÂ   rz   r{   s   @rD   ré   ré   _  sr   ø„ ñðgÐ5õ gð%°5·<±<ð %ÀEÇLÁLó %ð0 16ñ	à"Ÿ\™\ðð —Z‘Zðð #'¨¡+ð	ð
 
�‰÷rG   ré   c            	       ó¦   ‡ — e Zd Zdefˆ fd„Z	 	 	 d	dej                  dz  dej                  dz  dej                  dz  dej                  fd„Z	ˆ xZ
S )
ÚVideoPrismTextEmbeddingsr®   c                 ó  •— t         ‰| �  «        || _        |j                  }t	        j
                  |j                  |«      | _        | j                  dt        |j                  |j                  «      «       | j                  dt        j                  |j                  «      j                  d«      «       t	        j                  t        j                  dd|j                  «      «      | _        |j                  dz  | _        y )NÚposition_embeddingÚposition_ids©r0   rÐ   r0   rÑ   )rt   ru   r®   rÁ   rÊ   Ú	EmbeddingÚ
vocab_sizeÚtoken_embeddingÚregister_bufferr   Úmax_position_embeddingsr—   ÚarangeÚexpandrË   rÌ   Úcls_embÚscaling)rB   r®   Ú	embed_dimrx   s      €rD   ru   z!VideoPrismTextEmbeddings.__init__›  sÃ   ø€ Ü‰ÑÔØˆŒØ×&Ñ&ˆ	Ü!Ÿ|™|¨F×,=Ñ,=¸yÓIˆÔØ×ÑØ Ô"=¸f×>\Ñ>\Ð^d×^pÑ^pÓ"qô	
ð 	×Ñ˜^¬U¯\©\¸&×:XÑ:XÓ-Y×-`Ñ-`ÐahÓ-iÔjÜ—|‘|¤E§K¡K°°1°f×6HÑ6HÓ$IÓJˆŒØ×)Ñ)¨3Ñ.ˆ�rG   NÚ	input_idsrú   Úinputs_embedsr¢   c                 ó’  — |€| j                  |«      }|€"| j                  d d …d |j                  d   …f   }|| j                  z  }| j                  |   j                  |j                  ¬«      }||z   }| j                  | j                  z  }|j                  |j                  d   dd«      }t        j                  ||fd¬«      }|S )Nr0   )Údtyper   rÐ   ©rÝ   )rþ   rú   r·   r  rù   Útor	  r  r  r—   Úcat)rB   r  rú   r  rÍ   rÎ   r  s          rD   rÂ   z VideoPrismTextEmbeddings.forward§  sÎ   € ð Ð Ø ×0Ñ0°Ó;ˆMàÐØ×,Ñ,ªQÐ0H°-×2EÑ2EÀaÑ2HÐ0HÐ-HÑIˆLà%¨¯©Ñ4ˆØ"×5Ñ5°lÑC×FÑFÈ]×M`ÑM`ÐFÓaÐØ"Ð%8Ñ8ˆ
à—,‘, §¡Ñ-ˆØ—.‘. ×!1Ñ!1°!Ñ!4°b¸"Ó=ˆÜ—Y‘Y 
¨GÐ4¸!Ô<ˆ
ØÐrG   )NNN)rH   rI   rJ   rZ   ru   r—   Ú
LongTensorr˜   rÄ   rÂ   rz   r{   s   @rD   r÷   r÷   š  sk   ø„ ð
/Ð3õ 
/ð .2Ø04Ø26ñ	à×#Ñ# dÑ*ðð ×&Ñ&¨Ñ-ðð ×(Ñ(¨4Ñ/ð	ð
 
�‰÷rG   r÷   c                   ó¬   ‡ — e Zd Zdeez  fˆ fd„Z	 d	dej                  dej                  dz  dee	   de
ej                  ej                  f   fd„Zˆ xZS )
ÚVideoPrismAttentionr®   c                 óX   •— t         ‰| �  |«       | `d| _        |j                  | _        y ©Nç      ð?)rt   ru   Únum_attention_headsÚnum_key_value_groupsr:   r²   s     €rD   ru   zVideoPrismAttention.__init__¾  s,   ø€ Ü‰Ñ˜Ô ØÐ$Ø$'ˆÔ!Ø&,×&CÑ&CˆÕ#rG   NrÀ   Úattention_maskrC   r¢   c                 ó°  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }t        j                  | j                  j                  t        «      }	 |	| ||||f| j                  sdn| j                  | j                  | j                  dœ|¤Ž\  }
} |
j                   g |¢d‘­Ž j#                  «       }
| j%                  |
«      }
|
|fS )NrÐ   r0   r   r_   )rä   r  Úsoftcap)r·   Úhead_dimÚq_projrÛ   r¹   Úk_projÚv_projr   Úget_interfacer®   Ú_attn_implementationr   Útrainingre   r  r:   r½   Ú
contiguousÚo_proj)rB   rÀ   r  rC   rñ   Úhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚattention_interfaceÚattn_outputÚattn_weightss               rD   rÂ   zVideoPrismAttention.forwardÄ  sT  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LØ×/Ñ/ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(rG   rŽ   )rH   rI   rJ   r)   rZ   ru   r—   rÄ   r   r   rP   rÂ   rz   r{   s   @rD   r  r  ½  sm   ø„ ðDÐ5Ð8LÑLõ Dð /3ñ)à—|‘|ð)ð Ÿ™ tÑ+ð)ð Ð+Ñ,ð	)ð
 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷)rG   r  c                   óD   — e Zd Zdej                  dej                  fd„Zy)ÚVideoPrismLayerNormrÀ   r¢   c                 óŠ   — t        j                  || j                  | j                  dz   | j                  | j
                  «      S r  )ÚFÚ
layer_normÚnormalized_shapeÚweightÚbiasÚeps)rB   rÀ   s     rD   rÂ   zVideoPrismLayerNorm.forwardç  s9   € ô �|‰|˜M¨4×+@Ñ+@À$Ç+Á+ÐPSÑBSÐUY×U^ÑU^Ð`d×`hÑ`hÓiÐirG   N)rH   rI   rJ   r—   rÄ   rÂ   rX   rG   rD   r)  r)  æ  s    „ ðj U§\¡\ð j°e·l±lô jrG   r)  c                   ó*   ‡ — e Zd Zdeez  fˆ fd„Zˆ xZS )ÚVideoPrismLayerr®   c                 óÜ   •— t         ‰| �  |«       t        |«      | _        t	        |j
                  |j                  ¬«      | _        t	        |j
                  |j                  ¬«      | _        y ©N)r0  )	rt   ru   r  Ú	attentionr)  rÁ   Úlayer_norm_epsÚlayernorm_beforeÚlayernorm_afterr²   s     €rD   ru   zVideoPrismLayer.__init__ï  sR   ø€ Ü‰Ñ˜Ô Ü,¨VÓ4ˆŒÜ 3°F×4FÑ4FÈF×LaÑLaÔ bˆÔÜ2°6×3EÑ3EÈ6×K`ÑK`ÔaˆÕrG   )rH   rI   rJ   r)   rZ   ru   rz   r{   s   @rD   r2  r2  î  s   ø„ ðbÐ5Ð8LÑL÷ bñ brG   r2  c                   ól   — e Zd ZU eed<   dZdZdZg d¢ZdZ	 e
«       Z ej                  «       d„ «       Zy)	ÚVideoPrismPreTrainedModelr®   Úmodelr³   )ÚvideoÚtext)rÆ   ré   r2  r÷   Ú'VideoPrismMultiheadAttentionPoolingHeadFc                 ó®  — t        j                  | |«       t        |t        j                  t        j
                  f«      r t        j                  |j                  «       y t        |t        «      r t        j                  |j                  «       y t        |t        «      r t        j                  |j                  «       y t        |t        «      r?t        j                  |j                  «       t        j                  |j                  «       y t        |t         «      r t        j                  |j                  «       y t        |t"        «      rçt%        |j&                  j(                  |j&                  j*                  «      j-                  |j.                  j0                  |j.                  j2                  ¬«      }t        j4                  |j.                  |«       t        j4                  |j6                  t9        j:                  |j6                  j<                  d   «      j?                  d«      «       y t        |t@        «      r�t        jB                  |jD                  jF                  j                  |j&                  j*                  dz  ¬«       t        jB                  |jD                  jH                  |j&                  j*                  dz  ¬«       y y )N©Údevicer	  rÐ   rû   g      à¿)Ústd)%r   Ú_init_weightsÚ
isinstancerÊ   ÚLinearÚConv3dÚinitÚlecun_normal_r.  rÆ   rÍ   ré   r>  Úzeros_Úper_dim_scaleÚpooling_attention_queryr)  r÷   r   r®   r   rÁ   r  rù   rA  r	  Úcopy_rú   r—   r  r·   r  ÚVideoPrismTextModelÚnormal_rÎ   rþ   r  )rB   Úmodulerù   s      rD   rC  z'VideoPrismPreTrainedModel._init_weights  sØ  € ä×%Ñ% d¨FÔ3Ü�fœrŸy™y¬"¯)©)Ð4Ô5Ü×Ñ˜vŸ}™}Õ-ä˜Ô ;Ô<Ü×Ñ˜v×9Ñ9Õ:ä˜Ô <Ô=Ü×Ñ˜v×9Ñ9Õ:ä˜Ô GÔHÜ�K‰K˜×,Ñ,Ô-Ü×Ñ˜v×=Ñ=Õ>ä˜Ô 3Ô4Ü�K‰K˜Ÿ™Õ&ä˜Ô 8Ô9Ü!<Ø—‘×5Ñ5°v·}±}×7PÑ7Pó"ç‰b˜×1Ñ1×8Ñ8À×@YÑ@Y×@_Ñ@_ˆbÓ`ð ô �J‰J�v×0Ñ0Ð2DÔEÜ�J‰J�v×*Ñ*¬E¯L©L¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÕhä˜Ô 3Ô4Ü�L‰L˜×*Ñ*×:Ñ:×AÑAÀvÇ}Á}×G`ÑG`ÐbfÑGfÕgÜ�L‰L˜×*Ñ*×2Ñ2¸¿¹×8QÑ8QÐSWÑ8WÖXð 5rG   N)rH   rI   rJ   rh   rQ   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚ_no_split_modulesÚ_supports_sdpar@   Ú_input_embed_layerr—   Úno_gradrC  rX   rG   rD   r:  r:  ö  sM   … àÓØÐØ+€OØ(ÐòÐð €NÙ'Ó)Ðà€U‡]�]ƒ_ñYó ñYrG   r:  z¬
    The bare VideoPrism vision encoder outputting raw hidden-states without any specific head on top. This model is the backbone encoder used in VideoPrismVideoModel.
    c                   óÚ   ‡ — e Zd ZU eed<   dZdZdefˆ fd„Zdej                  fd„Z
dej                  fd„Zeee	 	 dd
ej                   d	z  ded	z  dee   defd„«       «       «       Zˆ xZS )ÚVideoPrismVisionModelr®   ©r<  r;  c                 ó@  •— t         ‰| �  |«       t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        |«      | _        t        |«      | _
        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        t        j                  t        |j"                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        | j'                  «        y c c}w c c}w r4  )rt   ru   r)  rÁ   r6  Ú
layernorm1Ú
layernorm2rÆ   Úspatial_embeddingsré   Útemporal_embeddingsrÊ   Ú
ModuleListÚranger4   r2  Úspatial_layersr6   Útemporal_layersÚ	post_init©rB   r®   ró   rx   s      €rD   ru   zVideoPrismVisionModel.__init__1  sË   ø€ Ü‰Ñ˜Ô Ü-¨f×.@Ñ.@Àf×F[ÑF[Ô\ˆŒÜ-¨f×.@Ñ.@Àf×F[ÑF[Ô\ˆŒÜ"=¸fÓ"EˆÔÜ#?ÀÓ#GˆÔ Ü Ÿm™mÌeÐTZ×TmÑTmÓNnÖ,oÈ¬_¸VÕ-DÒ,oÓpˆÔÜ!Ÿ}™}ÌuÐU[×UoÑUoÓOpÖ-qÈ!¬o¸fÕ.EÒ-qÓrˆÔØ�‰Õùò -pùÚ-qs   Â#DÃ'Dr¢   c                 ó.   — | j                   j                  S rŽ   ©r]  rã   r«   s    rD   Úget_input_embeddingsz*VideoPrismVisionModel.get_input_embeddings;  s   € Ø×&Ñ&×7Ñ7Ð7rG   Úvaluec                 ó&   — || j                   _        y rŽ   rf  ©rB   rh  s     rD   Úset_input_embeddingsz*VideoPrismVisionModel.set_input_embeddings>  s   € Ø38ˆ×ÑÕ0rG   Nr³   r´   rC   c                 ó,  — |€t        d«      ‚|j                  }| j                  ||«      }|}| j                  D ]  } ||fi |¤Ž}Œ | j	                  |«      }| j                  |||«      }	|	}
| j                  D ]  } ||
fi |¤Ž}
Œ | j                  |
«      }|j                  \  }}}|j                  |d   d||«      j                  dd«      j                  «       }|j                  \  }}}}|j                  |d   ||z  d«      }t        ||
|¬«      S )Nz'You have to specify pixel_values_videosr   rÐ   r0   r   )Úlast_hidden_stater•   r–   )r¸   r·   r]  ra  r[  r^  rb  r\  rÛ   r¹   r  r”   )rB   r³   r´   rC   rñ   Úspatial_embedsÚspatial_hidden_statesÚspatial_layerrô   Útemporal_embedsÚtemporal_hidden_statesÚtemporal_layerró   r/   rÝ   r°   s                   rD   rÂ   zVideoPrismVisionModel.forwardA  sM  € ð Ð&ÜÐFÓGÐGà)×/Ñ/ˆð ×0Ñ0Ð1DÐF^Ó_ˆØ .ÐØ!×0Ñ0ò 	SˆMÙ$1Ð2GÑ$RÈ6Ñ$RÑ!ð	Sà—?‘?Ð#8Ó9ˆð ×2Ñ2°8¸[ÐJbÓcˆØ!0ÐØ"×2Ñ2ò 	VˆNÙ%3Ð4JÑ%UÈfÑ%UÑ"ð	Và—?‘?Ð#9Ó:ˆð &Ÿ^™^Ñˆˆ:�sØ—=‘= ¨Q¡°°ZÀÓE×OÑOÐPQÐSTÓU×`Ñ`ÓbˆØ*2¯.©.Ñ'ˆˆ:�{ CØ—=‘= ¨Q¡°¸kÑ1IÈ2ÓNˆä:Ø&Ø'=Ø&;ô
ð 	
rG   ©NF)rH   rI   rJ   r)   rQ   rR  rP  ru   rÊ   ÚModulerg  rk  r   r   r   r—   r˜   rT   r   r   r”   rÂ   rz   r{   s   @rD   rX  rX  '  s«   ø… ð #Ó"Ø!ÐØÐðÐ5õ ð8 b§i¡ió 8ð9¨"¯)©)ó 9ð  ØØð 9=Ø05ñ#
à"×.Ñ.°Ñ5ð#
ð #'¨¡+ð#
ð Ð+Ñ,ð	#
ð
 
5ò#
ó ó ó  ô#
rG   rX  c                   ó¦   ‡ — e Zd Zdefˆ fd„Z	 d	dej                  dej                  dz  dee	   de
ej                  ej                  f   fd„Zˆ xZS )
r>  r®   c                 óˆ  •— t         ‰| �  |«       | `|j                  |j                  z  | _        d| _        t        j                  t        j                  | j                  «      «      | _
        t        | j                  dz  z  | _        t        j                  t        j                  dd|j                  «      «      | _        y )Nr  rÑ   r0   )rt   ru   r  Úintermediate_sizer  r  rÊ   rË   r—   rÌ   rJ  Ú_R_SOFTPLUS_0r  rÁ   rK  r²   s     €rD   ru   z0VideoPrismMultiheadAttentionPoolingHead.__init__k  s�   ø€ Ü‰Ñ˜Ô ØÐ$Ø×0Ñ0°F×4NÑ4NÑNˆŒØ$'ˆÔ!ÜŸ\™\¬%¯+©+°d·m±mÓ*DÓEˆÔÜ$¨¯©°sÑ(:Ñ;ˆŒÜ')§|¡|´E·K±KÀÀ1Àf×FXÑFXÓ4YÓ'ZˆÕ$rG   NrÀ   r  rC   r¢   c                 óŒ  — |j                   d d }g |¢d‘| j                  ‘­}| j                  j                  |d   dd«      } | j	                  |«      j
                  g |j                   d d ¢d‘| j                  ‘­Ž j                  dd«      }|| j                  z  t        j                  j                  | j                  «      z  }| j                  |«      j                  |«      j                  dd«      }	| j                  |«      j                  |«      j                  dd«      }
t        j                  | j                   j"                  t$        «      } || ||	|
|fd| j&                  sdn| j(                  d dœ|¤Ž\  }} |j*                  g |j                   d d ¢d‘­Ž j-                  «       }| j/                  |«      }||fS )NrÐ   r   r0   r   r  r_   )r  rä   r  )r·   r  rK  r  r  rÛ   r¹   r  rÊ   rÙ   ÚsoftplusrJ  r  r  r   r  r®   r  r   r  re   r½   r  r   )rB   rÀ   r  rC   rñ   r!  ÚqueryÚquery_layerr"  r#  r$  r%  r&  r'  s                 rD   rÂ   z/VideoPrismMultiheadAttentionPoolingHead.forwardt  sº  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà×,Ñ,×3Ñ3°KÀ±NÀBÈÓKˆØ-�d—k‘k %Ó(×-Ñ-ÐS¨u¯{©{¸3¸BÐ/?ÐSÀÐSÀTÇ]Á]ÒS×]Ñ]Ð^_ÐabÓcˆØ" T§\¡\Ñ1´B·M±M×4JÑ4JÈ4×K]ÑK]Ó4^Ñ^ˆà—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð Ø#Ÿ}š}‘C°$×2HÑ2HØñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð@¨5¯;©;°s¸Ð+;Ð@¸RÒ@×KÑKÓMˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(rG   rŽ   )rH   rI   rJ   r)   ru   r—   r˜   r  r   r   rP   rÂ   rz   r{   s   @rD   r>  r>  j  so   ø„ ð[Ð5õ [ð 37ñ")à×(Ñ(ð")ð ×(Ñ(¨4Ñ/ð")ð Ð+Ñ,ð	")ð
 
ˆu× Ñ  %×"3Ñ"3Ð3Ñ	4÷")rG   r>  z•
    The bare VideoPrism text encoder outputting last hidden states without any specific head on top. This model is used in VideoPrismClipModel.
    c                   óþ   ‡ — e Zd ZU eed<   dZdZdZddgZdZ	defˆ fd„Z
eee	 	 	 	 ddej                  d	z  d
ej                   d	z  dej                   d	z  dej                   d	z  dee   defd„«       «       «       Zˆ xZS )rM  r®   )r=  r;  r  r÷   r2  rþ   c                 óB  •— t         ‰| �  |«       t        |«      | _        t	        j
                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _	        t        |j                  |j                  ¬«      | _        | j                  «        y c c}w r4  )rt   ru   r÷   rÎ   rÊ   r_  r`  rU   r2  Úlayersr)  rÁ   r6  Ú	layernormrc  rd  s      €rD   ru   zVideoPrismTextModel.__init__¦  sq   ø€ Ü‰Ñ˜Ô Ü2°6Ó:ˆŒÜ—m‘mÄeÈF×LdÑLdÓFeÖ$fÀ¤_°VÕ%<Ò$fÓgˆŒÜ,¨V×-?Ñ-?ÀV×EZÑEZÔ[ˆŒØ�‰Õùò %gs   ÁBNr  r  rú   rC   r¢   c                 óø  — |d u |d uz  rt        d«      ‚| j                  |||¬«      }|�lt        j                  |j                  d   d|j
                  |j                  ¬«      }t        j                  ||fd¬«      }t        | j                  ||d ¬«      }| j                  D ]  } |||fi |¤Ž}Œ | j                  |«      }|d d …df   }	| j                  j                  rt        |	d¬«      }	t        ||	¬	«      S )
Nz:You must specify exactly one of input_ids or inputs_embeds)r  rú   r  r   r0   r@  r
  )r®   r  r  Úpast_key_valuesrÐ   )rm  Úpooler_output)r¸   rÎ   r—   Úonesr·   rA  r	  r  r	   r®   r€  r�  r<   r   r   )
rB   r  r  r  rú   rC   rÀ   Úcls_paddingÚlayerÚtext_embeddingss
             rD   rÂ   zVideoPrismTextModel.forward­  s  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàŸ™°)È,Ðfs˜ÓtˆàÐ%ÜŸ*™*Ø×#Ñ# AÑ&¨°.×2GÑ2GÈ~×OcÑOcôˆKô #ŸY™Y¨¸Ð'DÈ!ÔLˆNÜ/Ø—{‘{Ø+Ø-Ø $ô	ˆNð —[‘[ò 	KˆEÙ! -°ÑJÀ6ÑJ‰Mð	KàŸ™ }Ó5ˆà'ª¨2¨Ñ.ˆØ�;‰;×#Ò#Ü$ _¸"Ô=ˆOä)¸MÐYhÔiÐirG   )NNNN)rH   rI   rJ   rZ   rQ   rR  rP  rQ  rS  rU  ru   r   r   r   r—   r  rÄ   r   r   r   rÂ   rz   r{   s   @rD   rM  rM  ™  sÛ   ø… ð !Ó Ø ÐØÐØ!€OØ3Ð5FÐGÐØ*ÐðÐ3õ ð  ØØð .2Ø.2Ø-1Ø,0ñ!jà×#Ñ# dÑ*ð!jð Ÿ™ tÑ+ð!jð —|‘| dÑ*ð	!jð
 —l‘l TÑ)ð!jð Ð+Ñ,ð!jð 
$ò!jó ó ó  ô!jrG   rM  z¹
    VideoPrism video model consisting of the vision encoder backbone with auxiliary encoder layers and an attention pooling head on top. This model is used in VideoPrismClipModel.
    c                   óÀ   ‡ — e Zd ZU eed<   defˆ fd„Zdej                  fd„Zdej                  fd„Z	e
e	 ddej                  ded	z  d
ee   defd„«       «       Zˆ xZS )ÚVideoPrismVideoModelr®   c                 óv  •— t         ‰| �  |«       t        j                  |«      | _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _
        t        |«      | _        t        |j                  |j                  ¬«      | _        | j#                  «        y c c}w r4  )rt   ru   rX  Ú_from_configÚvision_modelrÊ   r_  r`  r;   r2  Úauxiliary_layersr>  Úheadr)  rÁ   r6  Úhead_layernormrc  rd  s      €rD   ru   zVideoPrismVideoModel.__init__Ü  sˆ   ø€ Ü‰Ñ˜Ô Ü1×>Ñ>¸vÓFˆÔÜ "§¡ÔPUÐV\×VqÑVqÓPrÖ.sÈ1¬¸vÕ/FÒ.sÓ tˆÔÜ;¸FÓCˆŒ	Ü1°&×2DÑ2DÈ&×J_ÑJ_Ô`ˆÔØ�‰Õùò /ts   ÁB6r¢   c                 ó6   — | j                   j                  «       S rŽ   ©r�  rg  r«   s    rD   rg  z)VideoPrismVideoModel.get_input_embeddingsä  ó   € Ø× Ñ ×5Ñ5Ó7Ð7rG   rh  c                 ó:   — | j                   j                  |«       y rŽ   ©r�  rk  rj  s     rD   rk  z)VideoPrismVideoModel.set_input_embeddingsç  ó   € Ø×Ñ×.Ñ.¨uÕ5rG   r³   r´   NrC   c                 óR  —  | j                   d||dœ|¤Ž}|j                  }| j                  D ]  } ||fi |¤Ž}Œ  | j                  |fi |¤Ž}| j	                  |d   «      }| j
                  j                  rt        |d¬«      }t        |||j                  |j                  ¬«      S )N©r³   r´   r   rÐ   r
  )rm  r„  rÀ   Ú
attentionsrX   )r�  rm  rŽ  r�  r�  r®   r<   r   r   rÀ   r™  )	rB   r³   r´   rC   Úvision_model_outputsÚauxiliary_hidden_statesr‡  Úhead_outputÚvideo_embeddingss	            rD   rÂ   zVideoPrismVideoModel.forwardê  sÍ   € ð  1˜t×0Ñ0ð  
Ø 3ÐNfñ 
Øjpñ 
Ðð #7×"HÑ"HÐØ×*Ñ*ò 	OˆEÙ&+Ð,CÑ&NÀvÑ&NÑ#ð	Oð  �d—i‘iÐ 7ÑB¸6ÑBˆØ×.Ñ.¨{¸1©~Ó>ÐØ�;‰;×#Ò#Ü%Ð&6¸BÔ?Ðä)Ø5Ø*Ø.×<Ñ<Ø+×6Ñ6ô	
ð 	
rG   rÃ   )rH   rI   rJ   r)   rQ   ru   rÊ   ru  rg  rk  r   r   r—   r˜   rT   r   r   r   rÂ   rz   r{   s   @rD   rŠ  rŠ  Ô  s�   ø… ð #Ó"ðÐ5õ ð8 b§i¡ió 8ð6¨"¯)©)ó 6ð Øð 16ñ
à"×.Ñ.ð
ð #'¨¡+ð
ð Ð+Ñ,ð	
ð
 
$ò
ó ó ô
rG   rŠ  zÆ
    VideoPrism model for video-text contrastive learning. This model consists of a VideoPrismVideoModel and a VideoPrismTextModel, and computes similarity scores between video and text inputs.
    c                   óÄ  ‡ — e Zd Zdefˆ fd„Zdej                  fd„Zdej                  fd„Ze	e
	 ddej                  d	ej                  dz  d
ee   deez  fd„«       «       Ze	e
	 ddej$                  dedz  d
ee   deez  fd„«       «       Ze	e
	 	 	 	 ddej$                  dej                  d	ej                  dz  dedz  dedz  dedz  d
ee   defd„«       «       Zˆ xZS )ÚVideoPrismClipModelr®   c                 óÔ   •— t         ‰| �  |«       t        j                  |j                  «      | _        t        j                  |j                  «      | _        | j                  «        y rŽ   )
rt   ru   rŠ  rŒ  r+   Úvideo_modelrM  Útext_configÚ
text_modelrc  r²   s     €rD   ru   zVideoPrismClipModel.__init__  sK   ø€ Ü‰Ñ˜Ô Ü/×<Ñ<¸V×=QÑ=QÓRˆÔÜ-×:Ñ:¸6×;MÑ;MÓNˆŒØ�‰ÕrG   r¢   c                 ó6   — | j                   j                  «       S rŽ   )r£  rg  r«   s    rD   rg  z(VideoPrismClipModel.get_input_embeddings  s   € Ø�‰×3Ñ3Ó5Ð5rG   rh  c                 ó:   — | j                   j                  |«       y rŽ   )r£  rk  rj  s     rD   rk  z(VideoPrismClipModel.set_input_embeddings  s   € Ø�‰×,Ñ,¨UÕ3rG   Nr  r  rC   c                 ó,   —  | j                   d||dœ|¤ŽS )a  
        Examples:

        ```python
        >>> from transformers import AutoTokenizer, VideoPrismClipModel

        >>> model = VideoPrismClipModel.from_pretrained("google/videoprism-lvt-base-f16r288")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/videoprism-lvt-base-f16r288")

        >>> inputs = tokenizer(["a video of a cat.", "a video of a dog."], padding="max_length", return_tensors="pt")
        >>> with torch.no_grad():
        ...     text_features = model.get_text_features(**inputs)
        ```©r  r  rX   )r£  )rB   r  r  rC   s       rD   Úget_text_featuresz%VideoPrismClipModel.get_text_features  s    € ð* ˆt�‰Ð\¨À>Ñ\ÐU[Ñ\Ð\rG   r³   r´   c                 ó,   —  | j                   d||dœ|¤ŽS )a÷  
        Examples:

        ```python
        >>> from transformers import VideoPrismProcessor, VideoPrismClipModel

        >>> model = VideoPrismClipModel.from_pretrained("google/videoprism-lvt-base-f16r288")
        >>> processor = VideoPrismProcessor.from_pretrained("google/videoprism-lvt-base-f16r288")

        >>> inputs = processor(videos="path/to/video.mp4", return_tensors="pt")
        >>> with torch.no_grad():
        ...     video_features = model.get_video_features(**inputs)
        ```r˜  rX   )r¡  )rB   r³   r´   rC   s       rD   Úget_video_featuresz&VideoPrismClipModel.get_video_features/  s.   € ð*  ˆt×Ñð 
Ø 3Ø%=ñ
ð ñ
ð 	
rG   ÚtemperatureÚreturn_lossc           	      ór  —  | j                   d||dœ|¤Ž} | j                  d||dœ|¤Ž}	|j                  }
|	j                  }|
j                  d   }|j                  d   }|
j	                  d|«      }|j	                  d|«      }t        j                  ||j                  «      }|�||z  }t        j                  |«      }|j                  }|t        j                  |dd¬«      z  }|t        j                  |dd¬«      z  }d}|r t        j                  |j                  d«      |j                  ¬«      }t        j                  |«       d	|z  z   }t
        j                  j                  j!                  ||z  «      }t        j                  |d¬
«       }|j#                  «       }t%        ||||||	|¬«      S )a  
        temperature (`float`, *optional*):
            A temperature scalar to scale the similarity scores. If not provided, no scaling is applied.
        return_loss (`bool`, *optional*):
            Whether or not to return the contrastive loss.
        r˜  r§  rÐ   Nr   T)rÝ   Úkeepdims)rA  r   r
  )r›   rœ   r�   rž   rŸ   r    r¡   rX   )rª  r¨  r„  r·   r½   r—   ÚmatmulÚTÚexpÚsumÚeyer„   rA  Ú	ones_likerÊ   rÙ   Ú
logsigmoidÚmeanrš   )rB   r³   r  r  r´   r«  r¬  rC   Úvideo_model_outputsÚtext_model_outputsr�  rˆ  Úvideo_emb_dimÚtext_emb_dimr�   rž   Úsimilarity_matrixr›   rœ   r¡   r³  Úm1_diag1ÚloglikÚnlls                           rD   rÂ   zVideoPrismClipModel.forwardJ  s½  € ð& 6˜d×5Ñ5ð 
Ø 3ÐNfñ
Øjpñ
Ðð 4˜T×3Ñ3Ðq¸iÐXfÑqÐjpÑqÐà.×<Ñ<ÐØ,×:Ñ:ˆØ(×.Ñ.¨rÑ2ˆØ&×,Ñ,¨RÑ0ˆà'×/Ñ/°°MÓBˆØ%×-Ñ-¨b°,Ó?ˆÜ!ŸL™L¨°{·}±}ÓEÐàÐ"Ø Ñ,Ðä Ÿ9™9Ð%6Ó7ÐØ*×,Ñ,ˆØ+¬e¯i©iÐ8HÈaÐZ^Ô._Ñ_ÐØ)¬E¯I©I°oÈ1ÐW[Ô,\Ñ\ˆð ˆÙä—)‘)˜O×0Ñ0°Ó3¸O×<RÑ<RÔSˆCÜŸ™¨Ó8Ð8¸1¸s¹7ÑBˆHÜ—X‘X×(Ñ(×3Ñ3°H¸Ñ4NÓOˆFÜ—9‘9˜V¨Ô,Ð,ˆCØ—8‘8“:ˆDä#Ø-Ø+Ø%Ø#Ø2Ø0Øô
ð 	
rG   rŽ   rÃ   )NFNN)rH   rI   rJ   rh   ru   rÊ   ru  rg  rk  r   r   r—   rÄ   r   r   rP   r   r¨  r˜   rT   rª  rS   rš   rÂ   rz   r{   s   @rD   rŸ  rŸ    s�  ø„ ðÐ/õ ð6 b§i¡ió 6ð4¨"¯)©)ó 4ð Øð /3ñ]à—<‘<ð]ð Ÿ™ tÑ+ð]ð Ð+Ñ,ð	]ð
 
Ð+Ñ	+ò]ó ó ð]ð* Øð 16ñ
à"×.Ñ.ð
ð #'¨¡+ð
ð Ð+Ñ,ð	
ð
 
Ð+Ñ	+ò
ó ó ð
ð2 Øð
 /3Ø05Ø$(Ø#'ñ9
à"×.Ñ.ð9
ð —<‘<ð9
ð Ÿ™ tÑ+ð	9
ð
 #'¨¡+ð9
ð ˜T‘\ð9
ð ˜D‘[ð9
ð Ð+Ñ,ð9
ð 
ò9
ó ó ô9
rG   rŸ  z
    VideoPrism Model transformer with a video classification head on top (a linear layer on top of the attention pooler).
    c                   óè   ‡ — e Zd ZU eed<   dZdZdefˆ fd„Zdej                  fd„Z
dej                  fd„Zee	 	 dd
ej                  dej                   d	z  ded	z  dee   def
d„«       «       Zˆ xZS )Ú VideoPrismForVideoClassificationr®   rY  r;  c                 óB  •— t         ‰| �  |«       t        j                  |«      | _        t        |«      | _        t        |j                  |j                  ¬«      | _
        t        j                  |j                  |j                  «      | _        | j                  «        y r4  )rt   ru   rX  rŒ  r�  r>  r�  r)  rÁ   r6  r�  rÊ   rE  Ú
num_labelsÚ
classifierrc  r²   s     €rD   ru   z)VideoPrismForVideoClassification.__init__’  sr   ø€ Ü‰Ñ˜Ô Ü1×>Ñ>¸vÓFˆÔÜ;¸FÓCˆŒ	Ü1°&×2DÑ2DÈ&×J_ÑJ_Ô`ˆÔÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒØ�‰ÕrG   r¢   c                 ó6   — | j                   j                  «       S rŽ   r’  r«   s    rD   rg  z5VideoPrismForVideoClassification.get_input_embeddingsš  r“  rG   rh  c                 ó:   — | j                   j                  |«       y rŽ   r•  rj  s     rD   rk  z5VideoPrismForVideoClassification.set_input_embeddings�  r–  rG   Nr³   Úlabelsr´   rC   c                 ó<  —  | j                   d||dœ|¤Ž}|j                  }| j                   | j                  |fi |¤Žd   «      }| j	                  |«      }d }	|� | j
                  ||| j                  fi |¤Ž}	t        |	||j                  |j                  ¬«      S )Nr˜  r   )r¡   ÚlogitsrÀ   r™  rX   )
r�  rm  r�  r�  rÃ  Úloss_functionr®   r   rÀ   r™  )
rB   r³   rÆ  r´   rC   rš  Úsequence_outputÚpooled_outputrÈ  r¡   s
             rD   rÂ   z(VideoPrismForVideoClassification.forward   sº   € ð  1˜t×0Ñ0ð  
Ø 3ÐNfñ 
Øjpñ 
Ðð /×@Ñ@ˆØ×+Ñ+¨I¨D¯I©I°oÑ,PÈÑ,PÐQRÑ,SÓTˆØ—‘ Ó/ˆØˆØÐØ%�4×%Ñ% f¨f°d·k±kÑLÀVÑLˆDä$ØØØ.×<Ñ<Ø+×6Ñ6ô	
ð 	
rG   rt  )rH   rI   rJ   r)   rQ   rR  rP  ru   rÊ   ru  rg  rk  r   r   r—   r˜   r  rT   r   r   r   rÂ   rz   r{   s   @rD   rÀ  rÀ  ˆ  s³   ø… ð #Ó"Ø!ÐØÐðÐ5õ ð8 b§i¡ió 8ð6¨"¯)©)ó 6ð Øð +/Ø05ñ	
à"×.Ñ.ð
ð × Ñ  4Ñ'ð
ð #'¨¡+ð	
ð
 Ð+Ñ,ð
ð 
ò
ó ó ô
rG   rÀ  )r)   rZ   rh   rX  r:  rŠ  rM  rŸ  rÀ  rk   rŒ   )YÚcollections.abcr   Údataclassesr   Útypingr   r—   Útorch.nnrÊ   Útorch.nn.functionalrÙ   r+  Úhuggingface_hub.dataclassesr   Ú r   rG  Úmasking_utilsr	   Úmodeling_outputsr
   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   r   r   Úutilsr   r   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úcodegen.modeling_codegenr   Úgemma2.modeling_gemma2r   Úqwen3_next.modeling_qwen3_nextr   Úsiglip.configuration_siglipr   r   Út5.tokenization_t5r    Úvivit.configuration_vivitr!   Úvivit.modeling_vivitr"   r#   r$   r%   r&   Ú
get_loggerrH   Úloggerry  r)   rZ   rh   rk   r}   rŒ   r”   rš   r­   rÆ   ré   ru  r÷   r  Ú	LayerNormr)  r2  r:  rX  r>  rM  rŠ  rŸ  rÀ  Ú__all__rX   rG   rD   ú<module>rå     sæ  ðõ  %Ý !Ý ã Ý ß Ð Ý .å &Ý /ß bÑ bß Fß HÑ Hß j× jÝ 7Ý 5Ý BÝ <Ý 3ß HÝ ,Ý 3÷õ ð 
ˆ×	Ñ	˜HÓ	%€à€ñ Ð;Ô<Øô".˜[ó ".ó ó =ð".ñJ Ð?Ô@Øô.Ð+ó .ó ó Að.ñ2 Ð?Ô@Øô*�|ó *ó ó Að*ô*+˜+ô +ôDÐ 0¸õ ð ô5˜.ó 5ó ð5ñ ÐiÔjØ
ô?°/ó ?ó ó kð?ñ ØBôð ô 
˜;ó  
ó óð 
ôFÐ"8ô ôB= /ô =ô@8 ?ô 8ôv ˜rŸy™yô  ôF&)˜.ô &)ôRj˜"Ÿ,™,ô jôb�jô bð ô-YÐ 4ó -Yó ð-Yñ` ðôô
;
Ð5ó ;
óð
;
ô|,)¨nô ,)ñ^ ðôô
3jÐ3ó 3jóð
3jñl ðôô
*
Ð4ó *
óð
*
ñZ ðôô
z
Ð3ó z
óð
z
ñz ðôô
+
Ð'@ó +
óð
+
ò\�rG   