Ë
    ÜÓ^jª£  ã                   óR  — d dl mZmZ d dlmZ d dlmZ d dlZd dlm	Z	 d dl
m	c mZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZ ddlmZmZ ddlmZ ddlm Z m!Z!m"Z"m#Z#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z,  e"d¬«      e G d„ de«      «       «       Z- e"d¬«      e G d„ de «      «       «       Z. G d„ de	j^                  «      Z0 G d„ de	j^                  «      Z1 G d„ de	j^                  «      Z2de3d e3d!ejh                  fd"„Z5 G d#„ d$e	j^                  «      Z6	 	 	 dQd%e	j^                  d&ejh                  d'ejh                  d(ejh                  d)ejh                  dz  d*e7e3z  d+e7dz  d,e7dz  d!e8ejh                  ejh                  f   fd-„Z9d.ejh                  d/e3d!ejh                  fd0„Z: G d1„ d2e	j^                  «      Z; G d3„ d4e	jx                  «      Z= G d5„ d6e	j^                  «      Z> G d7„ d8e«      Z?e" G d9„ d:e«      «       Z@ e"d;¬«       G d<„ d=e@«      «       ZAd>ZB G d?„ d@e	j^                  «      ZCdRdAejˆ                  d e3dBe7fdC„ZE e"dD¬«       G dE„ dFe@«      «       ZF e"dG¬«       G dH„ dIe@«      «       ZG e"dJ¬«       G dK„ dLe@«      «       ZH e"dM¬«       G dN„ dOe@«      «       ZIg dP¢ZJy)Sé    )ÚCallableÚIterable)Ú	dataclass)ÚAnyNé   )Úinitialization)ÚACT2FN)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚImageClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚ	torch_int)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚVideoPrismConfigÚVideoPrismTextConfigÚVideoPrismVisionConfigzFBase class for model outputs that include spatial and temporal states.)Úcustom_introc                   ób   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   y)Ú+BaseModelOutputWithSpatialAndTemporalStatesa™  
    last_temporal_hidden_state (`torch.FloatTensor`, *optional*):
        The last hidden state of the temporal encoder, typically of shape
        `(batch_size * num_patches, num_frames, hidden_size)`.
    last_spatial_hidden_state (`torch.FloatTensor`, *optional*):
        The last hidden state of the spatial encoder, typically of shape
        `(batch_size * num_frames, num_patches, hidden_size)`.
    NÚlast_temporal_hidden_stateÚlast_spatial_hidden_state)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r    ÚtorchÚFloatTensorÚ__annotations__r!   © ó    ú}/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/videoprism/modeling_videoprism.pyr   r   +   s6   … ñð <@Ð × 1Ñ 1°DÑ 8Ó?Ø:>Ð˜u×0Ñ0°4Ñ7Ô>r*   r   z+Base class for VideoPrismClipModel outputs.c                   ó  — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
ej                  dz  ed<   dZeed<   dZeed<   dZej                  dz  ed	<   d
ee   fd„Zy)ÚVideoPrismClipOutputa¼  
    logits_per_video (`torch.FloatTensor` of shape `(video_batch_size, text_batch_size)`):
        The scaled dot product scores between `video_embeds` and `text_embeds`. This represents the video-text
        similarity scores.
    logits_per_text (`torch.FloatTensor` of shape `(text_batch_size, video_batch_size)`):
        The scaled dot product scores between `text_embeds` and `video_embeds`. This represents the text-video
        similarity scores.
    video_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)`):
        The video embeddings obtained by applying the projection layer to the pooled output of [`VideoPrismVideoModel`].
    text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)`):
        The text embeddings obtained by applying the projection layer to the pooled output of [`VideoPrismTextModel`].
    video_model_output (`BaseModelOutputWithPooling`):
        The output of [`VideoPrismVideoModel`].
    text_model_output (`BaseModelOutputWithPooling`):
        The output of the [`VideoPrismTextModel`].
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
        Contrastive loss for video-text similarity.
    NÚlogits_per_videoÚlogits_per_textÚvideo_embedsÚtext_embedsÚvideo_model_outputÚtext_model_outputÚlossÚreturnc                 óH   ‡ — t        ˆ fd„‰ j                  «       D «       «      S )Nc              3   ód   •K  — | ]'  }|d vr‰|   nt        ‰|«      j                  «       –— Œ) y­w))r3   r2   N)ÚgetattrÚto_tuple)Ú.0ÚkÚselfs     €r+   ú	<genexpr>z0VideoPrismClipOutput.to_tuple.<locals>.<genexpr>\   s=   øè ø€ ò 
àð Ð KÑKˆD�ŠGÔQXÐY]Ð_`ÓQa×QjÑQjÓQlÓlñ
ùs   ƒ-0)ÚtupleÚkeys©r<   s   `r+   r9   zVideoPrismClipOutput.to_tuple[   s#   ø€ Üó 
à—Y‘Y“[ô
ó 
ð 	
r*   )r"   r#   r$   r%   r.   r&   r'   r(   r/   r0   r1   r2   r   r3   r4   r>   r   r9   r)   r*   r+   r-   r-   ;   s›   … ñ
ð& 26Ð�e×'Ñ'¨$Ñ.Ó5Ø04€O�U×&Ñ&¨Ñ-Ó4Ø-1€L�%×#Ñ# dÑ*Ó1Ø,0€K�×"Ñ" TÑ)Ó0Ø59ÐÐ2Ó9Ø48ÐÐ1Ó8Ø%)€Dˆ%×
Ñ
˜dÑ
"Ó)ð
˜% ™*ô 
r*   r-   c                   óf   ‡ — e Zd ZdZdefˆ fd„Zddej                  dedej                  fd„Z	ˆ xZ
S )	ÚVideoPrismTubeletEmbeddingsañ  
    VideoPrism Tubelet Embeddings.

    The authors of Videoprism use the Factorized Encoder architecture, i.e. "Model 2", introduced in the VIVIT paper (https://huggingface.co/papers/2103.15691).
    This differs from Vivit by using a convolution of `tubelet_size=(1, 18, 18)`, which is essentially a 2d convolution in the spatial dimension.
    The temporal dimension is also merged with the `batch_size` in order to make sure the image embeddings have no temporal component, unlike Vivit.
    Úconfigc                 ó–  •— t         ‰| �  «        |j                  }|j                  }t	        |t
        «      r|n||f}|| _        t        j                  |j                  |j                  ||¬«      | _
        | j                  d   |d   z  | j                  d   |d   z  g| _        | j                  d   | j                  d   z  | _        y )N)Úkernel_sizeÚstrider   r   é   )ÚsuperÚ__init__Útubelet_sizeÚ
image_sizeÚ
isinstancer   ÚnnÚConv3dÚnum_channelsÚhidden_sizeÚ
projectionÚpos_emb_shapeÚnum_patches)r<   rC   rJ   rK   Ú	__class__s       €r+   rI   z$VideoPrismTubeletEmbeddings.__init__k   sÁ   ø€ Ü‰ÑÔØ×*Ñ*ˆØ×&Ñ&ˆ
Ü#-¨j¼(Ô#C‘ZÈ*ÐV`ÐIaˆ
Ø$ˆŒÜŸ)™)Ø×Ñ ×!3Ñ!3ÀÐVbô
ˆŒð #Ÿo™o¨aÑ0°LÀ±OÑCÀTÇ_Á_ÐUVÑEWÐ[gÐhiÑ[jÑEjÐkˆÔØ×-Ñ-¨aÑ0°4×3EÑ3EÀaÑ3HÑHˆÕr*   Úpixel_values_videosÚinterpolate_pos_encodingr5   c                 ó´  — |j                   \  }}}}}|sV|| j                  d   k7  s|| j                  d   k7  r2t        d|› d|› d| j                  d   › d| j                  d   › d�	«      ‚|j                  dd«      }| j	                  |«      }|j                  d«      j                  dddd«      }|j                   \  }}}	}
|j                  ||z  |	|
«      }|S )	Nr   r   zImage size (Ú*z) doesn't match model (z[). Set interpolate_pos_encoding=True to automatically resize the model position embeddings.rG   r   )ÚshaperK   Ú
ValueErrorÚ	transposerQ   ÚflattenÚpermuteÚreshape)r<   rU   rV   Ú
batch_sizeÚ
num_framesrO   ÚheightÚwidthÚhidden_statesrS   rP   s              r+   Úforwardz#VideoPrismTubeletEmbeddings.forwardw   s  € Ø>Q×>WÑ>WÑ;ˆ
�J ¨f°eÙ'¨V°t·±ÀqÑ7IÒ-IÈUÐVZ×VeÑVeÐfgÑVhÒMhÜØ˜v˜h a¨ wÐ.EÀdÇoÁoÐVWÑFXÐEYÐYZÐ[_×[jÑ[jÐklÑ[mÐZnð  oJð  Kóð ð 2×;Ñ;¸A¸qÓAÐØŸ™Ð(;Ó<ˆà%×-Ñ-¨aÓ0×8Ñ8¸¸A¸qÀ!ÓDˆà;H×;NÑ;NÑ8ˆ
�J ¨[Ø%×-Ñ-¨j¸:Ñ.EÀ{ÐT_Ó`ˆàÐr*   ©F)r"   r#   r$   r%   r   rI   r&   ÚTensorÚboolrd   Ú__classcell__©rT   s   @r+   rB   rB   b   s=   ø„ ñð
IÐ5õ 
Iñ¨5¯<©<ð ÐSWð Ðdi×dpÑdp÷ r*   rB   c                   ó®   ‡ — e Zd ZdZdefˆ fd„Zdej                  dededej                  fd„Z		 dd	ej                  d
e
dz  dej                  fd„Zˆ xZS )ÚVideoPrismSpatialEmbeddingszY
    Construct the CLS token, position and tubelet patch embeddings for video input.
    rC   c                 óP  •— t         ‰| �  «        t        |«      | _        | j                  j                  }t        j                  t        j                  d||j                  «      «      | _
        t        j                  |j                  «      | _        |j                  dd  | _        y ©Nr   )rH   rI   rB   Úpatch_embeddingsrS   rM   Ú	Parameterr&   ÚzerosrP   Úposition_embeddingsÚDropoutÚhidden_dropout_probÚdropoutrJ   Ú
patch_size)r<   rC   rS   rT   s      €r+   rI   z$VideoPrismSpatialEmbeddings.__init__Ž   sz   ø€ Ü‰ÑÔÜ ;¸FÓ CˆÔØ×+Ñ+×7Ñ7ˆÜ#%§<¡<´·±¸A¸{ÈF×L^ÑL^Ó0_Ó#`ˆÔ Ü—z‘z &×"<Ñ"<Ó=ˆŒà ×-Ñ-¨a¨bÐ1ˆ�r*   Ú
embeddingsra   rb   r5   c                 ó8  — |j                   d   }| j                  j                   d   }t        j                  j	                  «       s||k(  r||k(  r| j                  S |j                   d   }|| j
                  d   z  }|| j
                  d   z  }t        |dz  «      }	| j                  j                  d|	|	|«      }
|
j                  dddd«      }
t        j                  j                  |
||fdd¬	«      }
|
j                  dddd«      j                  dd|«      }
|
S )
á   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   éÿÿÿÿr   ç      à?r   rG   ÚbilinearT©ÚsizeÚmodeÚ	antialias)rY   rq   r&   ÚjitÚ
is_tracingru   r   r^   r]   rM   Ú
functionalÚinterpolateÚview)r<   rv   ra   rb   rS   Únum_positionsÚdimÚnum_row_patchesÚnum_col_patchesÚsqrt_num_positionsÚpatch_pos_embeds              r+   rV   z4VideoPrismSpatialEmbeddings.interpolate_pos_encoding—   s*  € ð !×&Ñ& qÑ)ˆØ×0Ñ0×6Ñ6°qÑ9ˆô �y‰y×#Ñ#Ô%¨+¸Ò*FÈ6ÐUZÊ?Ø×+Ñ+Ð+à×Ñ˜rÑ"ˆà  D§O¡O°AÑ$6Ñ6ˆØ 4§?¡?°1Ñ#5Ñ5ˆä& }°cÑ'9Ó:ÐØ×2Ñ2×:Ñ:¸1Ð>PÐRdÐfiÓjˆØ)×1Ñ1°!°Q¸¸1Ó=ˆô Ÿ-™-×3Ñ3ØØ! ?Ð3ØØð	 4ó 
ˆð *×1Ñ1°!°Q¸¸1Ó=×BÑBÀ1ÀbÈ#ÓNˆØÐr*   rU   rV   Nc                 óÀ   — |j                   \  }}}}}| j                  ||«      }|r|| j                  |||«      z   }n|| j                  z   }| j	                  |«      }|S ©N)rY   rn   rV   rq   rt   )	r<   rU   rV   ÚbatchÚframesÚchannelra   rb   rv   s	            r+   rd   z#VideoPrismSpatialEmbeddings.forward½   ss   € ð
 1D×0IÑ0IÑ-ˆˆv�w ¨Ø×*Ñ*Ð+>Ð@XÓYˆ
ñ $Ø# d×&CÑ&CÀJÐPVÐX]Ó&^Ñ^‰Jà# d×&>Ñ&>Ñ>ˆJà—\‘\ *Ó-ˆ
àÐr*   re   )r"   r#   r$   r%   r   rI   r&   rf   ÚintrV   rg   rd   rh   ri   s   @r+   rk   rk   ‰   su   ø„ ñð2Ð5õ 2ð$°5·<±<ð $Èð $ÐUXð $Ð]b×]iÑ]ió $ðR 16ñà"Ÿ\™\ðð #'¨¡+ðð 
�‰÷	r*   rk   c            	       ó¾   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Z	 ddej                  dej                  d	e
d
z  dej                  fd„Zˆ xZS )ÚVideoPrismTemporalEmbeddingszÍ
    VideoPrism Temporal Embeddings.

    Receives embeddings from spatial encoder, reshapes the hidden state to
    (batch_size * num_patches, num_frames, hidden_size) and adds positional embeddings.
    rC   c                 óð   •— t         ‰| �  «        t        j                  t	        j
                  d|j                  |j                  «      «      | _        t        j                  |j                  «      | _        y rm   )rH   rI   rM   ro   r&   rp   r`   rP   rq   rr   rs   rt   ©r<   rC   rT   s     €r+   rI   z%VideoPrismTemporalEmbeddings.__init__Ø   sO   ø€ Ü‰ÑÔÜ#%§<¡<´·±¸A¸v×?PÑ?PÐRX×RdÑRdÓ0eÓ#fˆÔ Ü—z‘z &×"<Ñ"<Ó=ˆ�r*   rv   r5   c                 ót  — |j                   d   }| j                  j                   d   }t        j                  j	                  «       s||k(  r| j                  S | j                  }|j                   d   }|j                  d«      }t        j                  j                  |||fdd¬«      }|j                  d«      S )rx   r   ry   r{   Tr|   )
rY   rq   r&   r€   r�   Ú	unsqueezerM   r‚   rƒ   Úsqueeze)r<   rv   Útarget_emb_lengthÚsource_emb_lengthÚ
source_embr†   s         r+   rV   z5VideoPrismTemporalEmbeddings.interpolate_pos_encodingÝ   s»   € ð '×,Ñ,¨QÑ/ÐØ ×4Ñ4×:Ñ:¸1Ñ=Ðô �y‰y×#Ñ#Ô%Ð*;Ð?PÒ*PØ×+Ñ+Ð+à×-Ñ-ˆ
Ø×Ñ˜rÑ"ˆØ×)Ñ)¨!Ó,ˆ
Ü—]‘]×.Ñ.ØØ# SÐ)ØØð	 /ó 
ˆ
ð ×!Ñ! !Ó$Ð$r*   rU   Úinput_shaperV   Nc                 ó   — |�|\  }}}}}|j                   \  }	}
}|j                  |
|«      }|j                  dd«      }|j                  ||
z  ||«      }|r|| j	                  |«      z   }n|| j
                  z   }| j                  |«      }|S )NrG   r   )rY   r„   r[   r^   rV   rq   rt   )r<   rU   r›   rV   r�   rŽ   r�   ra   rb   Ú_Úfeaturesr†   rc   rv   s                 r+   rd   z$VideoPrismTemporalEmbeddings.forwardù   s«   € ð Ð"Ø4?Ñ1ˆE�6˜7 F¨EØ.×4Ñ4Ñˆˆ8�SØ+×0Ñ0°¸ÀÈ#ÓNˆØ%×/Ñ/°°1Ó5ˆØ"×*Ñ*¨5°8Ñ+;¸VÀSÓIˆ
ñ $Ø# d×&CÑ&CÀJÓ&OÑO‰Jà# d×&>Ñ&>Ñ>ˆJØ—\‘\ *Ó-ˆ
ØÐr*   re   )r"   r#   r$   r%   r   rI   r&   rf   rV   ÚSizerg   rd   rh   ri   s   @r+   r’   r’   Ð   sq   ø„ ñð>Ð5õ >ð
%°5·<±<ð %ÀEÇLÁLó %ð@ 16ñ	à"Ÿ\™\ðð —Z‘Zðð #'¨¡+ð	ð
 
�‰÷r*   r’   Únum_posr†   r5   c                 óŒ  — ddt        j                  d|dt         j                  ¬«      |z  z  z  }t        j                  dt        j                  | t         j                  ¬«      j	                  «       |«      j	                  «       }t        j
                  t        j                  |«      t        j                  |«      fd¬«      S )	Nç      ð?i'  r   rG   ©Údtypezi , j -> i jr   ©r†   )r&   ÚarangeÚint64ÚeinsumÚfloatÚcatÚsinÚcos)r    r†   Úinv_freqÚsinusoid_inps       r+   Úcreate_sinusoidal_positionsr¯     s…   € Ø�e¤§¡¨Q°°Q¼e¿k¹kÔ JÈSÑ PÑQÑR€HÜ—<‘< ´·±¸WÌEÏKÉKÔ0X×0^Ñ0^Ó0`ÐbjÓk×qÑqÓs€LÜ�9‰9”e—i‘i Ó-¬u¯y©y¸Ó/FÐGÈQÔOÐOr*   c            	       ó¦   ‡ — e Zd Zdefˆ fd„Z	 	 	 d	dej                  dz  dej                  dz  dej                  dz  dej                  fd„Z	ˆ xZ
S )
ÚVideoPrismTextEmbeddingsrC   c                 ó  •— t         ‰| �  «        || _        |j                  }t	        j
                  |j                  |«      | _        | j                  dt        |j                  |j                  «      «       | j                  dt        j                  |j                  «      j                  d«      «       t	        j                  t        j                  dd|j                  «      «      | _        |j                  dz  | _        y )NÚposition_embeddingÚposition_ids©r   ry   r   rz   )rH   rI   rC   rP   rM   Ú	EmbeddingÚ
vocab_sizeÚtoken_embeddingÚregister_bufferr¯   Úmax_position_embeddingsr&   r¦   Úexpandro   rp   Úcls_embÚscaling)r<   rC   Ú	embed_dimrT   s      €r+   rI   z!VideoPrismTextEmbeddings.__init__  sÃ   ø€ Ü‰ÑÔØˆŒØ×&Ñ&ˆ	Ü!Ÿ|™|¨F×,=Ñ,=¸yÓIˆÔØ×ÑØ Ô"=¸f×>\Ñ>\Ð^d×^pÑ^pÓ"qô	
ð 	×Ñ˜^¬U¯\©\¸&×:XÑ:XÓ-Y×-`Ñ-`ÐahÓ-iÔjÜ—|‘|¤E§K¡K°°1°f×6HÑ6HÓ$IÓJˆŒØ×)Ñ)¨3Ñ.ˆ�r*   NÚ	input_idsr´   Úinputs_embedsr5   c                 ó’  — |€| j                  |«      }|€"| j                  d d …d |j                  d   …f   }|| j                  z  }| j                  |   j                  |j                  ¬«      }||z   }| j                  | j                  z  }|j                  |j                  d   dd«      }t        j                  ||fd¬«      }|S )Nr   r£   r   ry   r¥   )r¸   r´   rY   r½   r³   Útor¤   r¼   r»   r&   rª   )r<   r¿   r´   rÀ   rq   rv   r¼   s          r+   rd   z VideoPrismTextEmbeddings.forward"  sÎ   € ð Ð Ø ×0Ñ0°Ó;ˆMàÐØ×,Ñ,ªQÐ0H°-×2EÑ2EÀaÑ2HÐ0HÐ-HÑIˆLà%¨¯©Ñ4ˆØ"×5Ñ5°lÑC×FÑFÈ]×M`ÑM`ÐFÓaÐØ"Ð%8Ñ8ˆ
à—,‘, §¡Ñ-ˆØ—.‘. ×!1Ñ!1°!Ñ!4°b¸"Ó=ˆÜ—Y‘Y 
¨GÐ4¸!Ô<ˆ
ØÐr*   )NNN)r"   r#   r$   r   rI   r&   Ú
LongTensorr'   rf   rd   rh   ri   s   @r+   r±   r±     sk   ø„ ð
/Ð3õ 
/ð .2Ø04Ø26ñ	à×#Ñ# dÑ*ðð ×&Ñ&¨Ñ-ðð ×(Ñ(¨4Ñ/ð	ð
 
�‰÷r*   r±   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskrt   r½   Úsoftcapc                 ó|  — |€| j                   dz  }t        || j                  «      }	t        || j                  «      }
t        j                  ||	j                  dd«      «      |z  }|�||z  }t        j                  |«      }||z  }|�||z   }t        j                  j                  |dt        j                  ¬«      j                  |j                  «      }t        j                  j                  ||| j                  ¬«      }t        j                  ||
«      }|j                  dd«      j                  «       }||fS )Nç      à¿rG   r   ry   )r†   r¤   )ÚpÚtrainingr   )Úhead_dimÚ	repeat_kvÚnum_key_value_groupsr&   Úmatmulr[   ÚtanhrM   r‚   ÚsoftmaxÚfloat32rÂ   r¤   rt   rÍ   Ú
contiguous)rÄ   rÅ   rÆ   rÇ   rÈ   rt   r½   rÉ   ÚkwargsÚ
key_statesÚvalue_statesÚattn_weightsÚattn_outputs                r+   Úeager_attention_forwardrÛ   8  s  € ð €Ø—/‘/ 4Ñ'ˆä˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LàÐØ# gÑ-ˆÜ—z‘z ,Ó/ˆØ# gÑ-ˆØÐ!Ø# nÑ4ˆô —=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØ˜Ð$Ð$r*   rc   Ún_repc                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r   N)rY   r»   r^   )rc   rÜ   r�   Únum_key_value_headsÚslenrÎ   s         r+   rÏ   rÏ   Z  so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr*   c                   ó¬   ‡ — e Zd Zdeez  fˆ fd„Z	 d	dej                  dej                  dz  dee	   de
ej                  ej                  f   fd„Zˆ xZS )
ÚVideoPrismAttentionrC   c                 ó6  •— t         ‰| �  «        || _        t        |d|j                  |j
                  z  «      | _        |j                  | _        | j                  dz  | _	        d| _
        t        j                  |j                  |j
                  | j                  z  |j                  ¬«      | _        t        j                  |j                  |j
                  | j                  z  |j                  ¬«      | _        t        j                  |j                  |j
                  | j                  z  |j                  ¬«      | _        t        j                  |j
                  | j                  z  |j                  d¬«      | _        d| _        |j&                  | _        y )NrÎ   rË   F©ÚbiasTr¢   )rH   rI   rC   r8   rP   Únum_attention_headsrÎ   Úattention_probs_dropout_probÚattention_dropoutr½   Ú	is_causalrM   ÚLinearÚqkv_biasÚq_projÚk_projÚv_projÚo_projrÐ   Úattn_logit_softcappingr”   s     €r+   rI   zVideoPrismAttention.__init__g  s4  ø€ Ü‰ÑÔØˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ!'×!DÑ!DˆÔØ—}‘} dÑ*ˆŒØˆŒä—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ðek×etÑetÔuˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ðek×etÑetÔuˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ðek×etÑetÔuˆŒÜ—i‘i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^ÐeiÔjˆŒØ$'ˆÔ!Ø&,×&CÑ&CˆÕ#r*   Nrc   rÈ   rÖ   r5   c                 ó°  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }t        j                  | j                  j                  t        «      }	 |	| ||||f| j                  sdn| j                  | j                  | j                  dœ|¤Ž\  }
} |
j                   g |¢d‘­Ž j#                  «       }
| j%                  |
«      }
|
|fS )Nry   r   rG   ç        )rt   r½   rÉ   )rY   rÎ   rë   r„   r[   rì   rí   r   Úget_interfacerC   Ú_attn_implementationrÛ   rÍ   rç   r½   rï   r^   rÕ   rî   )r<   rc   rÈ   rÖ   r›   Úhidden_shapeÚquery_statesr×   rØ   Úattention_interfacerÚ   rÙ   s               r+   rd   zVideoPrismAttention.forwardv  sT  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LØ×/Ñ/ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r*   rŒ   )r"   r#   r$   r   r   rI   r&   rf   r   r   r>   rd   rh   ri   s   @r+   rá   rá   f  sm   ø„ ðDÐ5Ð8LÑLõ Dð$ /3ñ)à—|‘|ð)ð Ÿ™ tÑ+ð)ð Ð+Ñ,ð	)ð
 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷)r*   rá   c                   óD   — e Zd Zdej                  dej                  fd„Zy)ÚVideoPrismLayerNormrc   r5   c                 óŠ   — t        j                  || j                  | j                  dz   | j                  | j
                  «      S )Nr¢   )ÚFÚ
layer_normÚnormalized_shapeÚweighträ   Úeps©r<   rc   s     r+   rd   zVideoPrismLayerNorm.forward™  s9   € ô �|‰|˜M¨4×+@Ñ+@À$Ç+Á+ÐPSÑBSÐUY×U^ÑU^Ð`d×`hÑ`hÓiÐir*   N)r"   r#   r$   r&   rf   rd   r)   r*   r+   rø   rø   ˜  s    „ ðj U§\¡\ð j°e·l±lô jr*   rø   c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚVideoPrismMLPrC   c                 ó  •— t         ‰| �  «        || _        t        |j                     | _        t        j                  |j                  |j                  «      | _
        t        j                  |j                  |j                  «      | _        y rŒ   )rH   rI   rC   r	   Ú
hidden_actÚactivation_fnrM   ré   rP   Úintermediate_sizeÚfc1Úfc2r”   s     €r+   rI   zVideoPrismMLP.__init__¡  sd   ø€ Ü‰ÑÔØˆŒÜ# F×$5Ñ$5Ñ6ˆÔÜ—9‘9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9‘9˜V×5Ñ5°v×7IÑ7IÓJˆ�r*   rc   r5   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rŒ   )r  r  r  rÿ   s     r+   rd   zVideoPrismMLP.forward¨  s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆàÐr*   )	r"   r#   r$   r   rI   r&   rf   rd   rh   ri   s   @r+   r  r     s-   ø„ ðKÐ/õ Kð U§\¡\ð °e·l±l÷ r*   r  c            	       óŽ   ‡ — e Zd Zdeez  fˆ fd„Z	 d	dej                  dej                  dz  dee	   dej                  fd„Z
ˆ xZS )
ÚVideoPrismLayerrC   c                 óB  •— t         ‰| �  «        t        |«      | _        t	        |j
                  |j                  ¬«      | _        t	        |j
                  |j                  ¬«      | _        t        |«      | _
        t        j                  |j                  «      | _        y ©N)rþ   )rH   rI   rá   Ú	attentionrø   rP   Úlayer_norm_epsÚlayernorm_beforeÚlayernorm_afterr  ÚmlprM   rr   rs   rt   r”   s     €r+   rI   zVideoPrismLayer.__init__±  sr   ø€ Ü‰ÑÔÜ,¨VÓ4ˆŒÜ 3°F×4FÑ4FÈF×LaÑLaÔ bˆÔÜ2°6×3EÑ3EÈ6×K`ÑK`ÔaˆÔÜ  Ó(ˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r*   Nrc   rÈ   rÖ   r5   c                 óú   — |}| j                  |«      } | j                  ||fi |¤Ž\  }}| j                  |«      }||z   }|}| j                  |«      }| j	                  |«      }| j                  |«      }||z   }|S rŒ   )r  r  rt   r  r  )r<   rc   rÈ   rÖ   Úresidualr�   s         r+   rd   zVideoPrismLayer.forward¹  s�   € ð !ˆØ×-Ñ-¨mÓ<ˆØ)˜4Ÿ>™>¨-¸ÑRÈ6ÑRÑˆ�qØŸ™ ]Ó3ˆØ%¨Ñ0ˆð !ˆØ×,Ñ,¨]Ó;ˆØŸ™ Ó/ˆØŸ™ ]Ó3ˆØ%¨Ñ0ˆàÐr*   rŒ   )r"   r#   r$   r   r   rI   r&   rf   r   r   rd   rh   ri   s   @r+   r
  r
  °  s]   ø„ ð>Ð5Ð8LÑLõ >ð /3ñà—|‘|ðð Ÿ™ tÑ+ðð Ð+Ñ,ð	ð
 
�‰÷r*   r
  c                   óˆ   ‡ — e Zd ZU eed<   dZdZdZdZg d¢Z	dZ
dZdZdZdZeedœZ ej&                  «       ˆ fd	„«       Zˆ xZS )
ÚVideoPrismPreTrainedModelrC   ÚmodelrU   )ÚvideoÚtextT)rk   r’   r
  r±   Ú'VideoPrismMultiheadAttentionPoolingHeadF)rc   Ú
attentionsc                 ó¢  •— t         ‰| �  |«       t        |t        j                  t        j
                  f«      r t        j                  |j                  «       yt        |t        «      r t        j                  |j                  «       yt        |t        «      r t        j                  |j                  «       yt        |t        «      r?t        j                  |j                  «       t        j                  |j                  «       yt        |t         «      r t        j                  |j                  «       yt        |t"        «      rçt%        |j&                  j(                  |j&                  j*                  «      j-                  |j.                  j0                  |j.                  j2                  ¬«      }t        j4                  |j.                  |«       t        j4                  |j6                  t9        j:                  |j6                  j<                  d   «      j?                  d«      «       yt        |t@        «      r�t        jB                  |jD                  jF                  j                  |j&                  j*                  dz  ¬«       t        jB                  |jD                  jH                  |j&                  j*                  dz  ¬«       yy)zInitialize the weights©Údevicer¤   ry   rµ   rË   )ÚstdN)%rH   Ú_init_weightsrL   rM   ré   rN   ÚinitÚlecun_normal_rý   rk   rq   r’   r  Úzeros_Úper_dim_scaleÚpooling_attention_queryrø   r±   r¯   rC   rº   rP   rÂ   r³   r  r¤   Úcopy_r´   r&   r¦   rY   r»   ÚVideoPrismTextModelÚnormal_rv   r¸   r¼   )r<   rÄ   r³   rT   s      €r+   r  z'VideoPrismPreTrainedModel._init_weightsê  sØ  ø€ ô 	‰Ñ˜fÔ%Ü�fœrŸy™y¬"¯)©)Ð4Ô5Ü×Ñ˜vŸ}™}Õ-ä˜Ô ;Ô<Ü×Ñ˜v×9Ñ9Õ:ä˜Ô <Ô=Ü×Ñ˜v×9Ñ9Õ:ä˜Ô GÔHÜ�K‰K˜×,Ñ,Ô-Ü×Ñ˜v×=Ñ=Õ>ä˜Ô 3Ô4Ü�K‰K˜Ÿ™Õ&ä˜Ô 8Ô9Ü!<Ø—‘×5Ñ5°v·}±}×7PÑ7Pó"ç‰b˜×1Ñ1×8Ñ8À×@YÑ@Y×@_Ñ@_ˆbÓ`ð ô �J‰J�v×0Ñ0Ð2DÔEÜ�J‰J�v×*Ñ*¬E¯L©L¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÕhä˜Ô 3Ô4Ü�L‰L˜×*Ñ*×:Ñ:×AÑAÀvÇ}Á}×G`ÑG`ÐbfÑGfÕgÜ�L‰L˜×*Ñ*×2Ñ2¸¿¹×8QÑ8QÐSWÑ8WÖXð 5r*   )r"   r#   r$   r   r(   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_sdpaÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_attention_backendÚ_can_compile_fullgraphr
  rá   Ú_can_record_outputsr&   Úno_gradr  rh   ri   s   @r+   r  r  Ð  ss   ø… àÓØÐØ+€OØ(ÐØ&*Ð#òÐð €NØÐØÐØ"&ÐØ!Ðà(Ø)ñÐð
 €U‡]�]ƒ_óYó ôYr*   r  z¬
    The bare VideoPrism vision encoder outputting raw hidden-states without any specific head on top. This model is the backbone encoder used in VideoPrismVideoModel.
    c                   óÚ   ‡ — e Zd ZU eed<   dZdZdefˆ fd„Zdej                  fd„Z
dej                  fd„Zeee	 	 dd
ej                   d	z  ded	z  dee   defd„«       «       «       Zˆ xZS )ÚVideoPrismVisionModelrC   ©r  r  c                 ó@  •— t         ‰| �  |«       t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        |«      | _        t        |«      | _
        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        t        j                  t        |j"                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        | j'                  «        y c c}w c c}w r  )rH   rI   rø   rP   r  Ú
layernorm1Ú
layernorm2rk   Úspatial_embeddingsr’   Útemporal_embeddingsrM   Ú
ModuleListÚrangeÚnum_spatial_layersr
  Úspatial_layersÚnum_temporal_layersÚtemporal_layersÚ	post_init©r<   rC   r�   rT   s      €r+   rI   zVideoPrismVisionModel.__init__  sË   ø€ Ü‰Ñ˜Ô Ü-¨f×.@Ñ.@Àf×F[ÑF[Ô\ˆŒÜ-¨f×.@Ñ.@Àf×F[ÑF[Ô\ˆŒÜ"=¸fÓ"EˆÔÜ#?ÀÓ#GˆÔ Ü Ÿm™mÌeÐTZ×TmÑTmÓNnÖ,oÈ¬_¸VÕ-DÒ,oÓpˆÔÜ!Ÿ}™}ÌuÐU[×UoÑUoÓOpÖ-qÈ!¬o¸fÕ.EÒ-qÓrˆÔØ�‰Õùò -pùÚ-qs   Â#DÃ'Dr5   c                 ó.   — | j                   j                  S rŒ   ©r:  rn   r@   s    r+   Úget_input_embeddingsz*VideoPrismVisionModel.get_input_embeddings  s   € Ø×&Ñ&×7Ñ7Ð7r*   rÇ   c                 ó&   — || j                   _        y rŒ   rE  ©r<   rÇ   s     r+   Úset_input_embeddingsz*VideoPrismVisionModel.set_input_embeddings!  s   € Ø38ˆ×ÑÕ0r*   NrU   rV   rÖ   c                 ó,  — |€t        d«      ‚|j                  }| j                  ||«      }|}| j                  D ]  } ||fi |¤Ž}Œ | j	                  |«      }| j                  |||«      }	|	}
| j                  D ]  } ||
fi |¤Ž}
Œ | j                  |
«      }|j                  \  }}}|j                  |d   d||«      j                  dd«      j                  «       }|j                  \  }}}}|j                  |d   ||z  d«      }t        ||
|¬«      S )Nz'You have to specify pixel_values_videosr   ry   r   rG   )Úlast_hidden_stater    r!   )rZ   rY   r:  r?  r8  r;  rA  r9  r„   r[   rÕ   r   )r<   rU   rV   rÖ   r›   Úspatial_embedsÚspatial_hidden_statesÚspatial_layerrž   Útemporal_embedsÚtemporal_hidden_statesÚtemporal_layerr�   r`   r†   rS   s                   r+   rd   zVideoPrismVisionModel.forward$  sM  € ð Ð&ÜÐFÓGÐGà)×/Ñ/ˆð ×0Ñ0Ð1DÐF^Ó_ˆØ .ÐØ!×0Ñ0ò 	SˆMÙ$1Ð2GÑ$RÈ6Ñ$RÑ!ð	Sà—?‘?Ð#8Ó9ˆð ×2Ñ2°8¸[ÐJbÓcˆØ!0ÐØ"×2Ñ2ò 	VˆNÙ%3Ð4JÑ%UÈfÑ%UÑ"ð	Và—?‘?Ð#9Ó:ˆð &Ÿ^™^Ñˆˆ:�sØ—=‘= ¨Q¡°°ZÀÓE×OÑOÐPQÐSTÓU×`Ñ`ÓbˆØ*2¯.©.Ñ'ˆˆ:�{ CØ—=‘= ¨Q¡°¸kÑ1IÈ2ÓNˆä:Ø&Ø'=Ø&;ô
ð 	
r*   ©NF)r"   r#   r$   r   r(   r*  r(  rI   rM   ÚModulerF  rI  r   r   r   r&   r'   rg   r   r   r   rd   rh   ri   s   @r+   r5  r5  
  s«   ø… ð #Ó"Ø!ÐØÐðÐ5õ ð8 b§i¡ió 8ð9¨"¯)©)ó 9ð  ØØð 9=Ø05ñ#
à"×.Ñ.°Ñ5ð#
ð #'¨¡+ð#
ð Ð+Ñ,ð	#
ð
 
5ò#
ó ó ó  ô#
r*   r5  g^$3eG÷?c                   ó¦   ‡ — e Zd Zdefˆ fd„Z	 d	dej                  dej                  dz  dee	   de
ej                  ej                  f   fd„Zˆ xZS )
r  rC   c                 óì  •— t         ‰| �  «        || _        |j                  |j                  z  | _        |j                  | _        t        | j
                  dz  z  | _	        d| _
        t        j                  |j                  |j                  | j
                  z  |j                  ¬«      | _        t        j                  |j                  |j                  | j
                  z  |j                  ¬«      | _        t        j                  |j                  |j                  | j
                  z  |j                  ¬«      | _        t        j                  |j                  | j
                  z  |j                  d¬«      | _        d| _        t        j(                  t+        j,                  | j
                  «      «      | _        t        j(                  t+        j,                  dd|j                  «      «      | _        y )Nrz   Frã   Tr¢   r   )rH   rI   rC   r  rå   rÎ   ræ   rç   Ú_R_SOFTPLUS_0r½   rè   rM   ré   rP   rê   rë   rì   rí   rî   rÐ   ro   r&   rp   r#  r$  r”   s     €r+   rI   z0VideoPrismMultiheadAttentionPoolingHead.__init__Q  sd  ø€ Ü‰ÑÔØˆŒØ×0Ñ0°F×4NÑ4NÑNˆŒØ!'×!DÑ!DˆÔÜ$¨¯©°sÑ(:Ñ;ˆŒØˆŒä—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ðek×etÑetÔuˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ðek×etÑetÔuˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ðek×etÑetÔuˆŒÜ—i‘i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^ÐeiÔjˆŒØ$'ˆÔ!ÜŸ\™\¬%¯+©+°d·m±mÓ*DÓEˆÔÜ')§|¡|´E·K±KÀÀ1Àf×FXÑFXÓ4YÓ'ZˆÕ$r*   Nrc   rÈ   rÖ   r5   c                 óŒ  — |j                   d d }g |¢d‘| j                  ‘­}| j                  j                  |d   dd«      } | j	                  |«      j
                  g |j                   d d ¢d‘| j                  ‘­Ž j                  dd«      }|| j                  z  t        j                  j                  | j                  «      z  }| j                  |«      j                  |«      j                  dd«      }	| j                  |«      j                  |«      j                  dd«      }
t        j                  | j                   j"                  t$        «      } || ||	|
|fd| j&                  sdn| j(                  d dœ|¤Ž\  }} |j*                  g |j                   d d ¢d‘­Ž j-                  «       }| j/                  |«      }||fS )Nry   r   r   rG   r¢   rñ   )r½   rt   rÉ   )rY   rÎ   r$  r»   rë   r„   r[   r½   rM   r‚   Úsoftplusr#  rì   rí   r   rò   rC   ró   rÛ   rÍ   rç   r^   rÕ   rî   )r<   rc   rÈ   rÖ   r›   rô   rÅ   Úquery_layerrõ   r×   rØ   rö   rÚ   rÙ   s                 r+   rd   z/VideoPrismMultiheadAttentionPoolingHead.forwarda  sº  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà×,Ñ,×3Ñ3°KÀ±NÀBÈÓKˆØ-�d—k‘k %Ó(×-Ñ-ÐS¨u¯{©{¸3¸BÐ/?ÐSÀÐSÀTÇ]Á]ÒS×]Ñ]Ð^_ÐabÓcˆØ" T§\¡\Ñ1´B·M±M×4JÑ4JÈ4×K]ÑK]Ó4^Ñ^ˆà—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð Ø#Ÿ}š}‘C°$×2HÑ2HØñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð@¨5¯;©;°s¸Ð+;Ð@¸RÒ@×KÑKÓMˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r*   rŒ   )r"   r#   r$   r   rI   r&   r'   rÃ   r   r   r>   rd   rh   ri   s   @r+   r  r  P  so   ø„ ð[Ð5õ [ð& 37ñ")à×(Ñ(ð")ð ×(Ñ(¨4Ñ/ð")ð Ð+Ñ,ð	")ð
 
ˆu× Ñ  %×"3Ñ"3Ð3Ñ	4÷")r*   r  Úxrþ   c                 ód   — t        j                  | | z  j                  |d¬«      |z   «      }| |z  S )zUThis function is intended to align with the l2norm implementation in the FLA library.T)r†   Úkeepdim)r&   ÚrsqrtÚsum)rZ  r†   rþ   Úinv_norms       r+   Úl2normr`  †  s0   € ä�{‰{˜A ™EŸ;™;¨3¸˜;Ó=ÀÑCÓD€HØˆx‰<Ðr*   z•
    The bare VideoPrism text encoder outputting last hidden states without any specific head on top. This model is used in VideoPrismClipModel.
    c                   óþ   ‡ — e Zd ZU eed<   dZdZdZddgZdZ	defˆ fd„Z
eee	 	 	 	 ddej                  d	z  d
ej                   d	z  dej                   d	z  dej                   d	z  dee   defd„«       «       «       Zˆ xZS )r&  rC   )r  r  r¿   r±   r
  r¸   c                 óB  •— t         ‰| �  |«       t        |«      | _        t	        j
                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _	        t        |j                  |j                  ¬«      | _        | j                  «        y c c}w r  )rH   rI   r±   rv   rM   r<  r=  Únum_hidden_layersr
  Úlayersrø   rP   r  Ú	layernormrB  rC  s      €r+   rI   zVideoPrismTextModel.__init__™  sq   ø€ Ü‰Ñ˜Ô Ü2°6Ó:ˆŒÜ—m‘mÄeÈF×LdÑLdÓFeÖ$fÀ¤_°VÕ%<Ò$fÓgˆŒÜ,¨V×-?Ñ-?ÀV×EZÑEZÔ[ˆŒØ�‰Õùò %gs   ÁBNrÈ   rÀ   r´   rÖ   r5   c                 óø  — |d u |d uz  rt        d«      ‚| j                  |||¬«      }|�lt        j                  |j                  d   d|j
                  |j                  ¬«      }t        j                  ||fd¬«      }t        | j                  ||d ¬«      }| j                  D ]  } |||fi |¤Ž}Œ | j                  |«      }|d d …df   }	| j                  j                  rt        |	d¬«      }	t        ||	¬	«      S )
Nz:You must specify exactly one of input_ids or inputs_embeds)r¿   r´   rÀ   r   r   r  r¥   )rC   rÀ   rÈ   Úpast_key_valuesry   )rK  Úpooler_output)rZ   rv   r&   ÚonesrY   r  r¤   rª   r
   rC   rd  re  Úapply_l2normr`  r   )
r<   r¿   rÈ   rÀ   r´   rÖ   rc   Úcls_paddingÚlayerÚtext_embeddingss
             r+   rd   zVideoPrismTextModel.forward   s  € ð ˜Ð -°tÐ";Ò<ÜÐYÓZÐZàŸ™°)È,Ðfs˜ÓtˆàÐ%ÜŸ*™*Ø×#Ñ# AÑ&¨°.×2GÑ2GÈ~×OcÑOcôˆKô #ŸY™Y¨¸Ð'DÈ!ÔLˆNÜ/Ø—{‘{Ø+Ø-Ø $ô	ˆNð —[‘[ò 	KˆEÙ! -°ÑJÀ6ÑJ‰Mð	KàŸ™ }Ó5ˆà'ª¨2¨Ñ.ˆØ�;‰;×#Ò#Ü$ _¸"Ô=ˆOä)¸MÐYhÔiÐir*   )NNNN)r"   r#   r$   r   r(   r*  r(  r)  r,  Ú_input_embed_layerrI   r   r   r   r&   rÃ   rf   r   r   r   rd   rh   ri   s   @r+   r&  r&  Œ  sÛ   ø… ð !Ó Ø ÐØÐØ!€OØ3Ð5FÐGÐØ*ÐðÐ3õ ð  ØØð .2Ø.2Ø-1Ø,0ñ!jà×#Ñ# dÑ*ð!jð Ÿ™ tÑ+ð!jð —|‘| dÑ*ð	!jð
 —l‘l TÑ)ð!jð Ð+Ñ,ð!jð 
$ò!jó ó ó  ô!jr*   r&  z¹
    VideoPrism video model consisting of the vision encoder backbone with auxiliary encoder layers and an attention pooling head on top. This model is used in VideoPrismClipModel.
    c                   óÀ   ‡ — e Zd ZU eed<   defˆ fd„Zdej                  fd„Zdej                  fd„Z	e
e	 ddej                  ded	z  d
ee   defd„«       «       Zˆ xZS )ÚVideoPrismVideoModelrC   c                 óv  •— t         ‰| �  |«       t        j                  |«      | _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _
        t        |«      | _        t        |j                  |j                  ¬«      | _        | j#                  «        y c c}w r  )rH   rI   r5  Ú_from_configÚvision_modelrM   r<  r=  Únum_auxiliary_layersr
  Úauxiliary_layersr  Úheadrø   rP   r  Úhead_layernormrB  rC  s      €r+   rI   zVideoPrismVideoModel.__init__Ï  sˆ   ø€ Ü‰Ñ˜Ô Ü1×>Ñ>¸vÓFˆÔÜ "§¡ÔPUÐV\×VqÑVqÓPrÖ.sÈ1¬¸vÕ/FÒ.sÓ tˆÔÜ;¸FÓCˆŒ	Ü1°&×2DÑ2DÈ&×J_ÑJ_Ô`ˆÔØ�‰Õùò /ts   ÁB6r5   c                 ó6   — | j                   j                  «       S rŒ   ©rs  rF  r@   s    r+   rF  z)VideoPrismVideoModel.get_input_embeddings×  ó   € Ø× Ñ ×5Ñ5Ó7Ð7r*   rÇ   c                 ó:   — | j                   j                  |«       y rŒ   ©rs  rI  rH  s     r+   rI  z)VideoPrismVideoModel.set_input_embeddingsÚ  ó   € Ø×Ñ×.Ñ.¨uÕ5r*   rU   rV   NrÖ   c                 óR  —  | j                   d||dœ|¤Ž}|j                  }| j                  D ]  } ||fi |¤Ž}Œ  | j                  |fi |¤Ž}| j	                  |d   «      }| j
                  j                  rt        |d¬«      }t        |||j                  |j                  ¬«      S )N©rU   rV   r   ry   r¥   )rK  rh  rc   r  r)   )rs  rK  ru  rv  rw  rC   rj  r`  r   rc   r  )	r<   rU   rV   rÖ   Úvision_model_outputsÚauxiliary_hidden_statesrl  Úhead_outputÚvideo_embeddingss	            r+   rd   zVideoPrismVideoModel.forwardÝ  sÍ   € ð  1˜t×0Ñ0ð  
Ø 3ÐNfñ 
Øjpñ 
Ðð #7×"HÑ"HÐØ×*Ñ*ò 	OˆEÙ&+Ð,CÑ&NÀvÑ&NÑ#ð	Oð  �d—i‘iÐ 7ÑB¸6ÑBˆØ×.Ñ.¨{¸1©~Ó>ÐØ�;‰;×#Ò#Ü%Ð&6¸BÔ?Ðä)Ø5Ø*Ø.×<Ñ<Ø+×6Ñ6ô	
ð 	
r*   re   )r"   r#   r$   r   r(   rI   rM   rS  rF  rI  r   r   r&   r'   rg   r   r   r   rd   rh   ri   s   @r+   rp  rp  Ç  s�   ø… ð #Ó"ðÐ5õ ð8 b§i¡ió 8ð6¨"¯)©)ó 6ð Øð 16ñ
à"×.Ñ.ð
ð #'¨¡+ð
ð Ð+Ñ,ð	
ð
 
$ò
ó ó ô
r*   rp  zÆ
    VideoPrism model for video-text contrastive learning. This model consists of a VideoPrismVideoModel and a VideoPrismTextModel, and computes similarity scores between video and text inputs.
    c                   óÄ  ‡ — e Zd Zdefˆ fd„Zdej                  fd„Zdej                  fd„Ze	e
	 ddej                  d	ej                  dz  d
ee   deez  fd„«       «       Ze	e
	 ddej$                  dedz  d
ee   deez  fd„«       «       Ze	e
	 	 	 	 ddej$                  dej                  d	ej                  dz  dedz  dedz  dedz  d
ee   defd„«       «       Zˆ xZS )ÚVideoPrismClipModelrC   c                 óÔ   •— t         ‰| �  |«       t        j                  |j                  «      | _        t        j                  |j                  «      | _        | j                  «        y rŒ   )
rH   rI   rp  rr  Úvision_configÚvideo_modelr&  Útext_configÚ
text_modelrB  r”   s     €r+   rI   zVideoPrismClipModel.__init__ÿ  sK   ø€ Ü‰Ñ˜Ô Ü/×<Ñ<¸V×=QÑ=QÓRˆÔÜ-×:Ñ:¸6×;MÑ;MÓNˆŒØ�‰Õr*   r5   c                 ó6   — | j                   j                  «       S rŒ   )rŠ  rF  r@   s    r+   rF  z(VideoPrismClipModel.get_input_embeddings  s   € Ø�‰×3Ñ3Ó5Ð5r*   rÇ   c                 ó:   — | j                   j                  |«       y rŒ   )rŠ  rI  rH  s     r+   rI  z(VideoPrismClipModel.set_input_embeddings  s   € Ø�‰×,Ñ,¨UÕ3r*   Nr¿   rÈ   rÖ   c                 ó,   —  | j                   d||dœ|¤ŽS )a  
        Examples:

        ```python
        >>> from transformers import AutoTokenizer, VideoPrismClipModel

        >>> model = VideoPrismClipModel.from_pretrained("google/videoprism-lvt-base-f16r288")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/videoprism-lvt-base-f16r288")

        >>> inputs = tokenizer(["a video of a cat.", "a video of a dog."], padding="max_length", return_tensors="pt")
        >>> with torch.no_grad():
        ...     text_features = model.get_text_features(**inputs)
        ```©r¿   rÈ   r)   )rŠ  )r<   r¿   rÈ   rÖ   s       r+   Úget_text_featuresz%VideoPrismClipModel.get_text_features  s    € ð* ˆt�‰Ð\¨À>Ñ\ÐU[Ñ\Ð\r*   rU   rV   c                 ó,   —  | j                   d||dœ|¤ŽS )a÷  
        Examples:

        ```python
        >>> from transformers import VideoPrismProcessor, VideoPrismClipModel

        >>> model = VideoPrismClipModel.from_pretrained("google/videoprism-lvt-base-f16r288")
        >>> processor = VideoPrismProcessor.from_pretrained("google/videoprism-lvt-base-f16r288")

        >>> inputs = processor(videos="path/to/video.mp4", return_tensors="pt")
        >>> with torch.no_grad():
        ...     video_features = model.get_video_features(**inputs)
        ```r  r)   )rˆ  )r<   rU   rV   rÖ   s       r+   Úget_video_featuresz&VideoPrismClipModel.get_video_features"  s.   € ð*  ˆt×Ñð 
Ø 3Ø%=ñ
ð ñ
ð 	
r*   ÚtemperatureÚreturn_lossc           	      ór  —  | j                   d||dœ|¤Ž} | j                  d||dœ|¤Ž}	|j                  }
|	j                  }|
j                  d   }|j                  d   }|
j	                  d|«      }|j	                  d|«      }t        j                  ||j                  «      }|�||z  }t        j                  |«      }|j                  }|t        j                  |dd¬«      z  }|t        j                  |dd¬«      z  }d}|r t        j                  |j                  d«      |j                  ¬«      }t        j                  |«       d	|z  z   }t
        j                  j                  j!                  ||z  «      }t        j                  |d¬
«       }|j#                  «       }t%        ||||||	|¬«      S )a  
        temperature (`float`, *optional*):
            A temperature scalar to scale the similarity scores. If not provided, no scaling is applied.
        return_loss (`bool`, *optional*):
            Whether or not to return the contrastive loss.
        r  rŽ  ry   Nr   T)r†   Úkeepdims)r  rG   r¥   )r.   r/   r0   r1   r2   r3   r4   r)   )r‘  r�  rh  rY   r^   r&   rÑ   ÚTÚexpr^  Úeyer}   r  Ú	ones_likerM   r‚   Ú
logsigmoidÚmeanr-   )r<   rU   r¿   rÈ   rV   r’  r“  rÖ   Úvideo_model_outputsÚtext_model_outputsrƒ  rm  Úvideo_emb_dimÚtext_emb_dimr0   r1   Úsimilarity_matrixr.   r/   r4   r˜  Úm1_diag1ÚloglikÚnlls                           r+   rd   zVideoPrismClipModel.forward=  s½  € ð& 6˜d×5Ñ5ð 
Ø 3ÐNfñ
Øjpñ
Ðð 4˜T×3Ñ3Ðq¸iÐXfÑqÐjpÑqÐà.×<Ñ<ÐØ,×:Ñ:ˆØ(×.Ñ.¨rÑ2ˆØ&×,Ñ,¨RÑ0ˆà'×/Ñ/°°MÓBˆØ%×-Ñ-¨b°,Ó?ˆÜ!ŸL™L¨°{·}±}ÓEÐàÐ"Ø Ñ,Ðä Ÿ9™9Ð%6Ó7ÐØ*×,Ñ,ˆØ+¬e¯i©iÐ8HÈaÐZ^Ô._Ñ_ÐØ)¬E¯I©I°oÈ1ÐW[Ô,\Ñ\ˆð ˆÙä—)‘)˜O×0Ñ0°Ó3¸O×<RÑ<RÔSˆCÜŸ™¨Ó8Ð8¸1¸s¹7ÑBˆHÜ—X‘X×(Ñ(×3Ñ3°H¸Ñ4NÓOˆFÜ—9‘9˜V¨Ô,Ð,ˆCØ—8‘8“:ˆDä#Ø-Ø+Ø%Ø#Ø2Ø0Øô
ð 	
r*   rŒ   re   )NFNN)r"   r#   r$   r   rI   rM   rS  rF  rI  r   r   r&   rf   r   r   r>   r   r�  r'   rg   r‘  r©   r-   rd   rh   ri   s   @r+   r…  r…  ù  s�  ø„ ðÐ/õ ð6 b§i¡ió 6ð4¨"¯)©)ó 4ð Øð /3ñ]à—<‘<ð]ð Ÿ™ tÑ+ð]ð Ð+Ñ,ð	]ð
 
Ð+Ñ	+ò]ó ó ð]ð* Øð 16ñ
à"×.Ñ.ð
ð #'¨¡+ð
ð Ð+Ñ,ð	
ð
 
Ð+Ñ	+ò
ó ó ð
ð2 Øð
 /3Ø05Ø$(Ø#'ñ9
à"×.Ñ.ð9
ð —<‘<ð9
ð Ÿ™ tÑ+ð	9
ð
 #'¨¡+ð9
ð ˜T‘\ð9
ð ˜D‘[ð9
ð Ð+Ñ,ð9
ð 
ò9
ó ó ô9
r*   r…  z
    VideoPrism Model transformer with a video classification head on top (a linear layer on top of the attention pooler).
    c                   óè   ‡ — e Zd ZU eed<   dZdZdefˆ fd„Zdej                  fd„Z
dej                  fd„Zee	 	 dd
ej                  dej                   d	z  ded	z  dee   def
d„«       «       Zˆ xZS )Ú VideoPrismForVideoClassificationrC   r6  r  c                 óB  •— t         ‰| �  |«       t        j                  |«      | _        t        |«      | _        t        |j                  |j                  ¬«      | _
        t        j                  |j                  |j                  «      | _        | j                  «        y r  )rH   rI   r5  rr  rs  r  rv  rø   rP   r  rw  rM   ré   Ú
num_labelsÚ
classifierrB  r”   s     €r+   rI   z)VideoPrismForVideoClassification.__init__…  sr   ø€ Ü‰Ñ˜Ô Ü1×>Ñ>¸vÓFˆÔÜ;¸FÓCˆŒ	Ü1°&×2DÑ2DÈ&×J_ÑJ_Ô`ˆÔÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒØ�‰Õr*   r5   c                 ó6   — | j                   j                  «       S rŒ   ry  r@   s    r+   rF  z5VideoPrismForVideoClassification.get_input_embeddings�  rz  r*   rÇ   c                 ó:   — | j                   j                  |«       y rŒ   r|  rH  s     r+   rI  z5VideoPrismForVideoClassification.set_input_embeddings�  r}  r*   NrU   ÚlabelsrV   rÖ   c                 ó<  —  | j                   d||dœ|¤Ž}|j                  }| j                   | j                  |fi |¤Žd   «      }| j	                  |«      }d }	|� | j
                  ||| j                  fi |¤Ž}	t        |	||j                  |j                  ¬«      S )Nr  r   )r4   Úlogitsrc   r  r)   )
rs  rK  rw  rv  r¨  Úloss_functionrC   r   rc   r  )
r<   rU   r«  rV   rÖ   r€  Úsequence_outputÚpooled_outputr­  r4   s
             r+   rd   z(VideoPrismForVideoClassification.forward“  sº   € ð  1˜t×0Ñ0ð  
Ø 3ÐNfñ 
Øjpñ 
Ðð /×@Ñ@ˆØ×+Ñ+¨I¨D¯I©I°oÑ,PÈÑ,PÐQRÑ,SÓTˆØ—‘ Ó/ˆØˆØÐØ%�4×%Ñ% f¨f°d·k±kÑLÀVÑLˆDä$ØØØ.×<Ñ<Ø+×6Ñ6ô	
ð 	
r*   rR  )r"   r#   r$   r   r(   r*  r(  rI   rM   rS  rF  rI  r   r   r&   r'   rÃ   rg   r   r   r   rd   rh   ri   s   @r+   r¥  r¥  {  s³   ø… ð #Ó"Ø!ÐØÐðÐ5õ ð8 b§i¡ió 8ð6¨"¯)©)ó 6ð Øð +/Ø05ñ	
à"×.Ñ.ð
ð × Ñ  4Ñ'ð
ð #'¨¡+ð	
ð
 Ð+Ñ,ð
ð 
ò
ó ó ô
r*   r¥  )r5  r  rp  r&  r…  r¥  )rñ   NN)ry   g�íµ ÷Æ°>)KÚcollections.abcr   r   Údataclassesr   Útypingr   r&   Útorch.nnrM   Útorch.nn.functionalr‚   rú   Ú r   r   Úactivationsr	   Úmasking_utilsr
   Úmodeling_layersr   Úmodeling_outputsr   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   Úconfiguration_videoprismr   r   r   r   r-   rS  rB   rk   r’   r�   rf   r¯   r±   r©   r>   rÛ   rÏ   rá   Ú	LayerNormrø   r  r
  r  r5  rV  r  r'   r`  r&  rp  r…  r¥  Ú__all__r)   r*   r+   ú<module>rÃ     s2  ð÷, /Ý !Ý ã Ý ß Ð å &Ý !Ý /Ý 9ß bÑ bß FÝ &ß aÕ aÝ 7Ý 5ß dÑ dñ ÐiÔjØ
ô?°/ó ?ó ó kð?ñ ØBôð ô 
˜;ó  
ó óð 
ôF$ "§)¡)ô $ôND "§)¡)ô DôN< 2§9¡9ô <ð~P¨ð P°3ð P¸5¿<¹<ó Pô ˜rŸy™yô  ðR Ø Ø ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �S‰[ð%ð �T‰\ð%ð �T‰\ð%ð ˆ5�<‰<˜Ÿ™Ð%Ñ&ó%ðD	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uô/)˜"Ÿ)™)ô /)ôdj˜"Ÿ,™,ô jô�B—I‘Iô ô Ð0ô ð@ ô6Y ó 6Yó ð6Yñr ðôô
;
Ð5ó ;
óð
;
ð| €ô3)¨b¯i©iô 3)ñlˆe×Ñð  cð °Uó ñ ðôô
3jÐ3ó 3jóð
3jñl ðôô
*
Ð4ó *
óð
*
ñZ ðôô
z
Ð3ó z
óð
z
ñz ðôô
+
Ð'@ó +
óð
+
ò\�r*   