Ë
    ÜÓ^j3¢  ã                   óÌ  — d dl mZ d dlmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZ dd	lmZmZ dd
lmZmZ ddlmZ ddlmZmZmZmZmZ ddlmZ ddlmZmZ ddl m!Z!  ejD                  e#«      Z$ ed¬«      e G d„ de«      «       «       Z% ed¬«      e G d„ de«      «       «       Z& G d„ dejN                  «      Z( G d„ dejN                  «      Z)	 dIdejN                  dejT                  dejT                  dejT                  d ejT                  dz  d!e+d"e+fd#„Z,d$„ Z- G d%„ d&ejN                  «      Z. G d'„ d(ejN                  «      Z/ G d)„ d*ejN                  «      Z0 G d+„ d,e«      Z1 G d-„ d.ejN                  «      Z2d/ejT                  d0e3ejT                     d1ejT                  fd2„Z4 G d3„ d4ejN                  «      Z5 G d5„ d6ejN                  «      Z6 G d7„ d8ejN                  «      Z7 G d9„ d:ejN                  «      Z8 G d;„ d<e«      Z9 G d=„ d>e«      Z: G d?„ d@ejN                  «      Z;e G dA„ dBe«      «       Z<e G dC„ dDe<«      «       Z= edE¬«       G dF„ dGe<«      «       Z>g dH¢Z?y)Jé    )ÚCallable)Ú	dataclassN)Únné   )Úinitialization)ÚACT2FN)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚImageClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚModelOutputÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚVJEPA2ConfigzO
    VJEPA Predictor outputs that also contains the masked encoder outputs
    )Úcustom_introc                   óä   — e Zd ZU dZej
                  ed<   dZej
                  dz  ed<   dZe	ej
                  df   dz  ed<   dZ
e	ej
                  df   dz  ed<   dZej
                  dz  ed<   y)	Ú$VJEPA2WithMaskedInputPredictorOutputaê  
    masked_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*, returned when `context_mask` is provided which is applied on VJEPA2Encoder outputs):
        The masked hidden state of the model.
    target_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*, returned when `target_mask` is provided which is applied on VJEPA2Encoder outputs):
        The target hidden state of the model.
    Úlast_hidden_stateNÚmasked_hidden_state.Úhidden_statesÚ
attentionsÚtarget_hidden_state)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚtorchÚFloatTensorÚ__annotations__r   r   Útupler   r    © ó    úu/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/vjepa2/modeling_vjepa2.pyr   r   #   s   … ñð ×(Ñ(Ó(Ø48Ð˜×*Ñ*¨TÑ1Ó8Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>Ø7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ó;Ø48Ð˜×*Ñ*¨TÑ1Ô8r*   r   zs
    VJEPA outputs that also contains the masked encoder outputs
    Optionally contains the predictor outputs
    c                   óâ   ‡ — e Zd ZU dZej
                  ed<   dZej
                  dz  ed<   dZe	ej
                  df   dz  ed<   dZ
e	ej
                  df   dz  ed<   dZedz  ed<   ˆ fd	„Zˆ xZS )
Ú VJEPA2WithMaskedInputModelOutputaq  
    masked_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*, returned when `context_mask` is provided which is applied on VJEPA2Encoder outputs):
        The masked hidden state of the model.
    predictor_output (`VJEPA2WithMaskedInputPredictorOutput`, *optional*):
        The output from the Predictor module.
    r   Nr   .r   r   Úpredictor_outputc                 óš   •— t        t        ‰| �	  «       «      }t        |d   t        «      r|d   j                  «       |d<   t        |«      S )Néÿÿÿÿ)ÚlistÚsuperÚto_tupleÚ
isinstancer   r(   )ÚselfÚoutputÚ	__class__s     €r+   r3   z)VJEPA2WithMaskedInputModelOutput.to_tupleM   sD   ø€ Ü”e‘gÑ&Ó(Ó)ˆÜ�f˜R‘jÔ"FÔGØ ™×,Ñ,Ó.ˆF�2‰JÜ�V‹}Ðr*   )r!   r"   r#   r$   r%   r&   r'   r   r   r(   r   r.   r   r3   Ú__classcell__©r7   s   @r+   r-   r-   8   s…   ø… ñð ×(Ñ(Ó(Ø48Ð˜×*Ñ*¨TÑ1Ó8Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>Ø7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ó;ØDHÐÐ:¸TÑAÓH÷ð r*   r-   c                   óx   ‡ — e Zd ZdZ	 d	dedefˆ fd„Zed„ «       Zde	j                  de	j                  fd„Zˆ xZS )
ÚVJEPA2PatchEmbeddings3Dz"
    Image to Patch Embedding
    ÚconfigÚhidden_sizec                 óH  •— t         ‰| �  «        |j                  | _        |j                  | _        || _        t        j                  |j                  ||j                  |j                  |j                  f|j                  |j                  |j                  f¬«      | _        y )N)Úin_channelsÚout_channelsÚkernel_sizeÚstride)	r2   Ú__init__Ú
patch_sizeÚtubelet_sizer=   r   ÚConv3dÚin_chansÚproj©r5   r<   r=   r7   s      €r+   rC   z VJEPA2PatchEmbeddings3D.__init__Y   s…   ø€ ô
 	‰ÑÔØ ×+Ñ+ˆŒØ"×/Ñ/ˆÔØ&ˆÔä—I‘IØŸ™Ø$Ø×,Ñ,¨f×.?Ñ.?À×ARÑARÐSØ×'Ñ'¨×):Ñ):¸F×<MÑ<MÐNô	
ˆ�	r*   c                 óœ   — | j                   | j                  z  | j                  | j                  z  z  | j                  | j                  z  z  S ©N©Úframes_per_cliprE   Ú	crop_sizerD   ©r<   s    r+   Únum_patchesz#VJEPA2PatchEmbeddings3D.num_patchesj   sO   € ð ×#Ñ# v×':Ñ':Ñ:Ø×Ñ 6×#4Ñ#4Ñ4ñ6à×Ñ 6×#4Ñ#4Ñ4ñ6ð	
r*   Úpixel_values_videosÚreturnc                 óf   — | j                  |«      j                  d«      j                  dd«      }|S )Né   r   )rH   ÚflattenÚ	transpose)r5   rQ   Úxs      r+   ÚforwardzVJEPA2PatchEmbeddings3D.forwardr   s.   € Ø�I‰IÐ)Ó*×2Ñ2°1Ó5×?Ñ?ÀÀ1ÓEˆØˆr*   ©é   )r!   r"   r#   r$   r   ÚintrC   ÚstaticmethodrP   r%   ÚTensorrX   r8   r9   s   @r+   r;   r;   T   sS   ø„ ñð  ñ
àð
ð õ
ð" ñ
ó ð
ð¨5¯<©<ð ¸E¿L¹L÷ r*   r;   c                   óf   ‡ — e Zd ZdZddedefˆ fd„Zdej                  dej                  fd„Z	ˆ xZ
S )	ÚVJEPA2Embeddingsú>
    Construct mask token, position and patch embeddings.
    r<   r=   c                 óº   •— t         ‰| �  «        || _        || _        t	        ||¬«      | _        | j
                  j                  | _        |j                  | _        y )N©r=   )r2   rC   r<   r=   r;   Úpatch_embeddingsrP   rD   rI   s      €r+   rC   zVJEPA2Embeddings.__init__|   sM   ø€ Ü‰ÑÔàˆŒØ&ˆÔÜ 7¸ÈKÔ XˆÔà×0Ñ0×<Ñ<ˆÔØ ×+Ñ+ˆ�r*   rQ   rR   c                 ól  — |j                   d   }|j                  ddddd«      }|| j                  j                  k  r)|j	                  dd| j                  j                  dd«      }| j
                  j                  j                  j                  }|j                  |¬«      }| j                  |«      }|S )Nr   r   rT   r   é   )Údtype)
ÚshapeÚpermuter<   rE   Úrepeatrc   rH   Úweightrf   Úto)r5   rQ   Ú
num_framesÚtarget_dtypeÚ
embeddingss        r+   rX   zVJEPA2Embeddings.forward†   s®   € Ø(×.Ñ.¨qÑ1ˆ
ð 2×9Ñ9¸!¸QÀÀ1ÀaÓHÐð ˜Ÿ™×0Ñ0Ò0Ø"5×"<Ñ"<¸QÀÀ4Ç;Á;×C[ÑC[Ð]^Ð`aÓ"bÐà×,Ñ,×1Ñ1×8Ñ8×>Ñ>ˆØ1×4Ñ4¸<Ð4ÓHÐØ×*Ñ*Ð+>Ó?ˆ
àÐr*   rY   )r!   r"   r#   r$   r   r[   rC   r%   r]   rX   r8   r9   s   @r+   r_   r_   w   s6   ø„ ññ,˜|ð ,¸#õ ,ð¨5¯<©<ð ¸E¿L¹L÷ r*   r_   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 ó²  — t        j                  ||j                  dd«      «      |z  }t        j                  j                  |dt         j                  ¬«      j                  |j                  «      }t        j                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )Nr0   éþÿÿÿ)Údimrf   )ÚpÚtrainingr   rT   )r%   ÚmatmulrV   r   Ú
functionalÚsoftmaxÚfloat32rk   rf   ru   rz   Ú
contiguous)
ro   rp   rq   rr   rs   rt   ru   ÚkwargsÚattn_weightsÚattn_outputs
             r+   Úeager_attention_forwardrƒ   š   s®   € ô —<‘<  s§}¡}°R¸Ó'<Ó=ÀÑG€Lô —=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€Lô —=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r*   c                 ó  — | j                  «       \  }}}}t        j                  |dz  | j                  | j                  ¬«      }||dz  z  }dd|z  z  }|j                  d«      |z  }|j                  «       }|j                  «       }	|j                  dddd«      }|	j                  dddd«      }	| j                  dd«      }
|
j                  d¬	«      \  }}t        j                  | |fd¬	«      }
|
j                  d
«      }
| |	z  |
|z  z   S )NrT   ©rf   Údeviceg       @g      ð?i'  r0   r   )r0   rT   ©rx   rw   )Úsizer%   Úarangerf   r†   Ú	unsqueezeÚsinÚcosri   Ú	unflattenÚunbindÚstackrU   )rW   ÚposÚBÚ	num_headsÚNÚDÚomegaÚfreqÚemb_sinÚemb_cosÚyÚy1Úy2s                r+   Úrotate_queries_or_keysrœ   ´   s  € ØŸ™›Ñ€A€y�!�Qô
 �L‰L˜˜a™ q§w¡w°q·x±xÔ@€EØ	ˆQ�‰WÑ€EØ�%˜‘,Ñ€EØ�=‰=˜Ó˜uÑ$€Dð �h‰h‹j€GØ�h‰h‹j€Gà�n‰n˜Q  1 aÓ(€GØ�n‰n˜Q  1 aÓ(€Gð 	
�‰�B˜Ó €AØ�X‰X˜"ˆXÓ�F€Bˆä�‰�b�S˜"�I 2Ô&€AØ	�	‰	�"‹€AØ�‰K˜A ™KÑ(Ð(r*   c                   ó¬   ‡ — e Zd Z	 	 ddededefˆ fd„Zd„ Zd„ Zdd„Zd	„ Z		 dd
e
j                  dz  dee
j                  e
j                  f   fd„Zˆ xZS )ÚVJEPA2RopeAttentionr<   r=   Únum_attention_headsc                 óz  •— t         ‰| �  «        || _        || _        || _        ||z  dk7  rt        d|f› d|› d�«      ‚t        ||z  «      | _        | j                  | j                  z  | _        t        j                  || j                  |j                  ¬«      | _        t        j                  || j                  |j                  ¬«      | _        t        j                  || j                  |j                  ¬«      | _        t        j                  ||«      | _        |j                   | _        t        j$                  | j"                  «      | _        | j                  j(                  | j                  j*                  z  | _        | j                  j.                  | j                  j0                  z  | _        t        d| j                  dz  dz  z  «      | _        t        d| j                  dz  dz  z  «      | _        t        d| j                  dz  dz  z  «      | _        | j                  dz  | _        d	| _        y )
Nr   zThe hidden size z4 is not a multiple of the number of attention heads ú.©ÚbiasrT   r   ç      à¿F)r2   rC   r<   r=   rŸ   Ú
ValueErrorr[   Úattention_head_sizeÚall_head_sizer   ÚLinearÚqkv_biasrp   rq   rr   rH   Úattention_probs_dropout_probÚdropout_probÚDropoutru   rN   rD   Ú	grid_sizerM   rE   Ú
grid_depthÚd_dimÚh_dimÚw_dimrt   Ú	is_causal)r5   r<   r=   rŸ   r7   s       €r+   rC   zVJEPA2RopeAttention.__init__Ð   sØ  ø€ ô 	‰ÑÔØˆŒØ&ˆÔØ#6ˆÔ ØÐ,Ñ,°Ò1ÜØ" K >Ð"2ð 3Ø,Ð-¨Qð0óð ô
 $' {Ð5HÑ'HÓ#IˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä—Y‘Y˜{¨D×,>Ñ,>ÀVÇ_Á_ÔUˆŒ
Ü—9‘9˜[¨$×*<Ñ*<À6Ç?Á?ÔSˆŒÜ—Y‘Y˜{¨D×,>Ñ,>ÀVÇ_Á_ÔUˆŒ
ä—I‘I˜k¨;Ó7ˆŒ	Ø"×?Ñ?ˆÔÜ—z‘z $×"3Ñ"3Ó4ˆŒàŸ™×.Ñ.°$·+±+×2HÑ2HÑHˆŒØŸ+™+×5Ñ5¸¿¹×9QÑ9QÑQˆŒä˜˜t×7Ñ7¸1Ñ<ÀÑBÑCÓDˆŒ
Ü˜˜t×7Ñ7¸1Ñ<ÀÑBÑCÓDˆŒ
Ü˜˜t×7Ñ7¸1Ñ<ÀÑBÑCÓDˆŒ
à×/Ñ/°Ñ5ˆŒØˆ�r*   c                 óP   — t        | j                  | j                  z  «      }||z  S rK   )r[   r­   )r5   ÚidsÚtokens_per_frames      r+   Ú_get_frame_posz"VJEPA2RopeAttention._get_frame_posõ   s&   € Ü˜tŸ~™~°·±Ñ>Ó?ÐØÐ&Ñ&Ð&r*   c                 óš   — t        | j                  | j                  z  «      }| j                  |«      }|||z  z
  }| j                  }||z  S rK   )r[   r­   r¶   )r5   r´   rµ   Ú	frame_idsÚtokens_per_rows        r+   Ú_get_height_posz#VJEPA2RopeAttention._get_height_posù   sN   € ä˜tŸ~™~°·±Ñ>Ó?ÐØ×'Ñ'¨Ó,ˆ	ØÐ$ yÑ0Ñ0ˆàŸ™ˆØ�nÑ$Ð$r*   Nc                 óŽ  — |j                   }|j                  d«      }|�-|j                  d«      j                  d| j                  d«      }nt        j                  ||¬«      }t        | j                  | j                  z  «      }| j                  |«      }| j                  }| j                  |«      }	|||z  z
  ||	z  z
  }
||	|
fS )Nr   ©r†   )r†   rˆ   rŠ   ri   rŸ   r%   r‰   r[   r­   r¶   rº   )r5   rW   Úmasksr†   Ú
token_sizer´   rµ   r¸   r¹   Ú
height_idsÚ	width_idss              r+   Úget_position_idsz$VJEPA2RopeAttention.get_position_ids  s½   € Ø—‘ˆØ—V‘V˜A“Yˆ
ð ÐØ—/‘/ !Ó$×+Ñ+¨A¨t×/GÑ/GÈÓK‰Cä—,‘,˜z°&Ô9ˆCä˜tŸ~™~°·±Ñ>Ó?ÐØ×'Ñ'¨Ó,ˆ	àŸ™ˆØ×)Ñ)¨#Ó.ˆ
ð Ð+¨iÑ7Ñ7¸>ÈJÑ;VÑVˆ	Ø˜* iÐ/Ð/r*   c                 óÔ  — |\  }}}d}t        |d||| j                  z   …f   |¬«      }|| j                  z  }t        |d||| j                  z   …f   |¬«      }|| j                  z  }t        |d||| j                  z   …f   |¬«      }	|| j                  z  }|| j                  k  r&|d|d …f   }
t        j                  |||	|
gd¬«      }|S t        j                  |||	gd¬«      }|S )Nr   .)r�   r0   r‡   )rœ   r¯   r°   r±   r¦   r%   Úcat)r5   ÚqkÚpos_idsÚd_maskÚh_maskÚw_maskÚsÚqkdÚqkhÚqkwÚqkrs              r+   Úapply_rotary_embeddingsz+VJEPA2RopeAttention.apply_rotary_embeddings  sý   € Ø!(Ñˆ�˜ØˆÜ$ R¨¨Q°°T·Z±Z±Ð-?Ð(?Ñ%@ÀfÔMˆØ	ˆT�Z‰Z‰ˆÜ$ R¨¨Q°°T·Z±Z±Ð-?Ð(?Ñ%@ÀfÔMˆØ	ˆT�Z‰Z‰ˆÜ$ R¨¨Q°°T·Z±Z±Ð-?Ð(?Ñ%@ÀfÔMˆØ	ˆT�Z‰Z‰ˆàˆt×'Ñ'Ò'Ø�S˜!™"�W‘+ˆCÜ—‘˜C  c¨3Ð/°RÔ8ˆBð ˆ	ô —‘˜C  c˜?°Ô3ˆBØˆ	r*   Úposition_maskrR   c           
      ó8  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }| j                  ||¬«      }| j                  ||«      }| j                  ||«      }t        j                  | j                  j                  t        «      }	 |	| |||d | j                  | j                  | j                   sdn| j"                  ¬«      \  }
}|
j%                  «       d d | j&                  fz   }| j)                  |
j+                  |«      «      }
|
|fS )Nr0   r   rT   )r½   ç        ©r²   rt   ru   rw   )rg   r¦   rp   ÚviewrV   rq   rr   rÁ   rÎ   r   Úget_interfacer<   Ú_attn_implementationrƒ   r²   rt   rz   r«   rˆ   r§   rH   Úreshape)r5   r   rÏ   Úinput_shapeÚhidden_shapeÚquery_layerÚ	key_layerÚvalue_layerrÅ   Úattention_interfaceÚcontext_layerÚattention_probsÚnew_context_layer_shapes                r+   rX   zVJEPA2RopeAttention.forward(  sˆ  € ð
 $×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆØ—j‘j Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆØ—H‘H˜]Ó+×0Ñ0°Ó>×HÑHÈÈAÓNˆ	Ø—j‘j Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆà×'Ñ'¨¸]Ð'ÓKˆØ×0Ñ0°¸GÓDˆ	Ø×2Ñ2°;ÀÓHˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ *=ØØØØØØ—n‘nØ—L‘LØ#Ÿ}š}‘C°$×2CÑ2Cô	*
Ñ&ˆ�ð #0×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØŸ	™	 -×"7Ñ"7Ð8OÓ"PÓQˆà˜oÐ-Ð-r*   )rZ   é   rK   )r!   r"   r#   r   r[   rC   r¶   rº   rÁ   rÎ   r%   r]   r(   rX   r8   r9   s   @r+   rž   rž   Ï   sz   ø„ ð  Ø#%ñ	#àð#ð ð#ð !õ	#òJ'ò%ó0ò*ð( .2ñ!.ð —|‘| dÑ*ð!.ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷	!.r*   rž   c                   óf   ‡ — e Zd Zddededefˆ fd„Zdej                  dej                  fd„Z	ˆ xZ
S )	Ú	VJEPA2MLPr<   r=   Ú	mlp_ratioc                 óê   •— t         ‰| �  «        |x}}t        ||z  «      }t        j                  ||d¬«      | _        t        |j                     | _        t        j                  ||d¬«      | _	        y ©NTr¢   )
r2   rC   r[   r   r¨   Úfc1r   Ú
hidden_actÚ
activationÚfc2)r5   r<   r=   rã   Úin_featuresÚout_featuresÚhidden_featuresr7   s          €r+   rC   zVJEPA2MLP.__init__M  sa   ø€ Ü‰ÑÔØ%0Ð0ˆ�lÜ˜k¨IÑ5Ó6ˆÜ—9‘9˜[¨/ÀÔEˆŒÜ  ×!2Ñ!2Ñ3ˆŒÜ—9‘9˜_¨lÀÔFˆ�r*   Úhidden_staterR   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rK   )ræ   rè   ré   )r5   rí   s     r+   rX   zVJEPA2MLP.forwardU  s2   € Ø—x‘x Ó-ˆØ—‘ |Ó4ˆØ—x‘x Ó-ˆØÐr*   )rZ   ç      @)r!   r"   r#   r   r[   ÚfloatrC   r%   r]   rX   r8   r9   s   @r+   râ   râ   L  s=   ø„ ñG˜|ð G¸#ð GÐQVõ Gð E§L¡Lð °U·\±\÷ r*   râ   c                   ór   ‡ — e Zd ZdZd	deddfˆ fd„Zdej                  dej                  fd„Zde	fd„Z
ˆ xZS )
ÚVJEPA2DropPathzÏStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    Ú	drop_probrR   Nc                 ó0   •— t         ‰| �  «        || _        y rK   )r2   rC   ró   )r5   ró   r7   s     €r+   rC   zVJEPA2DropPath.__init__d  s   ø€ Ü‰ÑÔØ"ˆ�r*   r   c                 óP  — | j                   dk(  s| j                  s|S d| j                   z
  }|j                  d   fd|j                  dz
  z  z   }t	        j
                  ||j                  |j                  ¬«      }t	        j                  ||z   «      }|j                  |«      |z  S )NrÑ   r   r   ©r   r…   )
ró   rz   rg   Úndimr%   Úrandrf   r†   ÚfloorÚdiv)r5   r   Ú	keep_probrg   Úrandom_tensors        r+   rX   zVJEPA2DropPath.forwardh  s—   € Ø�>‰>˜SÒ ¨¯ªØ Ð Ø˜Ÿ™Ñ&ˆ	Ø×$Ñ$ QÑ'Ð)¨D°M×4FÑ4FÈÑ4JÑ,KÑKˆÜŸ
™
 5°×0CÑ0CÈM×L`ÑL`ÔaˆÜŸ™ M°IÑ$=Ó>ˆØ× Ñ  Ó+¨mÑ;Ð;r*   c                 ó    — d| j                   › �S )Nzp=)ró   ©r5   s    r+   Ú
extra_reprzVJEPA2DropPath.extra_reprq  s   € Ø�D—N‘NÐ#Ð$Ð$r*   ©rÑ   )r!   r"   r#   r$   rð   rC   r%   r]   rX   Ústrrÿ   r8   r9   s   @r+   rò   rò   ]  sB   ø„ ññ# %ð #°$õ #ð< U§\¡\ð <°e·l±ló <ð%˜C÷ %r*   rò   c                   ó°   ‡ — e Zd ZdZ	 	 	 	 ddededededef
ˆ fd„Z	 dd	ej                  d
ej                  dz  de
e   deej                  df   fd„Zˆ xZS )ÚVJEPA2LayerzCThis corresponds to the Block class in the original implementation.r<   Údrop_path_rater=   rŸ   rã   c                 ó¢  •— t         ‰| �  «        || _        || _        || _        || _        t        j                  ||j                  ¬«      | _	        t        |||«      | _        |j                  dkD  rt        |«      nt        j                  «       | _        t        j                  ||j                  ¬«      | _        t#        |||¬«      | _        y )N©ÚepsrÑ   )r=   rã   )r2   rC   r<   r=   rŸ   rã   r   Ú	LayerNormÚlayer_norm_epsÚnorm1rž   Ú	attentionr  rò   ÚIdentityÚ	drop_pathÚnorm2râ   Úmlp)r5   r<   r  r=   rŸ   rã   r7   s         €r+   rC   zVJEPA2Layer.__init__x  s¥   ø€ ô 	‰ÑÔØˆŒØ&ˆÔØ#6ˆÔ Ø"ˆŒä—\‘\ +°6×3HÑ3HÔIˆŒ
Ü,¨V°[ÐBUÓVˆŒØ;A×;PÑ;PÐSVÒ;Vœ¨Ô7Ô\^×\gÑ\gÓ\iˆŒÜ—\‘\ +°6×3HÑ3HÔIˆŒ
Ü˜V°È	ÔRˆ�r*   Nr   rÏ   r€   rR   .c                 óô   — |}| j                  |«      }| j                  ||¬«      \  }}| j                  |«      |z   }|}| j                  |«      }| j	                  |«      }| j                  |«      |z   }||fS )N)rÏ   )r
  r  r  r  r  )r5   r   rÏ   r€   ÚresidualÚattention_outputr�   s          r+   rX   zVJEPA2Layer.forwardŒ  s�   € ð !ˆØŸ
™
 =Ó1ˆØ)-¯©ØØ'ð *8ó *
Ñ&Ð˜,ð Ÿ™Ð'7Ó8¸8ÑCˆð !ˆØŸ
™
 =Ó1ˆØŸ™ Ó/ˆØŸ™ }Ó5¸Ñ@ˆð ˜lÐ*Ð*r*   )rÑ   rZ   rà   rï   rK   )r!   r"   r#   r$   r   rð   r[   rC   r%   r]   r   r   r(   rX   r8   r9   s   @r+   r  r  u  s    ø„ ÙMð
 !$ØØ#%ØñSàðSð ðSð ð	Sð
 !ðSð õSð. .2ñ+à—|‘|ð+ð —|‘| dÑ*ð+ð Ð+Ñ,ð	+ð
 
ˆu�|‰|˜SÐ Ñ	!÷+r*   r  c                   ó\   ‡ — e Zd Zdefˆ fd„Z	 ddej                  dz  dee   de	fd„Z
ˆ xZS )	ÚVJEPA2Encoderr<   c                 ó^  •— t         ‰| �  «        || _        t        ||j                  ¬«      | _        t        |j                  «      D �cg c]2  }|j                  dkD  r|j                  |z  |j                  dz
  z  nd‘Œ4 }}t        j                  t        |j                  «      D �cg c]3  }t        |||   |j                  |j                  |j                  ¬«      ‘Œ5 c}«      | _        t        j                  |j                  |j                   ¬«      | _        d| _        y c c}w c c}w )Nrb   r   rÑ   ©r  r=   rŸ   rã   r  F)r2   rC   r<   r_   r=   rn   ÚrangeÚnum_hidden_layersr  r   Ú
ModuleListr  rŸ   rã   Úlayerr  r	  Ú	layernormÚgradient_checkpointing©r5   r<   ÚiÚdrop_path_ratesr7   s       €r+   rC   zVJEPA2Encoder.__init__¦  s  ø€ Ü‰ÑÔØˆŒä*¨6¸v×?QÑ?QÔRˆŒô ˜6×3Ñ3Ó4ö
àð LR×KcÑKcÐfgÒKgˆV×"Ñ" QÑ&¨&×*BÑ*BÀQÑ*FÒGÐmpÑpð
ˆð 
ô —]‘]ô ˜v×7Ñ7Ó8ö	ð ô ØØ#2°1Ñ#5Ø &× 2Ñ 2Ø(.×(BÑ(BØ$×.Ñ.öò	ó
ˆŒ
ô Ÿ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒØ&+ˆÕ#ùò#
ùò
	s   Á
7D%Â)8D*NrQ   r€   rR   c                 ó¶   — | j                  |«      }t        | j                  «      D ]  \  }} ||d fi |¤Ž}|d   }Œ | j                  |«      }t	        |¬«      S )Nr   ©r   )rn   Ú	enumerater  r  r
   )r5   rQ   r€   r   r  Úlayer_moduleÚlayer_outputss          r+   rX   zVJEPA2Encoder.forward¾  sj   € ð
 Ÿ™Ð(;Ó<ˆä(¨¯©Ó4ò 	-‰OˆAˆ|Ù(¨¸ÑGÀÑGˆMØ)¨!Ñ,‰Mð	-ð Ÿ™ }Ó5ˆäØ+ô
ð 	
r*   rK   )r!   r"   r#   r   rC   r%   r]   r   r   r
   rX   r8   r9   s   @r+   r  r  ¥  sD   ø„ ð,˜|õ ,ð4 48ñ
à"Ÿ\™\¨DÑ0ð
ð Ð+Ñ,ð
ð 
÷	
r*   r  Útensorr½   rR   c                 ó  — g }|D ]j  }|j                  | j                  «      }|j                  d«      j                  dd| j	                  d«      «      }|t        j                  | d|¬«      gz  }Œl t        j                  |d¬«      S )zþ
    Args:
        tensor (`torch.Tensor`):
            Tensor of shape [batch_size, num_patches, feature_dim]
        masks (`List[torch.Tensor]`):
            List of tensors of shape [batch_size, num_patches] containing indices of patches to keep
    r0   r   ©rx   Úindexr   r‡   )rk   r†   rŠ   ri   rˆ   r%   ÚgatherrÃ   )r%  r½   Úall_masked_tensorsÚmaskÚ	mask_keeps        r+   Úapply_masksr-  Ð  s�   € ð ÐØò MˆØ�w‰w�v—}‘}Ó%ˆØ—N‘N 2Ó&×-Ñ-¨a°°F·K±KÀ³OÓDˆ	ØœuŸ|™|¨F¸ÀÔKÐLÑLÑðMô
 �9‰9Ð'¨QÔ/Ð/r*   c                   óÒ   ‡ — e Zd ZdZdefˆ fd„Zed„ «       Z	 ddej                  de
ej                     de
ej                     ded	eej                  ej                  f   f
d
„Zˆ xZS )ÚVJEPA2PredictorEmbeddingsr`   r<   c                 ó˜  •— t         ‰| �  «        || _        t        j                  |j
                  |j                  «      | _        d| _        |j                  | _
        |j                  | _        t        j                  t        j                  | j                  dd|j                  «      «      | _        |j                   | _        || _        y )Nr   r   )r2   rC   r<   r   r¨   r=   Úpred_hidden_sizeÚpredictor_embeddingsÚnum_mask_tokensÚpred_zero_init_mask_tokensÚzero_init_mask_tokensÚpred_num_mask_tokensÚ	Parameterr%   ÚzerosÚmask_tokensrD   ©r5   r<   r7   s     €r+   rC   z"VJEPA2PredictorEmbeddings.__init__æ  s™   ø€ Ü‰ÑÔàˆŒÜ$&§I¡I¨f×.@Ñ.@À&×BYÑBYÓ$ZˆÔ!Ø ˆÔØ%+×%FÑ%FˆÔ"Ø%×:Ñ:ˆÔÜŸ<™<¬¯©°D×4HÑ4HÈ!ÈQÐPV×PgÑPgÓ(hÓiˆÔà ×+Ñ+ˆŒØˆ�r*   c                 ó   — | j                   dkD  rM| j                   | j                  z  | j                  | j                  z  z  | j                  | j                  z  z  S | j                  | j                  z  | j                  | j                  z  z  S ©Nr   rL   rO   s    r+   rP   z%VJEPA2PredictorEmbeddings.num_patchesó  s‹   € à×!Ñ! AÒ%à×'Ñ'¨6×+>Ñ+>Ñ>Ø×#Ñ# v×'8Ñ'8Ñ8ñ:à×#Ñ# v×'8Ñ'8Ñ8ñ:ðð ×$Ñ$¨×(9Ñ(9Ñ9¸f×>NÑ>NÐRX×RcÑRcÑ>cÑdÐdr*   r   Úcontext_maskÚtarget_maskÚ
mask_indexrR   c                 óì  — |j                  d«      }| j                  |«      }|| j                  z  }| j                  |   }|d   j	                  «       dz   }|j                  ||d«      }t        ||«      }|j                  t        |«      dd«      }t        j                  ||gd¬«      }	t        j                  |d¬«      }
t        j                  |d¬«      }t        j                  |
|gd¬«      }|	|fS )zø
        hidden_states : encoder outputs (context)
        context_mask: tokens of the context (outputs from the encoder)
        target_mask: tokens to predict
        mask_index: index of the target mask to choose (useful for multiclip?)
        r   r   r‡   )
rˆ   r2  r3  r9  Úmaxri   r-  Úlenr%   rÃ   )r5   r   r=  r>  r?  r‘   ÚcontextÚtargetÚmax_patch_numrn   ÚcmÚtmr½   s                r+   rX   z!VJEPA2PredictorEmbeddings.forwardþ  sì   € ð ×Ñ˜qÓ!ˆØ×+Ñ+¨MÓ:ˆð   $×"6Ñ"6Ñ6ˆ
Ø×!Ñ! *Ñ-ˆð $ A™×*Ñ*Ó,¨qÑ0ˆØ—‘˜q -°Ó3ˆÜ˜V [Ó1ˆð —.‘.¤ \Ó!2°A°qÓ9ˆÜ—Y‘Y ¨Ð0°aÔ8ˆ
ô �Y‰Y�|¨Ô+ˆÜ�Y‰Y�{¨Ô*ˆÜ—	‘	˜2˜r˜(¨Ô*ˆà˜5Ð Ð r*   rö   )r!   r"   r#   r$   r   rC   r\   rP   r%   r]   r1   r[   r(   rX   r8   r9   s   @r+   r/  r/  á  sŒ   ø„ ñð˜|õ ð ñeó ðeð ñ&!à—|‘|ð&!ð ˜5Ÿ<™<Ñ(ð&!ð ˜%Ÿ,™,Ñ'ð	&!ð
 ð&!ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷&!r*   r/  c            
       óš   ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zdej                  de	ej                     de	ej                     de
e   d	ef
d
„Zˆ xZS )ÚVJEPA2Predictorr<   c                 ó¨  •— t         ‰| �  «        || _        d| _        t	        |«      | _        t        |j                  «      D �cg c]2  }|j                  dkD  r|j                  |z  |j                  dz
  z  nd‘Œ4 }}t        j                  t        |j                  «      D �cg c]3  }t        |||   |j                  |j                  |j                  ¬«      ‘Œ5 c}«      | _        t        j                   |j                  |j"                  ¬«      | _        t        j&                  |j                  |j(                  d¬«      | _        y c c}w c c}w )NFr   rÑ   r  r  Tr¢   )r2   rC   r<   r  r/  rn   r  Úpred_num_hidden_layersr  r   r  r  r1  Úpred_num_attention_headsÚpred_mlp_ratior  r  r	  r  r¨   r=   rH   r  s       €r+   rC   zVJEPA2Predictor.__init__(  s5  ø€ Ü‰ÑÔØˆŒØ&+ˆÔ#Ü3°FÓ;ˆŒô ˜6×8Ñ8Ó9ö
ð ð ×0Ñ0°1Ò4ð ×%Ñ%¨Ñ)¨V×-JÑ-JÈQÑ-NÒOàñð
ˆð 
ô —]‘]ô ˜v×<Ñ<Ó=ö	ð ô ØØ#2°1Ñ#5Ø &× 7Ñ 7Ø(.×(GÑ(GØ$×3Ñ3öò	ó
ˆŒ
ô Ÿ™ f×&=Ñ&=À6×CXÑCXÔYˆŒÜ—I‘I˜f×5Ñ5°v×7IÑ7IÐPTÔUˆ�	ùò+
ùò	s   Á7E
Â$8Ec                 ó8  — |j                  |j                  «      }t        j                  |d|¬«      }|j                  |j                  «      }|j	                  d«      j                  dd|j                  d«      «      }t        j                  |d|¬«      }||fS )Nr   r'  r0   )rk   r†   r%   r)  rŠ   Úexpandrˆ   )r5   r   Úposition_masksÚargsortÚhidden_states_argsorts        r+   Úsort_tokenszVJEPA2Predictor.sort_tokensD  sˆ   € à—*‘*˜^×2Ñ2Ó3ˆÜŸ™ n¸!À7ÔKˆð —*‘*˜]×1Ñ1Ó2ˆØ '× 1Ñ 1°"Ó 5× <Ñ <¸RÀÀ]×EWÑEWÐXZÓE[Ó \ÐÜŸ™ ]¸ÐAVÔWˆà˜nÐ,Ð,r*   c                 óü   — |j                  |j                  «      }t        j                  |d¬«      }|j	                  d«      j                  dd|j                  d«      «      }t        j                  |d|¬«      }|S )Nr   r‡   r0   r'  )rk   r†   r%   rQ  rŠ   rO  rˆ   r)  )r5   r   rQ  Úreverse_argsorts       r+   Úunsort_tokenszVJEPA2Predictor.unsort_tokensP  si   € Ø—*‘*˜]×1Ñ1Ó2ˆÜŸ-™-¨°QÔ7ˆØ)×3Ñ3°BÓ7×>Ñ>¸rÀ2À}×GYÑGYÐZ\ÓG]Ó^ˆÜŸ™ ]¸ÀÔQˆØÐr*   Úencoder_hidden_statesr=  r>  r€   rR   c                 ó®  — t        ||«      }|j                  \  }}}| j                  |||«      \  }}	t        j                  |	d¬«      }
| j                  ||	|
«      \  }}	t        | j                  «      D ]  \  }} |||	fi |¤Ž}|d   }Œ | j                  |«      }| j                  ||
«      }|d d …|d …f   }| j                  |«      }t        |¬«      S )Nr   r‡   r   r!  )r-  rg   rn   r%   rQ  rS  r"  r  r  rV  rH   r
   )r5   rW  r=  r>  r€   Ú_ÚN_ctxtr”   r   rP  rQ  r  r#  r$  s                 r+   rX   zVJEPA2Predictor.forwardW  sï   € ô !,Ð,AÀ<Ó PÐØ,×2Ñ2‰ˆˆ6�1Ø(,¯©Ð8MÈ|Ð]hÓ(iÑ%ˆ�~ô —-‘- °AÔ6ˆØ(,×(8Ñ(8¸ÈÐX_Ó(`Ñ%ˆ�~ä(¨¯©Ó4ò 	-‰OˆAˆ|Ù(¨¸ÑQÈ&ÑQˆMØ)¨!Ñ,‰Mð	-ð Ÿ™ }Ó5ˆà×*Ñ*¨=¸'ÓBˆØ%¢a¨© jÑ1ˆàŸ	™	 -Ó0ˆäØ+ô
ð 	
r*   )r!   r"   r#   r   rC   rS  rV  r%   r]   r1   r   r   r
   rX   r8   r9   s   @r+   rI  rI  '  sl   ø„ ðV˜|õ Vò8
-òð
à$Ÿ|™|ð
ð ˜5Ÿ<™<Ñ(ð
ð ˜%Ÿ,™,Ñ'ð	
ð
 Ð+Ñ,ð
ð 
÷
r*   rI  c            	       ó    ‡ — e Zd ZdZdefˆ fd„Z	 d	dej                  dej                  dz  deej                  ej                  f   fd„Z	ˆ xZ
S )
ÚVJEPA2PoolerSelfAttentionz=Multi-headed attention from 'Attention Is All You Need' paperr<   c                 ó  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k7  r&t        d| j                  › d| j                  › d�«      ‚| j                  dz  | _	        |j                  | _        d| _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        y ©Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).r¤   F)r2   rC   r<   r=   Ú	embed_dimrŸ   r’   Úhead_dimr¥   ÚscaleÚattention_dropoutru   r²   r   r¨   Úk_projÚv_projÚq_projÚout_projr:  s     €r+   rC   z"VJEPA2PoolerSelfAttention.__init__{  s  ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Ñ8ˆŒØ�=‰=˜4Ÿ>™>Ñ)¨T¯^©^Ò;ÜØMÈdÏnÉnÐM]ð ^Ø—N‘NÐ# 2ð'óð ð —]‘] DÑ(ˆŒ
Ø×/Ñ/ˆŒØˆŒä—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜŸ	™	 $§.¡.°$·.±.ÓAˆ�r*   Nr   rs   rR   c           
      ó®  — |j                   dd }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }t        j                  | j                  j                  t        «      } || ||||| j                  | j                  | j                  sdn| j                  ¬«      \  }	}
 |	j                   g |¢d‘­Ž j#                  «       }	| j%                  |	«      }	|	|
fS )ú#Input shape: Batch x Time x ChannelNr0   r   rT   rÑ   rÒ   )rg   r`  re  rÓ   rV   rc  rd  r   rÔ   r<   rÕ   rƒ   r²   ra  rz   ru   rÖ   r   rf  )r5   r   rs   r×   rØ   ÚqueriesÚkeysÚvaluesrÜ   r‚   r�   s              r+   rX   z!VJEPA2PoolerSelfAttention.forward�  s<  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆØ—+‘+˜mÓ,×1Ñ1°,Ó?×IÑIÈ!ÈQÓOˆØ�{‰{˜=Ó)×.Ñ.¨|Ó<×FÑFÀqÈ!ÓLˆØ—‘˜]Ó+×0Ñ0°Ó>×HÑHÈÈAÓNˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØØ—n‘nØ—J‘JØ#Ÿ}š}‘C°$·,±,ô	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—m‘m KÓ0ˆà˜LÐ(Ð(r*   rK   ©r!   r"   r#   r$   r   rC   r%   r]   r(   rX   r8   r9   s   @r+   r\  r\  x  sY   ø„ ÙGðB˜|õ Bð. /3ñ)à—|‘|ð)ð Ÿ™ tÑ+ð)ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷	)r*   r\  c                   óÐ   ‡ — e Zd ZdZdefˆ fd„Z	 ddej                  dej                  dej                  dej                  dz  d	eej                  ej                  f   f
d
„Z	ˆ xZ
S )ÚVJEPA2PoolerCrossAttentionz_It's different from other cross-attention layers, doesn't have output projection layer (o_proj)r<   c                 óº  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k7  r&t        d| j                  › d| j                  › d�«      ‚| j                  dz  | _	        |j                  | _        d| _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        y r^  )r2   rC   r<   r=   r_  rŸ   r’   r`  r¥   ra  rb  ru   r²   r   r¨   rc  rd  re  r:  s     €r+   rC   z#VJEPA2PoolerCrossAttention.__init__µ  s   ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Ñ8ˆŒØ�=‰=˜4Ÿ>™>Ñ)¨T¯^©^Ò;ÜØMÈdÏnÉnÐM]ð ^Ø—N‘NÐ# 2ð'óð ð —]‘] DÑ(ˆŒ
Ø×/Ñ/ˆŒØˆŒä—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆ�r*   Nri  rj  rk  rs   rR   c           
      ó  — |j                   \  }}}|j                   d   }| j                  |«      }| j                  |«      }| j                  |«      }|j	                  ||| j
                  | j                  «      j                  dd«      }|j	                  ||| j
                  | j                  «      j                  dd«      }|j	                  ||| j
                  | j                  «      j                  dd«      }t        j                  | j                  j                  t        «      }	 |	| ||||| j                  | j                  | j                  sdn| j                   ¬«      \  }
}|
j#                  |||«      j%                  «       }
|
|fS )rh  r   rT   rÑ   rÒ   )rg   re  rc  rd  rÓ   r’   r`  rV   r   rÔ   r<   rÕ   rƒ   r²   ra  rz   ru   rÖ   r   )r5   ri  rj  rk  rs   Ú
batch_sizeÚq_seq_lengthr_  Úkv_seq_lengthrÜ   r‚   r�   s               r+   rX   z"VJEPA2PoolerCrossAttention.forwardÈ  sT  € ð /6¯m©mÑ+ˆ
�L )ØŸ
™
 1™ˆà—+‘+˜gÓ&ˆØ�{‰{˜4Ó ˆØ—‘˜VÓ$ˆà—,‘,˜z¨<¸¿¹ÈÏÉÓW×aÑaÐbcÐefÓgˆØ�y‰y˜ ]°D·N±NÀDÇMÁMÓR×\Ñ\Ð]^Ð`aÓbˆØ—‘˜Z¨¸¿¹ÈÏÉÓV×`Ñ`ÐabÐdeÓfˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØØ—n‘nØ—J‘JØ#Ÿ}š}‘C°$·,±,ô	%
Ñ!ˆ�\ð "×)Ñ)¨*°lÀIÓN×YÑYÓ[ˆà˜LÐ(Ð(r*   rK   rl  r9   s   @r+   rn  rn  °  su   ø„ Ùið@˜|õ @ð0 /3ñ%)à—‘ð%)ð �l‰lð%)ð —‘ð	%)ð
 Ÿ™ tÑ+ð%)ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷%)r*   rn  c                   ó’   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  deej                  ej                  f   fd„Zˆ xZ	S )ÚVJEPA2PoolerSelfAttentionLayerr<   c                 ó:  •— t         ‰| �  «        t        j                  |j                  |j
                  ¬«      | _        t        |«      | _        t        j                  |j                  |j
                  ¬«      | _	        t        ||j                  ¬«      | _        y ©Nr  rb   )r2   rC   r   r  r=   r	  Úlayer_norm1r\  Ú	self_attnÚlayer_norm2râ   r  r:  s     €r+   rC   z'VJEPA2PoolerSelfAttentionLayer.__init__ò  sl   ø€ Ü‰ÑÔÜŸ<™<¨×(:Ñ(:À×@UÑ@UÔVˆÔÜ2°6Ó:ˆŒÜŸ<™<¨×(:Ñ(:À×@UÑ@UÔVˆÔÜ˜V°×1CÑ1CÔDˆ�r*   r   rs   rR   c                 ó¸   — |}| j                  |«      }| j                  ||¬«      \  }}||z   }|}| j                  |«      }| j                  |«      }||z   }||fS )aR  
        Args:
            hidden_states (`torch.FloatTensor`):
                Input to the layer of shape `(batch, seq_len, embed_dim)`.
            attention_mask (`torch.FloatTensor`):
                Attention mask of shape `(batch, 1, q_len, k_v_seq_len)` where padding elements are indicated by very large negative values.
        )r   rs   )rx  ry  rz  r  )r5   r   rs   r  r�   s        r+   rX   z&VJEPA2PoolerSelfAttentionLayer.forwardù  s{   € ð !ˆØ×(Ñ(¨Ó7ˆØ&*§n¡nØ'Ø)ð '5ó '
Ñ#ˆ�|ð ! =Ñ0ˆà ˆØ×(Ñ(¨Ó7ˆØŸ™ Ó/ˆØ  =Ñ0ˆà˜lÐ*Ð*r*   ©
r!   r"   r#   r   rC   r%   r]   r(   rX   r8   r9   s   @r+   ru  ru  ñ  sL   ø„ ðE˜|õ Eð+à—|‘|ð+ð Ÿ™ð+ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷	+r*   ru  c                   ó´   ‡ — e Zd Zdefˆ fd„Z	 d	dej                  dej                  dej                  dz  deej                  ej                  f   fd„Zˆ xZ	S )
ÚVJEPA2PoolerCrossAttentionLayerr<   c                 ó:  •— t         ‰| �  «        t        j                  |j                  |j
                  ¬«      | _        t        |«      | _        t        j                  |j                  |j
                  ¬«      | _	        t        ||j                  ¬«      | _        y rw  )r2   rC   r   r  r=   r	  rx  rn  Ú
cross_attnrz  râ   r  r:  s     €r+   rC   z(VJEPA2PoolerCrossAttentionLayer.__init__  sl   ø€ Ü‰ÑÔÜŸ<™<¨×(:Ñ(:À×@UÑ@UÔVˆÔÜ4°VÓ<ˆŒÜŸ<™<¨×(:Ñ(:À×@UÑ@UÔVˆÔÜ˜V°×1CÑ1CÔDˆ�r*   Nri  rí   rs   rR   c                 ó¾   — |}| j                  |«      }| j                  ||||¬«      ^}}||z   }|}| j                  |«      }| j                  |«      }||z   }|g|¢­S )N©rs   )rx  r€  rz  r  )r5   ri  rí   rs   r  r�   s         r+   rX   z'VJEPA2PoolerCrossAttentionLayer.forward  s†   € ð ˆØ×'Ñ'¨Ó5ˆØ&*§o¡oØØØØ)ð	 '6ó '
Ð#ˆ�|ð   ,Ñ.ˆð  ˆØ×'Ñ'¨Ó5ˆØ—x‘x Ó-ˆØ ,Ñ.ˆàÐ*˜lÑ*Ð*r*   rK   r|  r9   s   @r+   r~  r~    sd   ø„ ðE˜|õ Eð /3ñ	+à—‘ð+ð —l‘lð+ð Ÿ™ tÑ+ð	+ð
 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷+r*   r~  c                   ó`   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Zˆ xZ	S )ÚVJEPA2AttentivePoolerzAttentive Poolerr<   c                 óF  •— t         ‰| �  «        t        j                  t	        j
                  dd|j                  «      «      | _        t        |«      | _	        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        y c c}w r<  )r2   rC   r   r7  r%   r8  r=   Úquery_tokensr~  Úcross_attention_layerr  r  Únum_pooler_layersru  Úself_attention_layers)r5   r<   rY  r7   s      €r+   rC   zVJEPA2AttentivePooler.__init__:  sr   ø€ Ü‰ÑÔÜŸL™L¬¯©°Q¸¸6×;MÑ;MÓ)NÓOˆÔÜ%DÀVÓ%LˆÔ"Ü%'§]¡]Ü=BÀ6×C[ÑC[Ó=\Ö]¸Ô+¨FÕ3Ò]ó&
ˆÕ"ùÚ]s   Á?Brí   rR   c                 óÞ   — | j                   D ]  } ||d ¬«      d   }Œ | j                  j                  |j                  d   dd«      }| j	                  ||«      d   }|j                  d«      S )Nr‚  r   r   )r‰  r†  ri   rg   r‡  Úsqueeze)r5   rí   r  ri  s       r+   rX   zVJEPA2AttentivePooler.forwardB  sw   € Ø×/Ñ/ò 	GˆEÙ  ¸dÔCÀAÑF‰Lð	Gà×#Ñ#×*Ñ*¨<×+=Ñ+=¸aÑ+@À!ÀQÓGˆØ×1Ñ1°'¸<ÓHÈÑKˆØ×#Ñ# AÓ&Ð&r*   )
r!   r"   r#   r$   r   rC   r%   r]   rX   r8   r9   s   @r+   r„  r„  7  s-   ø„ Ùð
˜|õ 
ð' E§L¡Lð '°U·\±\÷ 'r*   r„  c                   óž   ‡ — e Zd ZU eed<   dZdZdZdZg d¢Z	dZ
dZ eed¬«       eed	d¬
«      dœZ ej"                  «       ˆ fd„«       Zˆ xZS )ÚVJEPA2PreTrainedModelr<   Úvjepa2rQ   ÚvideoT)r  ru  r~  r/  zencoder.layer)Ú
layer_namer   )r(  r�  )r   r   c                 óZ  •— t         ‰| �  |«       | j                  j                  }t	        |t
        «      �rt        j                  |j                  |¬«       t        |j                  d«      D ]w  \  }}||dz  z  }t        j                  |j                  j                  j                  |¬«       t        j                  |j                  j                  j                  |¬«       Œy |t!        |j                  «      dz   dz  z  }t        j                  |j"                  j                  j                  j                  |¬«       yt	        |t$        «      rN|j&                  r t        j(                  |j*                  «       yt        j                  |j*                  |¬«       yt	        |t,        j.                  t,        j0                  t,        j2                  f«      rNt        j                  |j                  |¬«       |j4                  � t        j(                  |j4                  «       yyy)zInitialize the weights)Ústdr   g      à?N)r2   Ú_init_weightsr<   Úinitializer_ranger4   r„  ÚinitÚtrunc_normal_r†  r"  r‰  ry  rf  rj   r  ré   rB  r‡  r/  r5  Úzeros_r9  r   r¨   ÚConv2drF   r£   )r5   ro   Úinit_stdr  r  r’  r7   s         €r+   r“  z#VJEPA2PreTrainedModel._init_weights^  sŒ  ø€ ô 	‰Ñ˜fÔ%à—;‘;×0Ñ0ˆÜ�fÔ3Õ4Ü×Ñ˜v×2Ñ2¸ÕAÜ% f×&BÑ&BÀAÓFò B‘��5Ø ! S¡&Ñ)�Ü×"Ñ" 5§?¡?×#;Ñ#;×#BÑ#BÈÕLÜ×"Ñ" 5§9¡9§=¡=×#7Ñ#7¸SÖAðBð œc &×">Ñ">Ó?À!ÑCÈÑKÑKˆCÜ×Ñ˜v×;Ñ;×?Ñ?×CÑC×JÑJÐPSÖTÜ˜Ô 9Ô:Ø×+Ò+Ü—‘˜F×.Ñ.Õ/ä×"Ñ" 6×#5Ñ#5¸8ÖDÜ˜¤§¡¬B¯I©I´r·y±yÐ AÔBÜ×Ñ˜vŸ}™}°(Õ;Ø�{‰{Ð&Ü—‘˜FŸK™KÕ(ð 'ð Cr*   )r!   r"   r#   r   r'   Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_sdpaÚ_supports_flash_attnr   r  rž   Ú_can_record_outputsr%   Úno_gradr“  r8   r9   s   @r+   r�  r�  J  sp   ø… àÓØ ÐØ+€OØÐØ&*Ð#òÐð €NØÐá'¨ÀÔPÙ$Ð%8ÀÈoÔ^ñÐð
 €U‡]�]ƒ_ó)ó ô)r*   r�  c                   óþ   ‡ — e Zd Zdefˆ fd„Zdefd„Ze ed¬«      e		 	 	 dde
j                  d	ee
j                     dz  d
ee
j                     dz  dedee   defd„«       «       «       Zde
j                  fd„Zˆ xZS )ÚVJEPA2Modelr<   c                 ó’   •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        | j                  «        y rK   )r2   rC   r<   r  ÚencoderrI  Ú	predictorÚ	post_initr:  s     €r+   rC   zVJEPA2Model.__init__y  s;   ø€ Ü‰Ñ˜Ô ØˆŒä$ VÓ,ˆŒÜ(¨Ó0ˆŒð 	�‰Õr*   rR   c                 óB   — | j                   j                  j                  S rK   )r¦  rn   rc   rþ   s    r+   Úget_input_embeddingsz VJEPA2Model.get_input_embeddingsƒ  s   € Ø�|‰|×&Ñ&×7Ñ7Ð7r*   F)Útie_last_hidden_statesNrQ   r=  r>  Úskip_predictorr€   c                 ó´  — |€t        d«      ‚ | j                  d
d|i|¤Ž}|j                  }|€¨|€¦|j                  d«      }|j                  d«      }	t	        j
                  |	|j                  ¬«      j                  d«      j                  |df«      g}t	        j
                  |	|j                  ¬«      j                  d«      j                  |df«      g}|sN | j                  d
|||dœ|¤Ž}
t        |
j                  t        ||«      |
j                  |
j                  ¬«      }nd}t        |t        ||«      |j                  |j                  |¬	«      }|S )az  
        context_mask (`torch.Tensor` with shape `[batch_size, patch_size, 1]`, *optional*):
            The mask position ids indicating which encoder output patches are going to be exposed to the predictor.
            By default, this mask is created as torch.arange(N).unsqueeze(0).repeat(B,1), indicating full context
            available to the predictor.
        target_mask (`torch.Tensor` with shape `[batch_size, patch_size, 1]`, *optional*):
            The mask position ids indicating which encoder output patches are going to be used as a prediction target
            for the predictor. By default, this mask is created as torch.arange(N).unsqueeze(0).repeat(B,1), indicating
            that the predictor should predict all encoder patches.
        skip_predictor (bool):
            flag to skip the predictor forward, useful if you just need the encoder outputs
        Nz'You have to specify pixel_values_videosrQ   r   r   r¼   )rW  r=  r>  )r   r    r   r   )r   r   r   r   r.   r)   )r¥   r¦  r   rˆ   r%   r‰   r†   rŠ   ri   r§  r   r-  r   r   r-   )r5   rQ   r=  r>  r¬  r€   Úencoder_outputsÚsequence_outputr‘   r“   Úpredictor_outputsr.   Úencoder_outputs                r+   rX   zVJEPA2Model.forward†  sy  € ð. Ð&ÜÐFÓGÐGà+7¨4¯<©<ñ ,
Ø 3ð,
àñ,
ˆð *×;Ñ;ˆàÐ KÐ$7Ø#×(Ñ(¨Ó+ˆAØ×$Ñ$ QÓ'ˆAÜ!ŸL™L¨Ð3F×3MÑ3MÔN×XÑXÐYZÓ[×bÑbÐdeÐghÐciÓjÐkˆLÜ Ÿ<™<¨Ð2E×2LÑ2LÔM×WÑWÐXYÓZ×aÑaÐcdÐfgÐbhÓiÐjˆKáØ1?°·±ð 2Ø&5Ø)Ø'ñ2ð ñ	2Ðô  DØ"3×"EÑ"EÜ$/°ÀÓ$MØ/×=Ñ=Ø,×7Ñ7ô	 Ñð  $Ðä9Ø-Ü +¨O¸\Ó JØ)×7Ñ7Ø&×1Ñ1Ø-ô
ˆð Ðr*   c                 ó@   — | j                  |d¬«      }|j                  S )NT)r¬  )rX   r   )r5   rQ   r±  s      r+   Úget_vision_featureszVJEPA2Model.get_vision_featuresÆ  s!   € ØŸ™Ð&9È$˜ÓOˆØ×/Ñ/Ð/r*   )NNF)r!   r"   r#   r   rC   r;   rª  r   r   r   r%   r]   r1   Úboolr   r   r-   rX   r³  r8   r9   s   @r+   r¤  r¤  w  sÁ   ø„ ð˜|õ ð8Ð&=ó 8ð  Ù¨EÔ2Øð 37Ø15Ø$ñ;à"Ÿ\™\ð;ð ˜5Ÿ<™<Ñ(¨4Ñ/ð;ð ˜%Ÿ,™,Ñ'¨$Ñ.ð	;ð
 ð;ð Ð+Ñ,ð;ð 
*ò;ó ó 3ó  ð;ðz0¸%¿,¹,÷ 0r*   r¤  z}
    V-JEPA 2 Model transformer with a video classification head on top (a linear layer on top of the attentive pooler).
    c                   óŽ   ‡ — e Zd Zdefˆ fd„Zee	 d	dej                  dej                  dz  de	e
   deez  fd„«       «       Zˆ xZS )
ÚVJEPA2ForVideoClassificationr<   c                 ó  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        |«      | _        t        j                  |j                  |j                  d¬«      | _
        | j                  «        y rå   )r2   rC   Ú
num_labelsr¤  rŽ  r„  Úpoolerr   r¨   r=   Ú
classifierr¨  r:  s     €r+   rC   z%VJEPA2ForVideoClassification.__init__Ñ  sd   ø€ Ü‰Ñ˜Ô à ×+Ñ+ˆŒÜ! &Ó)ˆŒô ,¨FÓ3ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÐPTÔUˆŒð 	�‰Õr*   NrQ   Úlabelsr€   rR   c                 ó  —  | j                   d|ddœ|¤Ž}|j                  }| j                  |«      }| j                  |«      }d}|�| j	                  ||| j
                  ¬«      }t        |||j                  |j                  ¬«      S )ag  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Examples:

        ```python
        >>> import torch
        >>> import numpy as np
        >>> from transformers import AutoVideoProcessor, VJEPA2ForVideoClassification

        >>> device = "cuda"

        >>> video_processor = AutoVideoProcessor.from_pretrained("facebook/vjepa2-vitl-fpc16-256-ssv2")
        >>> model = VJEPA2ForVideoClassification.from_pretrained("facebook/vjepa2-vitl-fpc16-256-ssv2").to(device)

        >>> video = np.ones((64, 256, 256, 3))  # 64 frames, 256x256 RGB
        >>> inputs = video_processor(video, return_tensors="pt").to(device)

        >>> # For inference
        >>> with torch.no_grad():
        ...     outputs = model(**inputs)
        >>> logits = outputs.logits

        >>> predicted_label = logits.argmax(-1).item()
        >>> print(model.config.id2label[predicted_label])

        >>> # For training
        >>> labels = torch.ones(1, dtype=torch.long, device=device)
        >>> loss = model(**inputs, labels=labels).loss

        ```T)rQ   r¬  N)Úpooled_logitsr»  r<   )ÚlossÚlogitsr   r   r)   )	rŽ  r   r¹  rº  Úloss_functionr<   r   r   r   )	r5   rQ   r»  r€   Úoutputsr   Úpooler_outputr¿  r¾  s	            r+   rX   z$VJEPA2ForVideoClassification.forwardÞ  s£   € ðV �$—+‘+ð 
Ø 3Øñ
ð ñ
ˆð $×5Ñ5ÐØŸ™Ð$5Ó6ˆØ—‘ Ó/ˆàˆØÐØ×%Ñ%°FÀ6ÐRV×R]ÑR]Ð%Ó^ˆDä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r*   rK   )r!   r"   r#   r   rC   r   r   r%   r]   r   r   r(   r   rX   r8   r9   s   @r+   r¶  r¶  Ë  so   ø„ ð˜|õ ð Øð '+ñ<
à"Ÿ\™\ð<
ð —‘˜tÑ#ð<
ð Ð+Ñ,ð	<
ð
 
Ð&Ñ	&ò<
ó ó ô<
r*   r¶  )r¤  r�  r¶  r   )@Úcollections.abcr   Údataclassesr   r%   r   Ú r   r•  Úactivationsr   Úmodeling_layersr	   Úmodeling_outputsr
   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úconfiguration_vjepa2r   Ú
get_loggerr!   Úloggerr   r-   ÚModuler;   r_   r]   rð   rƒ   rœ   rž   râ   rò   r  r  r1   r-  r/  rI  r\  rn  ru  r~  r„  r�  r¤  r¶  Ú__all__r)   r*   r+   ú<module>rÓ     s‘  ðõ %Ý !ã Ý å &Ý !Ý 9ß Fß FÝ &ß _Õ _Ý 7ß EÝ .ð 
ˆ×	Ñ	˜HÓ	%€ñ ðôð
 ô9¨;ó 9ó óð9ñ ðôð ô {ó ó óðô* ˜bŸi™iô  ôF�r—y‘yô ðT ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ó%ò4)ô6z.˜"Ÿ)™)ô z.ôz�—	‘	ô ô"%�R—Y‘Yô %ô0-+Ð,ô -+ô`(
�B—I‘Iô (
ðV0˜Ÿ™ð 0¨T°%·,±,Ñ-?ð 0ÀEÇLÁLó 0ô"C! §	¡	ô C!ôLN
�b—i‘iô N
ôb5) §	¡	ô 5)ôp=) §¡ô =)ôB!+Ð%?ô !+ôH+Ð&@ô +ôD'˜BŸI™Iô 'ð& ô))˜Oó ))ó ð))ðX ôP0Ð'ó P0ó ðP0ñf ðôô
L
Ð#8ó L
óð
L
ò^ S�r*   