
    ^j]                        d dl mZmZ d dlZd dlmZ ddlmZ ddlm	Z	 ddl
mZ ddlmZ dd	lmZmZmZ dd
lmZmZ ddlmZ ddlmZmZmZ ddlmZmZ ddlmZ ddl m!Z!  G d dejD                        Z# G d dejD                        Z$	 	 d/dejD                  dejJ                  dejJ                  dejJ                  dejJ                  dz  de&dz  de&dee   fdZ' G d dejD                        Z( G d  d!ejD                        Z) G d" d#e      Z* G d$ d%ejD                        Z+e G d& d'e             Z,e G d( d)e,             Z- ed*+       G d, d-e,             Z.g d.Z/y)0    )CallableIterableN)nn   )initialization)ACT2FN)create_bidirectional_mask)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPoolingImageClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstring	torch_int)can_return_tuplemerge_with_config_defaults)capture_outputs   )VivitConfigc                   `     e Zd ZdZdef fdZdej                  dej                  fdZ xZ	S )VivitTubeletEmbeddingsay  
    This class turns `pixel_values` of shape `(batch_size, num_frames, num_channels, height, width)` into the initial
    `hidden_states` (tubelet embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer encoder.

    The seq_length equals (num_frames // tubelet_size[0]) * (height // tubelet_size[1]) * (width // tubelet_size[2]).
    configc                 N   t         |           |j                  }|j                  }t	        |t
              r|n||f}|j                  |d   z  |d   |d   z  z  |d   |d   z  z  | _        || _        t        j                  |j                  |j                  ||      | _        y )Nr   r      )kernel_sizestride)super__init__tubelet_size
image_size
isinstancer   
num_framesnum_patchesr   Conv3dnum_channelshidden_size
projection)selfr   r"   r#   	__class__s       s/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/vivit/modeling_vivit.pyr!   zVivitTubeletEmbeddings.__init__0   s    **&&
#-j(#CZ*V`Ia
 ,q/1!}Q/1!}Q/1 	
 %))!3!3Vb
    pixel_valuesreturnc                     |j                  dd      }| j                  |      j                  d      j                  dd      S )Nr   r   )	transposer*   flatten)r+   r/   s     r-   forwardzVivitTubeletEmbeddings.forward@   s;    #--a3|,44Q7AA!QGGr.   )
__name__
__module____qualname____doc__r   r!   torchTensorr4   __classcell__r,   s   @r-   r   r   '   s2    
{ 
 HELL HU\\ Hr.   r   c                        e Zd ZdZdef fdZdej                  dededej                  fdZ	dd	ej                  d
e
dej                  fdZ xZS )VivitEmbeddingszY
    Construct the CLS token, position and tubelet patch embeddings for video input.
    r   c                    t         |           t        j                  t	        j
                  dd|j                              | _        t        |      | _	        | j                  j                  }t        j                  t	        j
                  d|dz   |j                              | _        t        j                  |j                        | _        |j                  dd  | _        | j                  j"                  | _        y )Nr   )r    r!   r   	Parameterr9   zerosr)   	cls_tokenr   patch_embeddingsr&   position_embeddingsDropouthidden_dropout_probdropoutr"   
patch_sizer#   )r+   r   r&   r,   s      r-   r!   zVivitEmbeddings.__init__K   s    ekk!Q8J8J&KL 6v >++77#%<<A{QPVPbPb0c#d zz&"<"<= --ab1//::r.   
embeddingsheightwidthr0   c                    | j                   j                  }| j                  j                  d   dz
  }t        j
                  j                         s||k(  r||k(  r| j                  S | j                  ddddf   }| j                  ddddf   }|j                  d   }|| j                  d   z  }	|| j                  d   z  }
t        |dz        }|j                  d|||      }|j                  dddd      }t        j                  j                  ||	|
fdd	
      }|j                  dddd      j                  dd|      }t	        j                  ||fd      S )a   
        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
        images. This method is also adapted to support torch.jit tracing.

        Adapted from:
        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
        r   Nr   g      ?r   r   bicubicF)sizemodealign_cornersdim)rC   r&   rD   shaper9   jit
is_tracingrH   r   reshapepermuter   
functionalinterpolateviewcat)r+   rI   rJ   rK   r&   num_positionsclass_pos_embedpatch_pos_embedrS   
new_height	new_widthsqrt_num_positionss               r-   interpolate_pos_encodingz(VivitEmbeddings.interpolate_pos_encodingW   sf    ++770066q9A= yy##%+*F6UZ?+++221bqb59221ab59r"tq11
T__Q//	&}c'9:)11!5GI[]`a)11!Q1=--33i(	 4 
 *11!Q1=BB1b#Nyy/?;CCr.   r/   rc   c                    |j                   \  }}}}}| j                  |      }| j                  j                  |dd      }	t	        j
                  |	|fd      }|r|| j                  |||      z   }ne|| j                  d   k7  s|| j                  d   k7  r2t        d| d| d| j                  d    d| j                  d    d	      || j                  z   }| j                  |      }|S )	NrM   r   rR   r   zInput image size (*z) doesn't match model (z).)rT   rC   rB   expandr9   r\   rc   r#   
ValueErrorrD   rG   )
r+   r/   rc   
batch_sizer%   r(   rJ   rK   rI   
cls_tokenss
             r-   r4   zVivitEmbeddings.forward~   s   >J>P>P;
Jfe**<8
 ^^**:r2>
YY
J7Q?
##d&C&CJPVX]&^^J++u8J/J (% 9+,Adooa.@-AE  $d&>&>>J\\*-
r.   )F)r5   r6   r7   r8   r   r!   r9   r:   intrc   boolr4   r;   r<   s   @r-   r>   r>   F   sl    
;{ 
;%D5<< %D %DUX %D]b]i]i %DNELL D ]b]i]i r.   r>   modulequerykeyvalueattention_maskscalingrG   kwargsc                    ||j                  d      dz  }t        j                  ||j                  dd            |z  }|||z   }t        j
                  j                  |dt        j                        j                  |j                        }t        j
                  j                  ||| j                        }t        j                  ||      }	|	j                  dd      j                         }	|	|fS )NrM         r   r   )rS   dtype)ptrainingr   )rO   r9   matmulr2   r   rY   softmaxfloat32toru   rG   rw   
contiguous)
rl   rm   rn   ro   rp   rq   rG   rr   attn_weightsattn_outputs
             r-   eager_attention_forwardr      s     **R.D( <<s}}Q':;gEL!#n4==((2U]](SVVW\WbWbcL==((6??([L,,|U3K''1-88:K$$r.   c                        e Zd Zdef fdZ	 d	dej                  dej                  dz  dee   de	ej                  ej                  f   fdZ
 xZS )
VivitAttentionr   c                 (   t         |           || _        |j                  | _        t	        |d|j
                  |j                  z        | _        |j                  | _        | j                  dz  | _	        d| _
        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j
                  |j                  | j                  z  |j                        | _        t        j                  |j                  | j                  z  |j
                  d      | _        y )Nhead_dimrt   F)biasT)r    r!   r   num_attention_headsgetattrr)   r   attention_probs_dropout_probattention_dropoutrq   	is_causalr   Linearqkv_biasq_projk_projv_projo_projr+   r   r,   s     r-   r!   zVivitAttention.__init__   s,   #)#=#= 
F4F4F&JdJd4de!'!D!D}}d*ii 2 2F4N4NQUQ^Q^4^eketetuii 2 2F4N4NQUQ^Q^4^eketetuii 2 2F4N4NQUQ^Q^4^eketetuii : :T]] JFL^L^eijr.   Nhidden_statesrp   rr   r0   c                    |j                   d d }g |d| j                  }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }| j                  |      j                  |      j	                  dd      }t        j                  | j                  j                  t              }	 |	| ||||f| j                  sdn| j                  | j                  d|\  }
} |
j                  g |d j!                         }
| j#                  |
      }
|
|fS )NrM   r   r           )rG   rq   )rT   r   r   r[   r2   r   r   r   get_interfacer   _attn_implementationr   rw   r   rq   rW   r|   r   )r+   r   rp   rr   input_shapehidden_shapequery_states
key_statesvalue_statesattention_interfacer~   r}   s               r-   r4   zVivitAttention.forward   sK    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST(?(M(MKK,,.E)
 %8	%
  $}}C$2H2HLL	%
 	%
!\ *k));;;;FFHkk+.L((r.   N)r5   r6   r7   r   r!   r9   r:   r   r   tupler4   r;   r<   s   @r-   r   r      sf    k{ k" /3)||) t+) +,	)
 
u||U\\)	*)r.   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )VivitMLPr   c                    t         |           || _        t        |j                     | _        t        j                  |j                  |j                        | _
        t        j                  |j                  |j                        | _        y r   )r    r!   r   r   
hidden_actactivation_fnr   r   r)   intermediate_sizefc1fc2r   s     r-   r!   zVivitMLP.__init__   sd    #F$5$5699V//1I1IJ99V55v7I7IJr.   r   r0   c                 l    | j                  |      }| j                  |      }| j                  |      }|S r   )r   r   r   )r+   r   s     r-   r4   zVivitMLP.forward   s4    /**=9/r.   	r5   r6   r7   r   r!   r9   r:   r4   r;   r<   s   @r-   r   r      s,    K{ KU\\ ell r.   r   c            	            e Zd Zdef fdZ	 d	dej                  dej                  dz  dee   dej                  fdZ	 xZ
S )

VivitLayerr   c                 j   t         |           t        |      | _        t	        j
                  |j                  |j                        | _        t	        j
                  |j                  |j                        | _	        t        |      | _        t	        j                  |j                        | _        y )Neps)r    r!   r   	attentionr   	LayerNormr)   layer_norm_epslayernorm_beforelayernorm_afterr   mlprE   rF   rG   r   s     r-   r!   zVivitLayer.__init__   sz    '/ "V-?-?VEZEZ [!||F,>,>FDYDYZF#zz&"<"<=r.   Nr   rp   rr   r0   c                     |}| j                  |      } | j                  ||fi |\  }}| j                  |      }||z   }|}| j                  |      }| j	                  |      }| j                  |      }||z   }|S r   )r   r   rG   r   r   )r+   r   rp   rr   residual_s         r-   r4   zVivitLayer.forward   s     !--m<)4>>-R6Rq]3%0 !,,];/]3%0r.   r   )r5   r6   r7   r   r!   r9   r:   r   r   r4   r;   r<   s   @r-   r   r      sV    >{ > /3|| t+ +,	
 
r.   r   c                   \     e Zd Zdef fdZdej                  dej                  fdZ xZS )VivitPoolerr   c                     t         |           t        j                  |j                  |j
                        | _        t        |j                     | _	        y r   )
r    r!   r   r   r)   pooler_output_sizedenser   
pooler_act
activationr   s     r-   r!   zVivitPooler.__init__  s>    YYv1163L3LM
 !2!23r.   r   r0   c                 \    |d d df   }| j                  |      }| j                  |      }|S )Nr   )r   r   )r+   r   first_token_tensorpooled_outputs       r-   r4   zVivitPooler.forward  s6     +1a40

#566r.   r   r<   s   @r-   r   r     s*    4{ 4
U\\ ell r.   r   c                        e Zd ZU eed<   dZdZdZdZddgZ	dZ
dZdZdZdZeedZd	Z ej(                          fd
       Z xZS )VivitPreTrainedModelr   vivitr/   )videoTr>   r   )r   
attentionsrC   c                     t         |   |       t        |t              r?t	        j
                  |j                         t	        j
                  |j                         yy)zInitialize the weightsN)r    _init_weightsr$   r>   initzeros_rB   rD   )r+   rl   r,   s     r-   r   z"VivitPreTrainedModel._init_weights4  sE     	f%fo.KK(()KK223 /r.   )r5   r6   r7   r   __annotations__base_model_prefixmain_input_nameinput_modalitiessupports_gradient_checkpointing_no_split_modules_supports_sdpa_supports_flash_attn_supports_flex_attn_supports_attention_backend_can_compile_fullgraphr   r   _can_record_outputs_input_embed_layerr9   no_gradr   r;   r<   s   @r-   r   r   !  sx    $O!&*#*L9N"&!#$ ,U]]_4 4r.   r   c                        e Zd Zddedef fdZe ed      e	 	 	 dde	j                  dz  ded	e	j                  dz  d
ee   def
d                     Z xZS )
VivitModelr   add_pooling_layerc                    t         |   |       || _        t        |      | _        t        j                  t        |j                        D cg c]  }t        |       c}      | _
        t        j                  |j                  |j                        | _        |rt        |      nd| _        | j#                          yc c}w )zv
        add_pooling_layer (bool, *optional*, defaults to `True`):
            Whether to add a pooling layer
        r   N)r    r!   r   r>   rI   r   
ModuleListrangenum_hidden_layersr   layersr   r)   r   	layernormr   pooler	post_init)r+   r   r   r   r,   s       r-   r!   zVivitModel.__init__?  s    
 	 )&1mmvG_G_A`$aAZ%7$abf&8&8f>S>ST->k&)D	 %bs   CF)tie_last_hidden_statesNr/   rc   rp   rr   r0   c                    | j                  ||      }t        | j                  ||      }|}| j                  D ]  } |||fi |} | j	                  |      }| j
                  | j                  |      nd}	t        ||	      S )a  
        Examples:

        ```python
        >>> import av
        >>> import numpy as np

        >>> from transformers import VivitImageProcessor, VivitModel
        >>> from huggingface_hub import hf_hub_download

        >>> np.random.seed(0)


        >>> def read_video_pyav(container, indices):
        ...     '''
        ...     Decode the video with PyAV decoder.
        ...     Args:
        ...         container (`av.container.input.InputContainer`): PyAV container.
        ...         indices (`list[int]`): List of frame indices to decode.
        ...     Returns:
        ...         result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3).
        ...     '''
        ...     frames = []
        ...     container.seek(0)
        ...     start_index = indices[0]
        ...     end_index = indices[-1]
        ...     for i, frame in enumerate(container.decode(video=0)):
        ...         if i > end_index:
        ...             break
        ...         if i >= start_index and i in indices:
        ...             frames.append(frame)
        ...     return np.stack([x.to_ndarray(format="rgb24") for x in frames])


        >>> def sample_frame_indices(clip_len, frame_sample_rate, seg_len):
        ...     '''
        ...     Sample a given number of frame indices from the video.
        ...     Args:
        ...         clip_len (`int`): Total number of frames to sample.
        ...         frame_sample_rate (`int`): Sample every n-th frame.
        ...         seg_len (`int`): Maximum allowed index of sample's last frame.
        ...     Returns:
        ...         indices (`list[int]`): List of sampled frame indices
        ...     '''
        ...     converted_len = int(clip_len * frame_sample_rate)
        ...     end_idx = np.random.randint(converted_len, seg_len)
        ...     start_idx = end_idx - converted_len
        ...     indices = np.linspace(start_idx, end_idx, num=clip_len)
        ...     indices = np.clip(indices, start_idx, end_idx - 1).astype(np.int64)
        ...     return indices


        >>> # video clip consists of 300 frames (10 seconds at 30 FPS)
        >>> file_path = hf_hub_download(
        ...     repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset"
        ... )
        >>> container = av.open(file_path)

        >>> # sample 32 frames
        >>> indices = sample_frame_indices(clip_len=32, frame_sample_rate=1, seg_len=container.streams.video[0].frames)
        >>> video = read_video_pyav(container=container, indices=indices)

        >>> image_processor = VivitImageProcessor.from_pretrained("google/vivit-b-16x2-kinetics400")
        >>> model = VivitModel.from_pretrained("google/vivit-b-16x2-kinetics400")

        >>> # prepare video for the model
        >>> inputs = image_processor(list(video), return_tensors="pt")

        >>> # forward pass
        >>> outputs = model(**inputs)
        >>> last_hidden_states = outputs.last_hidden_state
        >>> list(last_hidden_states.shape)
        [1, 3137, 768]
        ```)rc   )r   inputs_embedsrp   N)last_hidden_statepooler_output)rI   r	   r   r   r   r   r   )
r+   r/   rc   rp   rr   embedding_outputr   layersequence_outputr   s
             r-   r4   zVivitModel.forwardM  s    j  ??<Rj?k2;;*)

 )[[ 	KE!-J6JM	K..78<8OO4UY)O[hiir.   )T)NFN)r5   r6   r7   r   rk   r!   r   r   r   r9   FloatTensorr:   r   r   r   r4   r;   r<   s   @r-   r   r   =  s    { t   E2 26)..2	^j''$.^j #'^j t+	^j
 +,^j 
$^j  3  ^jr.   r   a  
        ViViT Transformer model with a video classification head on top (a linear layer on top of the final hidden state of the
    [CLS] token) e.g. for Kinetics-400.

        <Tip>

            Note that it's possible to fine-tune ViT on higher resolution images than the ones it has been trained on, by
            setting `interpolate_pos_encoding` to `True` in the forward of the model. This will interpolate the pre-trained
            position embeddings to the higher resolution.

        </Tip>
    )custom_introc                        e Zd Zdef fdZee	 	 	 d
dej                  dz  dej                  dz  de
dee   def
d	              Z xZS )VivitForVideoClassificationr   c                 .   t         |   |       |j                  | _        t        |d      | _        |j                  dkD  r*t        j                  |j                  |j                        nt        j                         | _	        | j                          y )NF)r   r   )r    r!   
num_labelsr   r   r   r   r)   Identity
classifierr   r   s     r-   r!   z$VivitForVideoClassification.__init__  ss      ++%@
 OUN_N_bcNc"))F$6$68I8IJikititiv 	r.   Nr/   labelsrc   rr   r0   c                     | j                   |fd|i|}|j                  }| j                  |dddddf         }d}| | j                  ||| j                  fi |}t        |||j                  |j                        S )a  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

        Examples:

        ```python
        >>> import av
        >>> import numpy as np
        >>> import torch

        >>> from transformers import VivitImageProcessor, VivitForVideoClassification
        >>> from huggingface_hub import hf_hub_download

        >>> np.random.seed(0)


        >>> def read_video_pyav(container, indices):
        ...     '''
        ...     Decode the video with PyAV decoder.
        ...     Args:
        ...         container (`av.container.input.InputContainer`): PyAV container.
        ...         indices (`list[int]`): List of frame indices to decode.
        ...     Returns:
        ...         result (np.ndarray): np array of decoded frames of shape (num_frames, height, width, 3).
        ...     '''
        ...     frames = []
        ...     container.seek(0)
        ...     start_index = indices[0]
        ...     end_index = indices[-1]
        ...     for i, frame in enumerate(container.decode(video=0)):
        ...         if i > end_index:
        ...             break
        ...         if i >= start_index and i in indices:
        ...             frames.append(frame)
        ...     return np.stack([x.to_ndarray(format="rgb24") for x in frames])


        >>> def sample_frame_indices(clip_len, frame_sample_rate, seg_len):
        ...     '''
        ...     Sample a given number of frame indices from the video.
        ...     Args:
        ...         clip_len (`int`): Total number of frames to sample.
        ...         frame_sample_rate (`int`): Sample every n-th frame.
        ...         seg_len (`int`): Maximum allowed index of sample's last frame.
        ...     Returns:
        ...         indices (`list[int]`): List of sampled frame indices
        ...     '''
        ...     converted_len = int(clip_len * frame_sample_rate)
        ...     end_idx = np.random.randint(converted_len, seg_len)
        ...     start_idx = end_idx - converted_len
        ...     indices = np.linspace(start_idx, end_idx, num=clip_len)
        ...     indices = np.clip(indices, start_idx, end_idx - 1).astype(np.int64)
        ...     return indices


        >>> # video clip consists of 300 frames (10 seconds at 30 FPS)
        >>> file_path = hf_hub_download(
        ...     repo_id="nielsr/video-demo", filename="eating_spaghetti.mp4", repo_type="dataset"
        ... )
        >>> container = av.open(file_path)

        >>> # sample 32 frames
        >>> indices = sample_frame_indices(clip_len=32, frame_sample_rate=4, seg_len=container.streams.video[0].frames)
        >>> video = read_video_pyav(container=container, indices=indices)

        >>> image_processor = VivitImageProcessor.from_pretrained("google/vivit-b-16x2-kinetics400")
        >>> model = VivitForVideoClassification.from_pretrained("google/vivit-b-16x2-kinetics400")

        >>> inputs = image_processor(list(video), return_tensors="pt")

        >>> with torch.no_grad():
        ...     outputs = model(**inputs)
        ...     logits = outputs.logits

        >>> # model predicts one of the 400 Kinetics-400 classes
        >>> predicted_label = logits.argmax(-1).item()
        >>> print(model.config.id2label[predicted_label])
        LABEL_116
        ```rc   Nr   )losslogitsr   r   )r   r   r   loss_functionr   r   r   r   )	r+   r/   r   rc   rr   outputsr   r   r   s	            r-   r4   z#VivitForVideoClassification.forward  s    x $.4::$
3K$
OU$
 "33Aq!9:%4%%ffdkkLVLD$!//))	
 	
r.   )NNF)r5   r6   r7   r   r!   r   r   r9   r   
LongTensorrk   r   r   r   r4   r;   r<   s   @r-   r   r     s    
{ 
  26*.).	i
''$.i
   4'i
 #'	i

 +,i
 
i
  i
r.   r   )r   r   r   )Nr   )0collections.abcr   r   r9   r    r   r   activationsr   masking_utilsr	   modeling_layersr
   modeling_outputsr   r   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   utils.genericr   r   utils.output_capturingr   configuration_vivitr   Moduler   r>   r:   floatr   r   r   r   r   r   r   r   __all__ r.   r-   <module>r	     s  * /   & ! 6 9 b b F & B B I 5 ,HRYY H>Lbii Lj !%II%<<% 
% <<	%
 LL4'% T\% % '(%8.)RYY .)bryy  + @"))  4? 4 46 pj% pj pjf x
"6 x
x
v Pr.   