
    ^j                        d Z ddlZddlmZ ddlZddlmZ ddlmZ ddl	m
Z
 ddlmZ dd	lmZ dd
lmZ ddlmZmZmZ ddlmZ ddlmZmZ ddlmZ  ej8                  e      Zee G d de                    Z G d dej@                        Z! G d dej@                        Z" G d dej@                        Z# G d dej@                        Z$ G d dej@                        Z% G d dej@                        Z& G d dej@                        Z' G d  d!e      Z( G d" d#ej@                        Z) G d$ d%ej@                        Z*e G d& d'e             Z+ G d( d)ej@                        Z, G d* d+ej@                        Z-e,e-d,Z. ed-.       G d/ d0e+             Z/ G d1 d2ej@                        Z0 ed3.       G d4 d5e+             Z1g d6Z2y)7zPyTorch TVP Model    N)	dataclass)nn   )initialization)ACT2FN)load_backbone)create_bidirectional_mask)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPoolingModelOutput)PreTrainedModel)auto_docstringlogging   )	TvpConfigc                       e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)TvpVideoGroundingOutputa  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
        Temporal-Distance IoU loss for video grounding.
    logits (`torch.FloatTensor` of shape `(batch_size, 2)`):
        Contains start_time/duration and end_time/duration. It is the time slot of the videos corresponding to the
        input texts.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.
    Nlosslogits.hidden_states
attentions)__name__
__module____qualname____doc__r   torchFloatTensor__annotations__r   r   tupler        o/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/tvp/modeling_tvp.pyr   r   $   sq    	 &*D%

d
")'+FE$+:>M5**C/047>7;Je'',-4;r"   r   c                   :     e Zd ZdZ fdZd Zd Zd Zd Z xZ	S )TvpLossa~  
    This class computes the losses for `TvpForVideoGrounding`. The process happens in two steps: 1) we compute
    hungarian assignment between ground truth boxes and the outputs of the model 2) we supervise each pair of matched
    ground-truth / prediction (supervise class and box).

    Args:
        losses (`list[str]`):
            List of all the losses to be applied.
    c                     t         |           | j                  | j                  | j                  d| _        |D ]  }|| j
                  vst        d| d       || _        y )NioudistancedurationzLoss z not supported)super__init__loss_iouloss_distanceloss_durationloss_map
ValueErrorlosses)selfr2   r   	__class__s      r#   r,   zTvpLoss.__init__C   sj    ==****

  	?D4==( 5n!=>>	? r"   c                     t        j                  ||      t        j                  ||      z
  }t        j                  ||      t        j                  ||      z
  }d|j                  d      |z  z
  }|S )z6
        Measure the intersection over union.
        r   r   min)r   r7   maxclamp)	r3   
start_timeend_timecandidates_start_timecandidates_end_timer*   interunionr(   s	            r#   r-   zTvpLoss.loss_iouP   si     		-x8599EZ\f;gg		-x8599EZ\f;gg%++!+$u,,
r"   c                 P   t        j                  t        j                  ||      d      }t        j                  t        j                  ||      d      }t        j                  t        j                  ||      t        j                  ||      z
  |      j                  d      }|S )z5
        Measure the distance of mid points.
        g       @g?r6   )r   divaddr8   r7   r9   )	r3   r:   r;   r<   r=   r*   mid_candidatesmid_groundtruthdistance_diffs	            r#   r.   zTvpLoss.loss_distanceZ   s     599-BDW#XZ]^))EIIj($CSI		IIno6>Sb9ccem

%C%. 	 r"   c                     t        j                  ||      }t        j                  ||      }t        j                  t        j                  t        j                  ||      |            }|j	                  d      }|S )z5
        Measure the difference of duration.
        g?r6   )r   subsquarerA   r9   )	r3   r:   r;   r<   r=   r*   duration_candidatesduration_groundtruthduration_diffs	            r#   r/   zTvpLoss.loss_durationf   sh     $ii(;=RS$yy:>UYYuyy9LNb/cem%no%+++4r"   c                    |\  }}}t        j                  ||      }|dddf   j                         |dddf   j                         }}i }	| j                  D ],  }
|	j	                  |
 | j
                  |
   |||||      i       . |	S )am  
        This performs the loss computation.

        Args:
            logits (`torch.FloatTensor`):
                The output logits of head module.
            labels (`list[torch.FloatTensor]`):
                List of tensors ([start, end, duration]), which contains start time, end time of the video corresponding to the text, and also the duration.
        Nr   r   )r   mulfloatr2   updater0   )r3   r   labelsr*   r:   r;   
candidatesr<   r=   losses_dictr   s              r#   forwardzTvpLoss.forwardq   s     *0&*hYYvx0
5?15E5K5K5MzZ[]^Z^O_OeOeOg2KK 	D*t}}T*:xAVXkmuvw	
 r"   )
r   r   r   r   r,   r-   r.   r/   rS   __classcell__r4   s   @r#   r%   r%   8   s!    
	r"   r%   c                   $     e Zd Z fdZd Z xZS )TvpVisionModelc           	      \   t         |           t        |      | _        |j                  |j                  j
                  d   }nt        | j                  d      rDt        | j                  j                  d      r$| j                  j                  j
                  d   }nbt        | j                  d      rAt        | j                  j                  d      r!| j                  j                  j                  }nt        d      t        j                  ||j                  ddddd	      | _        y )
Nconfighidden_sizeshidden_sizezBackbone config not foundr   r   F)kernel_sizestridepaddinggroupsbias)r+   r,   r   backbonebackbone_configr[   hasattrrZ   r\   r1   r   Conv2dgrid_encoder_conv)r3   rZ   in_channelsr4   s      r#   r,   zTvpVisionModel.__init__   s    %f-!!- 00==bAKT]]H-'$--:N:NP^2_--..;;B?KT]]H-'$--:N:NP]2^--..::K899!#"
r"   c                    |j                   \  }}}}}|j                  ||z  |||      }| j                  |      d   d   }| j                  |      }t        j
                  j                  |dd      }t        j
                  j                  |d      }|j                   dd  \  }	}
}|j                  |||	|
|      }|j                  ddd	d
d      }|S )Nfeature_mapsr      )r]   r^   T)inplacer   r      )	shapeviewrb   rf   r   
functional
max_pool2drelupermute)r3   pixel_values
batch_size
num_framesnum_channelsheightwidthgrid_feat_outputsgridnew_channel
new_height	new_widths               r#   rS   zTvpVisionModel.forward   s    >J>P>P;
Jfe#((j)@,PVX]^ MM,7GJ%%&78}}''!A'F}}!!$!5-1ZZ_*ZyyZj)T||Aq!Q*r"   r   r   r   r,   rS   rT   rU   s   @r#   rW   rW      s    
.r"   rW   c                   ~     e Zd ZdZ fdZdej                  dededej                  fdZdde	fd	Z
dde	fd
Z xZS )TvpVisualInputEmbeddingz;
    Takes input of both image and video (multi-frame)
    c                 r   t         |           t        j                  |j                  |j
                        | _        t        j                  |j                  |j
                        | _        t        j                  |j                  |j
                        | _
        t        j                  d|j
                        | _        t        j                  |j
                  |j                        | _        t        j                  |j                         | _        |j                  | _        |j                  | _	        y )Nr   eps)r+   r,   r   	Embeddingmax_position_embeddingsr\   position_embeddings max_grid_row_position_embeddingsrow_position_embeddings max_grid_col_position_embeddingscol_position_embeddingstoken_type_embeddings	LayerNormlayer_norm_eps
layer_normDropouthidden_dropout_probdropoutr3   rZ   r4   s     r#   r,   z TvpVisualInputEmbedding.__init__   s    #%<<0N0NPVPbPb#c ')||F4[4[]c]o]o'p$')||F4[4[]c]o]o'p$%'\\!V5G5G%H",,v'9'9v?T?TUzz&"<"<=060W0W-060W0W-r"   	embeddingrx   ry   returnc                     dx}}|| j                   kD  r|| j                   z  }|| j                  kD  r|| j                  z  }|j                  dddd      }t        j                  j                  |||fdd      }|j                  dddd      }|S )z
        This method allows to interpolate the pre-trained pad weights , to be able to use the model on collection of high
        resolution images (high resolution videos).

        r   r   r   rj   bicubicFscale_factormodealign_corners)r   r   rs   r   rp   interpolate)r3   r   rx   ry   h0w0s         r#   interpolate_pos_encodingz0TvpVisualInputEmbedding.interpolate_pos_encoding   s     RD999$???B4888>>>B%%aAq1	MM--b	 . 
	 %%aAq1	r"   r   c                    |j                   \  }}}}t        | j                  |      }t        j                  |t        j
                  |j                        }| j                  |      }	dt        |j                         dz
  z  |d|fz   }
 |	j                  |
 }	t        | j                  |      }t        j                  |t        j
                  |j                        }| j                  |      }|d||f} |j                  | }|	|z   }|r6|| j                  kD  s|| j                  kD  r|| j                  |||      z   }|S ||z   }|S )af  
        Args:
            grid: (batch_size, height, width, hidden_dim)
            interpolate_pos_encoding: (`bool`, *optional*, defaults to `False`):
                Whether to interpolate the pre-trained position encodings.
        Returns:
            grid + col_position_embeddings.view(*col_shape): (batch_size, *, height, width, hidden_dim)
        dtypedevice)r   r   r   )rn   r7   r   r   arangelongr   r   lenro   r   r   r   )r3   r{   r   ru   rx   ry   
hidden_dim
row_heightrow_position_idsr   	row_shape	row_widthcol_position_idsr   	col_shapepositional_embeddingss                   r#   add_2d_positional_embeddingsz4TvpVisualInputEmbedding.add_2d_positional_embeddings   sQ    15

-
FE: >>G
 <<
%**T[[Y"&">">?O"PC

Oa/0J:3NN	">"9">">	"J ==uE	 <<	DKKX"&">">?O"PIz:	">"9">">	"J 7:Q Q $T:::edFkFk>k$778MvW\]]D  //Dr"   c                    |j                   \  }}}}}|j                  d      }| j                  ||      }|j                  |d|      }|j                   dd }	|j                  }
t        j                  |	t
        j                  |
      }| j                  |      }||z   }| j                  |      }| j                  |      }|S )a  
        Args:
            grid: Array of shape (batch_size, num_frames, height, width, num_channels).
                It contains processed frames extracted from videos, and is generated by Tvp image preprocessor. Note,
                num_frames can be 1
            interpolate_pos_encoding: (bool, *optional*, defaults to `False`):
                Whether to interpolate the pre-trained position encodings.

        Returns:
            embeddings: The embedding of grid with size (batch_size, height*width, num_channels)

        r   r   rY   Nr   )rn   meanr   ro   r   r   zerosr   r   r   r   )r3   r{   r   ru   rv   rx   ry   rw   visual_tokensvisual_tokens_shaper   token_type_idsr   
embeddingss                 r#   rS   zTvpVisualInputEmbedding.forward  s     ?Cjj;
J|yy|00Ph0i		*b,?+11#26%% %8

SYZ $ : :> J"%::
__Z0
\\*-
r"   F)r   r   r   r   r,   r   Tensorintr   boolr   rS   rT   rU   s   @r#   r   r      sT    
X%,,  TW \a\h\h .'4 'Rd r"   r   c                   *     e Zd ZdZ fdZddZ xZS )TvpTextInputEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                    t         |           t        j                  |j                  |j
                  |j                        | _        t        j                  |j                  |j
                        | _	        t        j                  |j                  |j
                        | _        t        j                  |j
                  |j                        | _        t        j                  |j                         | _        y )N)padding_idxr   )r+   r,   r   r   
vocab_sizer\   pad_token_idword_embeddingsr   r   type_vocab_sizer   r   r   r   r   r   r   r   s     r#   r,   zTvpTextInputEmbeddings.__init__$  s    !||F,=,=v?Q?Q_e_r_rs#%<<0N0NPVPbPb#c %'\\&2H2H&J\J\%]",,v'9'9v?T?TUzz&"<"<=r"   c                 .   ||j                         }n|j                         d d }|d   }||j                  n|j                  }|Ft        j                  |t        j                  |      }|j                  d      j                  |      }|&t        j                  |t        j                  |      }|| j                  |      }| j                  |      }| j                  |      }	||z   |	z   }
| j                  |
      }
| j                  |
      }
|
S )NrY   r   r   r   )sizer   r   r   r   	unsqueezeexpandr   r   r   r   r   r   )r3   	input_idsr   position_idsinputs_embedsinput_shape
seq_lengthr   r   r   r   s              r#   rS   zTvpTextInputEmbeddings.forward,  s    #..*K',,.s3K ^
%.%:!!@T@T <<
%**VTL'11!4;;KHL!"[[EJJvVN  00;M"66|D $ : :> J"%88;PP
__Z0
\\*-
r"   NNNNr   r   r   r   r,   rS   rT   rU   s   @r#   r   r   !  s    Q>r"   r   c                   ^     e Zd Z fdZdej
                  dedefdZ	 	 d	dedz  fdZ	 xZ
S )
TvpAttentionc                    t         |           |j                  |j                  z  dk7  r1t	        |d      s%t        d|j                   d|j                         |j                  | _        t        |j                  |j                  z        | _        | j                  | j                  z  | _        t        j                  |j                  | j                        | _        t        j                  |j                  | j                        | _        t        j                  |j                  | j                        | _        t        j                  |j                        | _        t        j                  |j                  |j                        | _        t        j$                  |j                  |j&                        | _        t        j                  |j*                        | _        y )Nr   embedding_sizezThe hidden size z4 is not a multiple of the number of attention heads r   )r+   r,   r\   num_attention_headsrd   r1   r   attention_head_sizeall_head_sizer   Linearquerykeyvaluer   attention_probs_dropout_probattn_dropoutdenser   r   r   r   r   r   s     r#   r,   zTvpAttention.__init__F  sx    : ::a?PVXhHi"6#5#5"66jkq  lF  lF  kG  H  $*#=#= #&v'9'9F<V<V'V#W !558P8PPYYv1143E3EF
99V//1C1CDYYv1143E3EF
JJv'J'JKYYv1163E3EF
,,v'9'9v?T?TUzz&"<"<=r"   tensorsequence_lengthru   c                     |j                  ||| j                  | j                        j                  dd      j	                         S )Nr   rj   )ro   r   r   	transpose
contiguous)r3   r   r   ru   s       r#   _reshapezTvpAttention._reshapeZ  s7    KK
OT5M5MtOgOghYq!_Z\	
r"   Noutput_attentionsc                 ,   |j                   d d \  }}| j                  |      }| j                  |      }| j                  |      }| j	                  |||      }	| j	                  |||      }
| j	                  |||      }t        j                  |	|
j                  dd            }|t        j                  | j                        z  }|||z   }t        j                  j                  |d      }| j                  |      }t        j                  ||      }|j                  dd      j                         }|j!                  ||| j"                        }| j%                  |      }| j'                  |      }| j)                  ||z         }|r||f}|S |f}|S )Nrj   rY   dimr   )rn   r   r   r   r   r   matmulr   mathsqrtr   r   rp   softmaxr   r   reshaper   r   r   r   )r3   r   attention_maskr   ru   r   mixed_query_layermixed_key_layermixed_value_layerquery_layer	key_layervalue_layerattention_scoresattention_probsattn_outputoutputss                   r#   rS   zTvpAttention.forwarda  s    '4&9&9"1&=#
O JJ}5((=1 JJ}5mm$5
SMM/?JO	mm$5
S !<<Y5H5HR5PQ+dii8P8P.QQ%/.@ --//0@b/I ++O<ll?K@!++Aq1<<>!))*otGYGYZjj-ll;/ookM&AB4E;0 MX>r"   NN)r   r   r   r,   r   r   r   r   r   rS   rT   rU   s   @r#   r   r   E  sA    >(
u|| 
c 
s 
 )-	&  $;	&r"   r   c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )TvpIntermediatec                    t         |           t        j                  |j                  |j
                        | _        t        |j                  t              rt        |j                     | _        y |j                  | _        y N)r+   r,   r   r   r\   intermediate_sizer   
isinstance
hidden_actstrr   intermediate_act_fnr   s     r#   r,   zTvpIntermediate.__init__  s]    YYv1163K3KL
f''-'-f.?.?'@D$'-'8'8D$r"   r   r   c                 J    | j                  |      }| j                  |      }|S r   )r   r  )r3   r   s     r#   rS   zTvpIntermediate.forward  s&    

=100?r"   r   r   r   r,   r   r   rS   rT   rU   s   @r#   r   r     s#    9U\\ ell r"   r   c                   n     e Zd Z fdZdej
                  dej
                  dej
                  fdZ xZS )TvpOutputLayerc                 (   t         |           t        j                  |j                  |j
                        | _        t        j                  |j
                  |j                        | _	        t        j                  |j                        | _        y )Nr   )r+   r,   r   r   r   r\   r   r   r   r   r   r   r   r   s     r#   r,   zTvpOutputLayer.__init__  s`    YYv779K9KL
,,v'9'9v?T?TUzz&"<"<=r"   r   input_tensorr   c                 r    | j                  |      }| j                  |      }| j                  ||z         }|S r   )r   r   r   )r3   r   r  s      r#   rS   zTvpOutputLayer.forward  s7    

=1]3(DEr"   r  rU   s   @r#   r  r    s1    >U\\  RWR^R^ r"   r  c                   6     e Zd Z fdZ	 	 ddedz  fdZ xZS )TvpEncodeLayerc                     t         |           t        |      | _        t	        |      | _        t        |      | _        y r   )r+   r,   r   	attentionr   intermediater  outputr   s     r#   r,   zTvpEncodeLayer.__init__  s3    %f-+F3$V,r"   Nr   c                     | j                  |||      }|d   }|dd  }| j                  |      }| j                  ||      }|f|z   }|S )N)r   r   r   )r  r  r  )	r3   r   r   r   self_attention_outputsattention_outputr   intermediate_outputlayer_outputs	            r#   rS   zTvpEncodeLayer.forward  so     "&/ "0 "

 2!4(,"//0@A{{#68HI/G+r"   r   )r   r   r   r,   r   rS   rT   rU   s   @r#   r
  r
    s#    - )-	  $;	r"   r
  c            
       X     e Zd Z fdZ	 	 	 	 ddedz  dedz  dedz  deez  fdZ xZS )	
TvpEncoderc                     t         |           || _        t        j                  t        |j                        D cg c]  }t        |       c}      | _        d| _	        y c c}w )NF)
r+   r,   rZ   r   
ModuleListrangenum_hidden_layersr
  layergradient_checkpointing)r3   rZ   _r4   s      r#   r,   zTvpEncoder.__init__  sN    ]]E&JbJbDc#dqN6$:#de
&+# $es   A#Nr   output_hidden_statesreturn_dictr   c                    ||n| j                   j                  }||n| j                   j                  }||n| j                   j                  }d}d}t	        | j
                        D ](  \  }}	|r||fz   } |	|||      }
|
d   }|s ||
d   fz   }* |r||fz   }|s|f}|r||fz   }|r||fz   }|S t        ||r|nd |r|      S d       S )Nr!   r   r   )last_hidden_stater   r   )rZ   r  r   r  	enumerater  r   )r3   r   r   r   r  r  all_hidden_statesall_attentionsilayer_modulelayer_outputsr   s               r#   rS   zTvpEncoder.forward  s+    &1%<k$++BYBY1B1N-TXT_T_TqTq$8$D $++JjJj 	 (4 	FOA|#$58H$H!(HYZM)!,M !/=3C2E!E	F   1]4D D$&G#!%6$88 !^$55N+/C+):~
 	
 AE
 	
r"   r   )	r   r   r   r,   r   r    r   rS   rT   rU   s   @r#   r  r    sT    , )-,0#'*
  $;	*

 #Tk*
 D[*
 
	 *
r"   r  c                   V     e Zd Z fdZdej
                  dej
                  fdZ xZS )	TvpPoolerc                     t         |           t        j                  |j                  |j                        | _        t        j                         | _        y r   )r+   r,   r   r   r\   r   Tanh
activationr   s     r#   r,   zTvpPooler.__init__  s9    YYv1163E3EF
'')r"   r   r   c                 \    |d d df   }| j                  |      }| j                  |      }|S )Nr   )r   r+  )r3   r   first_token_tensorpooled_outputs       r#   rS   zTvpPooler.forward  s6     +1a40

#566r"   r  rU   s   @r#   r(  r(    s#    $
U\\ ell r"   r(  c                   x     e Zd ZU eed<   dZdZdZ ej                         de
j                  f fd       Z xZS )TvpPreTrainedModelrZ   model)videotextTmodulec                    t         |   |       t        |t        j                        rOt        j                  |j                  dd       |j                  Pt        j                  |j                  d       n/t        |t              rt        j                  |j                         t        |d      rt        j                  |j                         t        |d      rt        j                  |j                         t        |d      rt        j                  |j                          t        |d	      r t        j                  |j"                         yy)
zInitialize the weightsfan_outrr   )r   nonlinearityNr   pad_uppad_downpad_left	pad_right)r+   _init_weightsr   r   re   initkaiming_normal_weightra   	constant_TvpModelnormal_text_promptrd   r8  r9  r:  r;  )r3   r4  r4   s     r#   r<  z TvpPreTrainedModel._init_weights  s     	f%fbii(  YVT{{&v{{A.)LL++,68$LL'6:&LL)6:&LL)6;'LL))* (r"   )r   r   r   r   r   base_model_prefixinput_modalitiessupports_gradient_checkpointingr   no_gradr   Moduler<  rT   rU   s   @r#   r0  r0    s@    (&*#U]]_+BII + +r"   r0  c                   (     e Zd ZdZ fdZd Z xZS )TvpFrameDownPadPrompterz>
    Pad frames extracted from videos only at the bottom.
    c           	      |   |j                   dvrt        d      t        |           |j                  | _        |j
                  | _        |j                  | _        |j                   | _         t        j                  t        j                  d|j
                  d|j                  |j                  g            | _        y )NrB   replaceremove9`visual_prompter_apply` must be in (add, replace, remove)r   r   )visual_prompter_applyr1   r+   r,   visual_prompt_size	frame_nummax_img_sizer   	Parameterr   randnr9  r   s     r#   r,   z TvpFrameDownPadPrompter.__init__'  s    ''/KKXYY"(";";))"//%+%A%A"KKF,,a1J1JFL_L_`a
r"   c                    | j                   dk7  rst        j                  | j                  | j                  g|j                  |j
                        }d|| j                  | j                  z
  | j                  d d f<   ||z  }| j                   dk7  rt        j                  |j                  d   |j                  d   d| j                  | j                  g|j
                        }| j                  | j                  z
  }| j                  |d d d d d d || j                  d d f<   ||j                  |j                        z  }|S )	NrB   r   g        rN  r   r   r   r   )rP  r   onesrS  r   r   rQ  r   rn   r9  to)r3   rt   visual_prompt_maskpromptstart_points        r#   rS   zTvpFrameDownPadPrompter.forward5  s1   %%.!&""D$5$56l>P>PYeYlYl" fit0043J3JJTM^M^^`aab..L%%1[[##A&(:(:1(=q$BSBSUYUfUfg#**F ++d.E.EEKBF--F1aK$*;*;;Q>?FIIl&8&899Lr"   r   rU   s   @r#   rJ  rJ  "  s    
r"   rJ  c                   p     e Zd ZdZ fdZdej                  dededej                  fdZd
de	fd	Z
 xZS )TvpFramePadPrompterz?
    Pad frames extracted from videos in the surroundings.
    c           
         |j                   dvrt        d      t        |           |j                  | _        |j
                  | _        |j                   | _         |j
                  |j                  dz  z
  | _        t        j                  t        j                  d|j                  d|j                  |j
                  g            | _        t        j                  t        j                  d|j                  d|j                  |j
                  g            | _        t        j                  t        j                  d|j                  d|j
                  |j                  dz  z
  |j                  g            | _        t        j                  t        j                  d|j                  d|j
                  |j                  dz  z
  |j                  g            | _        y )NrL  rO  rj   r   r   )rP  r1   r+   r,   rv   rS  rQ  	base_sizer   rT  r   rU  r8  r9  r:  r;  r   s     r#   r,   zTvpFramePadPrompter.__init__L  s   ''/KKXYY ++"//%+%A%A",,v/H/H1/LLllKKF--q&2K2KVM`M`ab
 KKF--q&2K2KVM`M`ab
 KK%%''&*C*Ca*GG--

 KK%%''&*C*Ca*GG--

r"   r[  rx   ry   r   c                    || j                   z  || j                   z  }}|j                  \  }}}}	}
|j                  ||z  ||	|
      }t        j                  j                  |||fdd      }|j                  |||||      }|S )z
        This method allows to interpolate the pre-trained pad weights, to be able to use the model on collection of high
        resolution images (high resolution videos).

        r   Fr   )rS  rn   r   r   rp   r   )r3   r[  rx   ry   r   r   batchrv   channelsprompt_heightprompt_widths              r#   interpolate_pad_encodingz,TvpFramePadPrompter.interpolate_pad_encodingr  s     $+++UT5F5F-FBCI<<@z8]L 
 2Hm\Z**b	 + 
 z8VUKr"   rf  c                 Z   |r|j                   d   |j                   d   fn| j                  | j                  f\  }}| j                  dvrt        d| j                         | j                  dv r3t	        j
                  ||g|j                  |j                        }||z  }| j                  dv rt	        j                  d| j                  d	| j                  | j                  |j                  
      }t	        j                  | j                  || j                  gd      }t	        j                  | j                  || j                  gd	      }t	        j                  |j!                  d      |gz        }|r| j#                  |||      }||j%                  |j                        z   }|S )Nr   rY   )rB   rN  rM  z$Invalid visual_prompter_apply value )rM  rN  r   )rM  rB   r   r   rW  rm   r   r   )rn   rS  rP  r1   r   rX  r   r   r   rv   r`  catr:  r;  r8  r9  r   rf  rY  )r3   rt   rf  rx   ry   rZ  baser[  s           r#   rS   zTvpFramePadPrompter.forward  s{    ( #\%7%7%;<##T%6%67 	
 %%-IICDD^D^C_`aa%%)>>!&VUO<CUCU^j^q^q!r..L%%);;;;q$//1dnndnn]i]p]pqDYYtT^^D!LFYYVT]]CKFYY|003vh>?F'66vvuM'&))L4F4F*GGLr"   r   )r   r   r   r   r,   r   r   r   rf  r   rS   rT   rU   s   @r#   r^  r^  G  sG    $
Lu|| S QT Y^YeYe 0d r"   r^  )framedownpadframepadzw
    The bare Tvp Model transformer outputting BaseModelOutputWithPooling object without any specific head on top.
    )custom_introc                        e Zd Z fdZd Zd Ze	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de
dz  d	e
dz  d
e
dz  de
deez  fd       Z xZS )rA  c                 "   t         |   |       || _        t        |      | _        t        |      | _        t        |      | _        t        |      | _
        t        |      | _        t        j                  t        j                   dd|j"                  g            | _        t        j&                  |j(                        | _        |j,                  t.        vrt1        d      t/        |j,                     |      | _        | j5                          y )Nr   
   z:`visual_prompter_type` must be in (framedownpad, framepad))r+   r,   rZ   rW   vision_modelr   r   r   visual_embeddingsr  encoderr(  poolerr   rT  r   rU  r\   rC  r   r   r   visual_prompter_typeTVP_PROMPTER_CLASSES_MAPPINGr1   visual_prompter	post_initr   s     r#   r,   zTvpModel.__init__  s     *6208!8!@!&)'<<QF<N<N4O(PQzz&"<"<=&&.JJYZZ;F<W<WXY_`r"   c                 .    | j                   j                  S r   r   r   )r3   s    r#   get_input_embeddingszTvpModel.get_input_embeddings  s    ...r"   c                 &    || j                   _        y r   ry  )r3   r   s     r#   set_input_embeddingszTvpModel.set_input_embeddings  s    */'r"   Nr   rt   r   r   r  r  r   r   c                    ||n| j                   j                  }| j                  | j                  ||            }| j	                  |      }	| j                  ||      }
||j                  |
j                  dd       }t        j                  |j                  d   d      j                  |j                  |j                        }t        j                  |||gd	
      }| j                  j                  |	j                  d   d	d	      }t        j                  ||	|
gd
      }t!        | j                   ||      }| j#                  |||||      }|r|j$                  n|d   }| j'                  |      }| j)                  |      }| j)                  |      }|s
||f|dd z   S t+        |||j,                  |j.                        S )a  
        Examples:
        ```python
        >>> import torch
        >>> from transformers import AutoConfig, AutoTokenizer, TvpModel

        >>> model = TvpModel.from_pretrained("Jiqing/tiny-random-tvp")

        >>> tokenizer = AutoTokenizer.from_pretrained("Jiqing/tiny-random-tvp")

        >>> pixel_values = torch.rand(1, 1, 3, 448, 448)
        >>> text_inputs = tokenizer("This is an example input", return_tensors="pt")
        >>> output = model(text_inputs.input_ids, pixel_values, text_inputs.attention_mask)
        ```N)rf  )r   r   rj   r   ro  )r   r   rY   r   r   )rZ   r   r   )r   r   r  r  )r   pooler_outputr   r   )rZ   r  rp  rv  r   rq  new_onesrn   r   rX  rY  r   r   rh  rC  r   r	   rr  r   rs  r   r   r   r   )r3   r   rt   r   r   r  r  r   kwargstext_embedding_outputvisual_embedding_outputvisual_attention_maskpt_maskrC  embedding_outputencoder_outputsr   r.  s                     r#   rS   zTvpModel.forward  s   4 &1%<k$++BYBY((  H` a
 !%) D"&"8"83K #9 #
 %$2$;$;<S<Y<YZ\[\<]$^!jj!5!5a!8"=@@%,,N4H4H A G #YYAV'W]_`N&&--.C.I.I!.LbRTU 99k3HJa%bhij2;;*)
 ,,)/!5# ' 
 BMO==RabcRd$56 LL):;]3%}58KKK)/')77&11	
 	
r"   )NNNNNNF)r   r   r   r,   rz  r|  r   r   
LongTensorr   r   r    r   rS   rT   rU   s   @r#   rA  rA    s     /0  .21526)-,0#').I
##d*I
 ''$.I
 ((4/	I

  $;I
 #TkI
 D[I
 #'I
 
+	+I
 I
r"   rA  c                   $     e Zd Z fdZd Z xZS )TvpVideoGroundingHeadc                 :   t         |           t        j                  |j                  |j                  dz        | _        t        j                  |j                  dz  d      | _        t        j                         | _        t        j                         | _
        y )Nrj   )r+   r,   r   r   r\   layer_0layer_1ReLUactivation_0Sigmoidactivation_1r   s     r#   r,   zTvpVideoGroundingHead.__init__  sj    yy!3!3V5G5G!5KLyy!3!3a!7;GGIJJLr"   c                     | j                  | j                  |            }| j                  | j                  |            }|S r   )r  r  r  r  )r3   r~  r   s      r#   rS   zTvpVideoGroundingHead.forward  s9    ""4<<#>?""4<<#78r"   r   rU   s   @r#   r  r    s    )r"   r  zb
    Tvp Model with a video grounding head on top computing IoU, distance, and duration loss.
    c                        e Zd Z fdZe	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  deej                     dz  de
dz  de
dz  d	e
dz  d
e
deez  fd       Z xZS )TvpForVideoGroundingc                     t         |   |       || _        t        |      | _        t        |      | _        | j                          y r   )r+   r,   rZ   rA  r1  r  video_grounding_headrw  r   s     r#   r,   zTvpForVideoGrounding.__init__$  s:     f%
$9&$A!r"   Nr   rt   r   rP   r   r  r  r   r   c	           	         ||n| j                   j                  }| j                  |||||||      }
|
d   }| j                  |      }d}|pt	        g d      }|j                  | j                          |||      }|d   | j                   j                  |d   z  z   | j                   j                  |d   z  z   }|s|f|
dd z   }
||f|
z   }
|
S t        |||
j                  |
j                  	      S )
a  
        labels (`torch.FloatTensor` of shape `(batch_size, 3)`, *optional*):
            The labels contains duration, start time, and end time of the video corresponding to the text.

        Examples:
        ```python
        >>> import torch
        >>> from transformers import AutoConfig, AutoTokenizer, TvpForVideoGrounding

        >>> model = TvpForVideoGrounding.from_pretrained("Jiqing/tiny-random-tvp")

        >>> tokenizer = AutoTokenizer.from_pretrained("Jiqing/tiny-random-tvp")

        >>> pixel_values = torch.rand(1, 1, 3, 448, 448)
        >>> text_inputs = tokenizer("This is an example input", return_tensors="pt")
        >>> output = model(text_inputs.input_ids, pixel_values, text_inputs.attention_mask)
        ```N)r   r  r  r   r   r'   r(   r)   r*   rj   )r   r   r   r   )rZ   r  r1  r  r%   rY  r   distance_loss_weightduration_loss_weightr   r   r   )r3   r   rt   r   rP   r   r  r  r   r  r   r~  r   r   	criterion	loss_dicts                   r#   rS   zTvpForVideoGrounding.forward,  s)   < &1%<k$++BYBY**/!5#%=  
  
**=9 ?@ILL%!&&1I% ++22Yz5JJK++22Yz5JJK 
 i'!"+-G'G+N&!//))	
 	
r"   )NNNNNNNF)r   r   r   r,   r   r   r  r   r    r   r   r   rS   rT   rU   s   @r#   r  r    s      .21526-1)-,0#').?
##d*?
 ''$.?
 ((4/	?

 ell#d*?
  $;?
 #Tk?
 D[?
 #'?
 
(	(?
 ?
r"   r  )rA  r0  r  )3r   r   dataclassesr   r   r    r   r=  activationsr   backbone_utilsr   masking_utilsr	   modeling_layersr
   modeling_outputsr   r   r   modeling_utilsr   utilsr   r   configuration_tvpr   
get_loggerr   loggerr   rH  r%   rW   r   r   r   r   r  r
  r  r(  r0  rJ  r^  ru  rA  r  r  __all__r!   r"   r#   <module>r     s     !   & ! + 6 9 X X - , ( 
		H	% 
<k <  <$Mbii M`%RYY %Pnbii nb!RYY !HB299 BLbii RYY / 41
 1
j		  + + +6"bii "JW")) Wv ,#   
a
! a

a
HBII  
I
- I

I
X Er"   