Ë
    ÜÓ^j–’  ã                   ó’  — d Z ddlZddlmZ ddlZddlmZ ddlmZ ddl	m
Z
 ddlmZ dd	lmZ dd
lmZ ddlmZmZmZ ddlmZ ddlmZmZ ddlmZ  ej8                  e«      Zee G d„ de«      «       «       Z G d„ dej@                  «      Z! G d„ dej@                  «      Z" G d„ dej@                  «      Z# G d„ dej@                  «      Z$ G d„ dej@                  «      Z% G d„ dej@                  «      Z& G d„ dej@                  «      Z' G d „ d!e«      Z( G d"„ d#ej@                  «      Z) G d$„ d%ej@                  «      Z*e G d&„ d'e«      «       Z+ G d(„ d)ej@                  «      Z, G d*„ d+ej@                  «      Z-e,e-d,œZ. ed-¬.«       G d/„ d0e+«      «       Z/ G d1„ d2ej@                  «      Z0 ed3¬.«       G d4„ d5e+«      «       Z1g d6¢Z2y)7zPyTorch TVP Modelé    N)Ú	dataclass)Únné   )Úinitialization)ÚACT2FN)Úload_backbone)Úcreate_bidirectional_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚBaseModelOutputWithPoolingÚModelOutput)ÚPreTrainedModel)Úauto_docstringÚloggingé   )Ú	TvpConfigc                   óÆ   — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	e
ej                  df   dz  ed<   dZe
ej                  df   dz  ed<   y)ÚTvpVideoGroundingOutputa€  
    loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`):
        Temporal-Distance IoU loss for video grounding.
    logits (`torch.FloatTensor` of shape `(batch_size, 2)`):
        Contains start_time/duration and end_time/duration. It is the time slot of the videos corresponding to the
        input texts.
    attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`.
    NÚlossÚlogits.Úhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚFloatTensorÚ__annotations__r   r   Útupler   © ó    úo/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/tvp/modeling_tvp.pyr   r   $   sq   … ñ	ð &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø'+€FˆE×Ñ Ñ$Ó+Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>Ø7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ô;r"   r   c                   ó:   ‡ — e Zd ZdZˆ fd„Zd„ Zd„ Zd„ Zd„ Zˆ xZ	S )ÚTvpLossa~  
    This class computes the losses for `TvpForVideoGrounding`. The process happens in two steps: 1) we compute
    hungarian assignment between ground truth boxes and the outputs of the model 2) we supervise each pair of matched
    ground-truth / prediction (supervise class and box).

    Args:
        losses (`list[str]`):
            List of all the losses to be applied.
    c                 óÊ   •— t         ‰| �  «        | j                  | j                  | j                  dœ| _        |D ]  }|| j
                  vsŒt        d|› d�«      ‚ || _        y )N©ÚiouÚdistanceÚdurationzLoss z not supported)ÚsuperÚ__init__Úloss_iouÚloss_distanceÚloss_durationÚloss_mapÚ
ValueErrorÚlosses)Úselfr2   r   Ú	__class__s      €r#   r,   zTvpLoss.__init__C   sj   ø€ Ü‰ÑÔà—=‘=Ø×*Ñ*Ø×*Ñ*ñ
ˆŒð
 ò 	?ˆDØ˜4Ÿ=™=Ò(Ü  5¨¨¨nÐ!=Ó>Ð>ð	?ð ˆ�r"   c                 óê   — t        j                  ||«      t        j                  ||«      z
  }t        j                  ||«      t        j                  ||«      z
  }d|j                  d¬«      |z  z
  }|S )z6
        Measure the intersection over union.
        r   r   ©Úmin)r   r7   ÚmaxÚclamp)	r3   Ú
start_timeÚend_timeÚcandidates_start_timeÚcandidates_end_timer*   ÚinterÚunionr(   s	            r#   r-   zTvpLoss.loss_iouP   si   € ô —	‘	Ð-¨xÓ8¼5¿9¹9ÐEZÐ\fÓ;gÑgˆÜ—	‘	Ð-¨xÓ8¼5¿9¹9ÐEZÐ\fÓ;gÑgˆØ�%—+‘+ !�+Ó$ uÑ,Ñ,ˆàˆ
r"   c                 óP  — t        j                  t        j                  ||«      d«      }t        j                  t        j                  ||«      d«      }t        j                  t        j                  ||«      t        j                  ||«      z
  |«      j                  d¬«      }|S )z5
        Measure the distance of mid points.
        g       @gš™™™™™É?r6   )r   ÚdivÚaddr8   r7   r9   )	r3   r:   r;   r<   r=   r*   Úmid_candidatesÚmid_groundtruthÚdistance_diffs	            r#   r.   zTvpLoss.loss_distanceZ   sƒ   € ô Ÿ™¤5§9¡9Ð-BÐDWÓ#XÐZ]Ó^ˆÜŸ)™)¤E§I¡I¨j¸(Ó$CÀSÓIˆÜŸ	™	Ü�I‰I�n oÓ6¼¿¹À>ÐSbÓ9cÑcÐemó
ç
‰%�Cˆ%‹.ð 	ð Ðr"   c                 óü   — t        j                  ||«      }t        j                  ||«      }t        j                  t        j                  t        j                  ||«      |«      «      }|j	                  d¬«      }|S )z5
        Measure the difference of duration.
        gš™™™™™Ù?r6   )r   ÚsubÚsquarerA   r9   )	r3   r:   r;   r<   r=   r*   Úduration_candidatesÚduration_groundtruthÚduration_diffs	            r#   r/   zTvpLoss.loss_durationf   sh   € ô $Ÿi™iÐ(;Ð=RÓSÐÜ$Ÿy™y¨°:Ó>ÐÜŸ™¤U§Y¡Y¬u¯y©yÐ9LÐNbÓ/cÐemÓ%nÓoˆØ%×+Ñ+°Ð+Ó4ˆàÐr"   c                 ó  — |\  }}}t        j                  ||«      }|dd…df   j                  «       |dd…df   j                  «       }}i }	| j                  D ],  }
|	j	                  |
 | j
                  |
   |||||«      i«       Œ. |	S )am  
        This performs the loss computation.

        Args:
            logits (`torch.FloatTensor`):
                The output logits of head module.
            labels (`list[torch.FloatTensor]`):
                List of tensors ([start, end, duration]), which contains start time, end time of the video corresponding to the text, and also the duration.
        Nr   r   )r   ÚmulÚfloatr2   Úupdater0   )r3   r   Úlabelsr*   r:   r;   Ú
candidatesr<   r=   Úlosses_dictr   s              r#   ÚforwardzTvpLoss.forwardq   s¤   € ð *0Ñ&ˆ�*˜hÜ—Y‘Y˜v xÓ0ˆ
Ø5?ÂÀ1ÀÑ5E×5KÑ5KÓ5MÈzÒZ[Ð]^ÐZ^ÑO_×OeÑOeÓOgÐ2ÐàˆØ—K‘Kò 	ˆDØ×ÑØÐ*�t—}‘} TÑ*¨:°xÐAVÐXkÐmuÓvÐwõð	ð
 Ðr"   )
r   r   r   r   r,   r-   r.   r/   rS   Ú__classcell__©r4   s   @r#   r%   r%   8   s!   ø„ ñôòò
ò	ör"   r%   c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚTvpVisionModelc           	      ó\  •— t         ‰| �  «        t        |«      | _        |j                  �|j                  j
                  d   }n¼t        | j                  d«      rDt        | j                  j                  d«      r$| j                  j                  j
                  d   }nbt        | j                  d«      rAt        | j                  j                  d«      r!| j                  j                  j                  }nt        d«      ‚t        j                  ||j                  ddddd¬	«      | _        y )
NéÿÿÿÿÚconfigÚhidden_sizesÚhidden_sizezBackbone config not foundr   r   F)Úkernel_sizeÚstrideÚpaddingÚgroupsÚbias)r+   r,   r   ÚbackboneÚbackbone_configr[   ÚhasattrrZ   r\   r1   r   ÚConv2dÚgrid_encoder_conv)r3   rZ   Úin_channelsr4   s      €r#   r,   zTvpVisionModel.__init__‰   sç   ø€ Ü‰ÑÔÜ% fÓ-ˆŒà×!Ñ!Ð-Ø ×0Ñ0×=Ñ=¸bÑA‰KÜ�T—]‘] HÔ-´'¸$¿-¹-×:NÑ:NÐP^Ô2_ØŸ-™-×.Ñ.×;Ñ;¸BÑ?‰KÜ�T—]‘] HÔ-´'¸$¿-¹-×:NÑ:NÐP]Ô2^ØŸ-™-×.Ñ.×:Ñ:‰KäÐ8Ó9Ð9ä!#§¡ØØ×ÑØØØØØô"
ˆÕr"   c                 ó¨  — |j                   \  }}}}}|j                  ||z  |||«      }| j                  |«      d   d   }| j                  |«      }t        j
                  j                  |dd¬«      }t        j
                  j                  |d¬«      }|j                   dd  \  }	}
}|j                  |||	|
|«      }|j                  ddd	d
d«      }|S )NÚfeature_mapsr   é   )r]   r^   T)Úinplaceéýÿÿÿr   r   é   )	ÚshapeÚviewrb   rf   r   Ú
functionalÚ
max_pool2dÚreluÚpermute)r3   Úpixel_valuesÚ
batch_sizeÚ
num_framesÚnum_channelsÚheightÚwidthÚgrid_feat_outputsÚgridÚnew_channelÚ
new_heightÚ	new_widths               r#   rS   zTvpVisionModel.forward    sÚ   € Ø>J×>PÑ>PÑ;ˆ
�J ¨f°eà#×(Ñ(¨°jÑ)@À,ÐPVÐX]Ó^ˆØ ŸM™M¨,Ó7¸ÑGÈÑJÐØ×%Ñ%Ð&7Ó8ˆÜ�}‰}×'Ñ'¨¸!ÀAÐ'ÓFˆÜ�}‰}×!Ñ! $°Ð!Ó5ˆØ-1¯Z©Z¸¸¨_Ñ*ˆ�Z à�y‰y˜ Z°¸jÈ)ÓTˆà�|‰|˜A˜q ! Q¨Ó*ˆØˆr"   ©r   r   r   r,   rS   rT   rU   s   @r#   rW   rW   ˆ   s   ø„ ô
ö.r"   rW   c                   ó~   ‡ — e Zd ZdZˆ fd„Zdej                  dededej                  fd„Zdde	fd	„Z
dde	fd
„Zˆ xZS )ÚTvpVisualInputEmbeddingz;
    Takes input of both image and video (multi-frame)
    c                 ór  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j                  |j
                  «      | _        t        j                  |j                  |j
                  «      | _
        t        j                  d|j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                   «      | _        |j                  | _        |j                  | _	        y )Nr   ©Úeps)r+   r,   r   Ú	EmbeddingÚmax_position_embeddingsr\   Úposition_embeddingsÚ max_grid_row_position_embeddingsÚrow_position_embeddingsÚ max_grid_col_position_embeddingsÚcol_position_embeddingsÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚ
layer_normÚDropoutÚhidden_dropout_probÚdropout©r3   rZ   r4   s     €r#   r,   z TvpVisualInputEmbedding.__init__µ   s×   ø€ Ü‰ÑÔä#%§<¡<°×0NÑ0NÐPV×PbÑPbÓ#cˆÔ Ü')§|¡|°F×4[Ñ4[Ð]c×]oÑ]oÓ'pˆÔ$Ü')§|¡|°F×4[Ñ4[Ð]c×]oÑ]oÓ'pˆÔ$Ü%'§\¡\°!°V×5GÑ5GÓ%HˆÔ"ÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒØ06×0WÑ0WˆÔ-Ø06×0WÑ0WˆÕ-r"   Ú	embeddingrx   ry   Úreturnc                 ó   — dx}}|| j                   kD  r|| j                   z  }|| j                  kD  r|| j                  z  }|j                  dddd«      }t        j                  j                  |||fdd¬«      }|j                  dddd«      }|S )z¸
        This method allows to interpolate the pre-trained pad weights , to be able to use the model on collection of high
        resolution images (high resolution videos).

        r   r   r   rj   ÚbicubicF©Úscale_factorÚmodeÚalign_corners)rˆ   rŠ   rs   r   rp   Úinterpolate)r3   r”   rx   ry   Úh0Úw0s         r#   Úinterpolate_pos_encodingz0TvpVisualInputEmbedding.interpolate_pos_encodingÁ   s¦   € ð ˆˆˆRà�D×9Ñ9Ò9Ø˜$×?Ñ?Ñ?ˆBà�4×8Ñ8Ò8Ø˜×>Ñ>Ñ>ˆBØ×%Ñ% a¨¨A¨qÓ1ˆ	Ü—M‘M×-Ñ-ØØ˜b˜ØØð	 .ó 
ˆ	ð ×%Ñ% a¨¨A¨qÓ1ˆ	ØÐr"   rŸ   c                 ó’  — |j                   \  }}}}t        | j                  |«      }t        j                  |t        j
                  |j                  ¬«      }| j                  |«      }	dt        |j                   «      dz
  z  |d|fz   }
 |	j                  |
Ž }	t        | j                  |«      }t        j                  |t        j
                  |j                  ¬«      }| j                  |«      }|d||f} |j                  |Ž }|	|z   }|r6|| j                  kD  s|| j                  kD  r|| j                  |||«      z   }|S ||z   }|S )af  
        Args:
            grid: (batch_size, height, width, hidden_dim)
            interpolate_pos_encoding: (`bool`, *optional*, defaults to `False`):
                Whether to interpolate the pre-trained position encodings.
        Returns:
            grid + col_position_embeddings.view(*col_shape): (batch_size, *, height, width, hidden_dim)
        ©ÚdtypeÚdevice)r   r   r   )rn   r7   rˆ   r   ÚarangeÚlongr£   r‰   Úlenro   rŠ   r‹   rŸ   )r3   r{   rŸ   ru   rx   ry   Ú
hidden_dimÚ
row_heightÚrow_position_idsr‰   Ú	row_shapeÚ	row_widthÚcol_position_idsr‹   Ú	col_shapeÚpositional_embeddingss                   r#   Úadd_2d_positional_embeddingsz4TvpVisualInputEmbedding.add_2d_positional_embeddingsØ   sQ  € ð 15·
±
Ñ-ˆ
�F˜E :ô ˜×>Ñ>ÀÓGˆ
Ü Ÿ<™<¨
¼%¿*¹*ÈTÏ[É[ÔYÐà"&×">Ñ">Ð?OÓ"PÐØœC §
¡
›O¨aÑ/Ñ0°JÀÀ:Ð3NÑNˆ	à">Ð"9×">Ñ">À	Ð"JÐô ˜×=Ñ=¸uÓEˆ	Ü Ÿ<™<¨	¼¿¹ÈDÏKÉKÔXÐà"&×">Ñ">Ð?OÓ"PÐØ  I¨zÐ:ˆ	à">Ð"9×">Ñ">À	Ð"JÐà 7Ð:QÑ QÐñ $Ø�T×:Ñ:Ò:¸eÀd×FkÑFkÒ>kà˜$×7Ñ7Ð8MÈvÐW\Ó]Ñ]ˆDð ˆð Ð/Ñ/ˆDØˆr"   c                 óŠ  — |j                   \  }}}}}|j                  d«      }| j                  ||¬«      }|j                  |d|«      }|j                   dd }	|j                  }
t        j                  |	t
        j                  |
¬«      }| j                  |«      }||z   }| j                  |«      }| j                  |«      }|S )a  
        Args:
            grid: Array of shape (batch_size, num_frames, height, width, num_channels).
                It contains processed frames extracted from videos, and is generated by Tvp image preprocessor. Note,
                num_frames can be 1
            interpolate_pos_encoding: (bool, *optional*, defaults to `False`):
                Whether to interpolate the pre-trained position encodings.

        Returns:
            embeddings: The embedding of grid with size (batch_size, height*width, num_channels)

        r   ©rŸ   rY   Nr¡   )rn   Úmeanr¯   ro   r£   r   Úzerosr¥   rŒ   r�   r’   )r3   r{   rŸ   ru   rv   rx   ry   rw   Úvisual_tokensÚvisual_tokens_shaper£   Útoken_type_idsrŒ   Ú
embeddingss                 r#   rS   zTvpVisualInputEmbedding.forward  sÄ   € ð ?C¿j¹jÑ;ˆ
�J ¨¨|à�y‰y˜‹|ˆØ×0Ñ0°ÐPhÐ0ÓiˆàŸ	™	 *¨b°,Ó?ˆØ+×1Ñ1°#°2Ð6ÐØ×%Ñ%ˆô Ÿ™Ð%8ÄÇ
Á
ÐSYÔZˆØ $× :Ñ :¸>Ó JÐà"Ð%:Ñ:ˆ
Ø—_‘_ ZÓ0ˆ
Ø—\‘\ *Ó-ˆ
ØÐr"   ©F)r   r   r   r   r,   r   ÚTensorÚintrŸ   Úboolr¯   rS   rT   rU   s   @r#   r�   r�   °   sT   ø„ ñô
Xð°%·,±,ð Èð ÐTWð Ð\a×\hÑ\hó ñ.'È4ó 'ñR°d÷ r"   r�   c                   ó*   ‡ — e Zd ZdZˆ fd„Zdd„Zˆ xZS )ÚTvpTextInputEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 óü  •— t         ‰| �  «        t        j                  |j                  |j
                  |j                  ¬«      | _        t        j                  |j                  |j
                  «      | _	        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                   «      | _        y )N)Úpadding_idxrƒ   )r+   r,   r   r…   Ú
vocab_sizer\   Úpad_token_idÚword_embeddingsr†   r‡   Útype_vocab_sizerŒ   r�   rŽ   r�   r�   r‘   r’   r“   s     €r#   r,   zTvpTextInputEmbeddings.__init__$  s©   ø€ Ü‰ÑÔÜ!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ#%§<¡<°×0NÑ0NÐPV×PbÑPbÓ#cˆÔ Ü%'§\¡\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"ÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r"   c                 ó.  — |�|j                  «       }n|j                  «       d d }|d   }|�|j                  n|j                  }|€Ft        j                  |t        j                  |¬«      }|j                  d«      j                  |«      }|€&t        j                  |t        j                  |¬«      }|€| j                  |«      }| j                  |«      }| j                  |«      }	||z   |	z   }
| j                  |
«      }
| j                  |
«      }
|
S )NrY   r   r¡   r   )Úsizer£   r   r¤   r¥   Ú	unsqueezeÚexpandr³   rÂ   r‡   rŒ   r�   r’   )r3   Ú	input_idsr¶   Úposition_idsÚinputs_embedsÚinput_shapeÚ
seq_lengthr£   r‡   rŒ   r·   s              r#   rS   zTvpTextInputEmbeddings.forward,  s  € ØÐ Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà  ‘^ˆ
Ø%.Ð%:�×!Ò!À×@TÑ@TˆØÐÜ Ÿ<™<¨
¼%¿*¹*ÈVÔTˆLØ'×1Ñ1°!Ó4×;Ñ;¸KÓHˆLØÐ!Ü"Ÿ[™[¨¼E¿J¹JÈvÔVˆNàÐ Ø ×0Ñ0°Ó;ˆMØ"×6Ñ6°|ÓDÐØ $× :Ñ :¸>Ó JÐà"Ð%8Ñ8Ð;PÑPˆ
Ø—_‘_ ZÓ0ˆ
Ø—\‘\ *Ó-ˆ
ØÐr"   ©NNNN©r   r   r   r   r,   rS   rT   rU   s   @r#   r½   r½   !  s   ø„ ÙQô>÷r"   r½   c                   ó^   ‡ — e Zd Zˆ fd„Zdej
                  dedefd„Z	 	 d	dedz  fd„Z	ˆ xZ
S )
ÚTvpAttentionc                 óÐ  •— t         ‰| �  «        |j                  |j                  z  dk7  r1t	        |d«      s%t        d|j                  › d|j                  › �«      ‚|j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        t        j$                  |j                  |j&                  ¬«      | _        t        j                  |j*                  «      | _        y )Nr   Úembedding_sizezThe hidden size z4 is not a multiple of the number of attention heads rƒ   )r+   r,   r\   Únum_attention_headsrd   r1   rº   Úattention_head_sizeÚall_head_sizer   ÚLinearÚqueryÚkeyÚvaluer�   Úattention_probs_dropout_probÚattn_dropoutÚdenser�   rŽ   r�   r‘   r’   r“   s     €r#   r,   zTvpAttention.__init__F  sx  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ" 6×#5Ñ#5Ð"6Ð6jÐkq÷  lFñ  lFð  kGð  Hóð ð $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ÜŸJ™J v×'JÑ'JÓKˆÔä—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r"   ÚtensorÚsequence_lengthru   c                 óŽ   — |j                  ||| j                  | j                  «      j                  dd«      j	                  «       S )Nr   rj   )ro   rÓ   rÔ   Ú	transposeÚ
contiguous)r3   rÝ   rÞ   ru   s       r#   Ú_reshapezTvpAttention._reshapeZ  s7   € à�K‰K˜
 O°T×5MÑ5MÈt×OgÑOgÓhß‰Y�q˜!‹_ß‰Z‹\ð	
r"   NÚoutput_attentionsc                 ó,  — |j                   d d \  }}| j                  |«      }| j                  |«      }| j                  |«      }| j	                  |||«      }	| j	                  |||«      }
| j	                  |||«      }t        j                  |	|
j                  dd«      «      }|t        j                  | j                  «      z  }|�||z   }t        j                  j                  |d¬«      }| j                  |«      }t        j                  ||«      }|j                  dd«      j                  «       }|j!                  ||| j"                  «      }| j%                  |«      }| j'                  |«      }| j)                  ||z   «      }|r||f}|S |f}|S )Nrj   rY   éþÿÿÿ©Údimr   )rn   r×   rØ   rÙ   râ   r   Úmatmulrà   ÚmathÚsqrtrÔ   r   rp   ÚsoftmaxrÛ   rá   ÚreshaperÕ   rÜ   r’   r�   )r3   r   Úattention_maskrã   ru   rÞ   Úmixed_query_layerÚmixed_key_layerÚmixed_value_layerÚquery_layerÚ	key_layerÚvalue_layerÚattention_scoresÚattention_probsÚattn_outputÚoutputss                   r#   rS   zTvpAttention.forwarda  s�  € ð '4×&9Ñ&9¸"¸1Ð&=Ñ#ˆ
�OØ ŸJ™J }Ó5ÐàŸ(™( =Ó1ˆØ ŸJ™J }Ó5Ðà—m‘mÐ$5°È
ÓSˆØ—M‘M /°?ÀJÓOˆ	Ø—m‘mÐ$5°È
ÓSˆô !Ÿ<™<¨°Y×5HÑ5HÈÈRÓ5PÓQÐØ+¬d¯i©i¸×8PÑ8PÓ.QÑQÐØÐ%Ø/°.Ñ@Ðô Ÿ-™-×/Ñ/Ð0@ÀbÐ/ÓIˆð ×+Ñ+¨OÓ<ˆä—l‘l ?°KÓ@ˆØ!×+Ñ+¨A¨qÓ1×<Ñ<Ó>ˆØ!×)Ñ)¨*°oÀt×GYÑGYÓZˆà—j‘j Ó-ˆØ—l‘l ;Ó/ˆØ—o‘o k°MÑ&AÓBˆá4E�; Ð0ˆØˆð MXÈ>ˆØˆr"   ©NN)r   r   r   r,   r   r¹   rº   râ   r»   rS   rT   rU   s   @r#   rÐ   rÐ   E  sA   ø„ ô>ð(
˜uŸ|™|ð 
¸cð 
Èsó 
ð Ø)-ñ	&ð   $™;÷	&r"   rÐ   c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚTvpIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y ©N)r+   r,   r   rÖ   r\   Úintermediate_sizerÜ   Ú
isinstanceÚ
hidden_actÚstrr   Úintermediate_act_fnr“   s     €r#   r,   zTvpIntermediate.__init__Œ  s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$r"   r   r•   c                 óJ   — | j                  |«      }| j                  |«      }|S rü   )rÜ   r  )r3   r   s     r#   rS   zTvpIntermediate.forward”  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr"   ©r   r   r   r,   r   r¹   rS   rT   rU   s   @r#   rú   rú   ‹  s#   ø„ ô9ð U§\¡\ð °e·l±l÷ r"   rú   c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚTvpOutputLayerc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _	        t        j                  |j                  «      | _        y )Nrƒ   )r+   r,   r   rÖ   rý   r\   rÜ   r�   rŽ   r�   r�   r‘   r’   r“   s     €r#   r,   zTvpOutputLayer.__init__›  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r"   r   Úinput_tensorr•   c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S rü   )rÜ   r’   r�   )r3   r   r  s      r#   rS   zTvpOutputLayer.forward¡  s7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™¨¸Ñ(DÓEˆØÐr"   r  rU   s   @r#   r  r  š  s1   ø„ ô>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ r"   r  c                   ó6   ‡ — e Zd Zˆ fd„Z	 	 ddedz  fd„Zˆ xZS )ÚTvpEncodeLayerc                 ó‚   •— t         ‰| �  «        t        |«      | _        t	        |«      | _        t        |«      | _        y rü   )r+   r,   rÐ   Ú	attentionrú   Úintermediater  Úoutputr“   s     €r#   r,   zTvpEncodeLayer.__init__©  s3   ø€ Ü‰ÑÔÜ% fÓ-ˆŒÜ+¨FÓ3ˆÔÜ$ VÓ,ˆ�r"   Nrã   c                 ó”   — | j                  |||¬«      }|d   }|dd  }| j                  |«      }| j                  ||«      }|f|z   }|S )N)rã   r   r   )r  r  r  )	r3   r   rí   rã   Úself_attention_outputsÚattention_outputr÷   Úintermediate_outputÚlayer_outputs	            r#   rS   zTvpEncodeLayer.forward¯  so   € ð "&§¡ØØØ/ð "0ó "
Ðð
 2°!Ñ4ÐØ(¨¨Ð,ˆØ"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6Ð8HÓIˆØ�/ GÑ+ˆØˆr"   rø   )r   r   r   r,   r»   rS   rT   rU   s   @r#   r
  r
  ¨  s#   ø„ ô-ð Ø)-ñ	ð   $™;÷	r"   r
  c            
       óX   ‡ — e Zd Zˆ fd„Z	 	 	 	 ddedz  dedz  dedz  deez  fd„Zˆ xZS )	Ú
TvpEncoderc                 óÐ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _	        y c c}w )NF)
r+   r,   rZ   r   Ú
ModuleListÚrangeÚnum_hidden_layersr
  ÚlayerÚgradient_checkpointing)r3   rZ   Ú_r4   s      €r#   r,   zTvpEncoder.__init__Ã  sN   ø€ Ü‰ÑÔØˆŒÜ—]‘]ÄEÈ&×JbÑJbÓDcÖ#d¸q¤N°6Õ$:Ò#dÓeˆŒ
Ø&+ˆÕ#ùò $es   ½A#Nrã   Úoutput_hidden_statesÚreturn_dictr•   c                 óš  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }d}d}t	        | j
                  «      D ](  \  }}	|r||fz   } |	|||«      }
|
d   }|sŒ ||
d   fz   }Œ* |r||fz   }|s|f}|r||fz   }|r||fz   }|S t        ||r|nd |r|¬«      S d ¬«      S )Nr!   r   r   )Úlast_hidden_stater   r   )rZ   r  rã   r  Ú	enumerater  r   )r3   r   rí   rã   r  r  Úall_hidden_statesÚall_attentionsÚiÚlayer_moduleÚlayer_outputsr÷   s               r#   rS   zTvpEncoder.forwardÉ  s+  € ð &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆØ1BÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð ÐØˆä(¨¯©Ó4ò 	F‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!á(¨¸ÐHYÓZˆMà)¨!Ñ,ˆMÚ Ø!/°=ÀÑ3CÐ2EÑ!E‘ð	Fñ  Ø 1°]Ð4DÑ DÐáØ$Ð&ˆGÙ#Ø!Ð%6Ð$8Ñ8�Ù Ø! ^Ð$5Ñ5�ØˆNäØ+Ù/CÑ+ÈÙ):�~ô
ð 	
ð AEô
ð 	
r"   rÍ   )	r   r   r   r,   r»   r    r   rS   rT   rU   s   @r#   r  r  Â  sT   ø„ ô,ð Ø)-Ø,0Ø#'ñ*
ð   $™;ð	*
ð
 # T™kð*
ð ˜D‘[ð*
ð 
�Ñ	 ÷*
r"   r  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )Ú	TvpPoolerc                 ó²   •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  «       | _        y rü   )r+   r,   r   rÖ   r\   rÜ   ÚTanhÚ
activationr“   s     €r#   r,   zTvpPooler.__init__ø  s9   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'™'›)ˆ�r"   r   r•   c                 ó\   — |d d …df   }| j                  |«      }| j                  |«      }|S )Nr   )rÜ   r+  )r3   r   Úfirst_token_tensorÚpooled_outputs       r#   rS   zTvpPooler.forwardý  s6   € ð +ª1¨a¨4Ñ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐr"   r  rU   s   @r#   r(  r(  ÷  s#   ø„ ô$ð
 U§\¡\ð °e·l±l÷ r"   r(  c                   óx   ‡ — e Zd ZU eed<   dZdZdZ ej                  «       de
j                  fˆ fd„«       Zˆ xZS )ÚTvpPreTrainedModelrZ   Úmodel)ÚvideoÚtextTÚmodulec                 ó®  •— t         ‰| �  |«       t        |t        j                  «      rOt        j                  |j                  dd¬«       |j                  �Pt        j                  |j                  d«       n/t        |t        «      rt        j                  |j                  «       t        |d«      rt        j                  |j                  «       t        |d«      rt        j                  |j                  «       t        |d«      rt        j                  |j                   «       t        |d	«      r t        j                  |j"                  «       yy)
zInitialize the weightsÚfan_outrr   )rš   ÚnonlinearityNr   Úpad_upÚpad_downÚpad_leftÚ	pad_right)r+   Ú_init_weightsrþ   r   re   ÚinitÚkaiming_normal_Úweightra   Ú	constant_ÚTvpModelÚnormal_Útext_promptrd   r8  r9  r:  r;  )r3   r4  r4   s     €r#   r<  z TvpPreTrainedModel._init_weights  sÝ   ø€ ô 	‰Ñ˜fÔ%Ü�fœbŸi™iÔ(Ü× Ñ  §¡°YÈVÕTØ�{‰{Ð&Ü—‘˜vŸ{™{¨AÕ.Ü˜¤Ô)Ü�L‰L˜×+Ñ+Ô,ä�6˜8Ô$Ü�L‰L˜Ÿ™Ô'Ü�6˜:Ô&Ü�L‰L˜Ÿ™Ô)Ü�6˜:Ô&Ü�L‰L˜Ÿ™Ô)Ü�6˜;Ô'Ü�L‰L˜×)Ñ)Õ*ð (r"   )r   r   r   r   r   Úbase_model_prefixÚinput_modalitiesÚsupports_gradient_checkpointingr   Úno_gradr   ÚModuler<  rT   rU   s   @r#   r0  r0    s@   ø… àÓØÐØ(ÐØ&*Ð#à€U‡]�]ƒ_ð+ B§I¡Iô +ó ô+r"   r0  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚTvpFrameDownPadPrompterz>
    Pad frames extracted from videos only at the bottom.
    c           	      ó|  •— |j                   dvrt        d«      ‚t        ‰| �  «        |j                  | _        |j
                  | _        |j                  | _        |j                   | _         t        j                  t        j                  d|j
                  d|j                  |j                  g«      «      | _        y )N©rB   ÚreplaceÚremoveú9`visual_prompter_apply` must be in (add, replace, remove)r   r   )Úvisual_prompter_applyr1   r+   r,   Úvisual_prompt_sizeÚ	frame_numÚmax_img_sizer   Ú	Parameterr   Úrandnr9  r“   s     €r#   r,   z TvpFrameDownPadPrompter.__init__'  s™   ø€ Ø×'Ñ'Ð/KÑKÜÐXÓYÐYä‰ÑÔØ"(×";Ñ";ˆÔØ×)Ñ)ˆŒØ"×/Ñ/ˆÔØ%+×%AÑ%AˆÔ"äŸ™Ü�K‰K˜˜F×,Ñ,¨a°×1JÑ1JÈF×L_ÑL_Ð`Óaó
ˆ�r"   c                 óŽ  — | j                   dk7  rst        j                  | j                  | j                  g|j                  |j
                  ¬«      }d|| j                  | j                  z
  | j                  …d d …f<   ||z  }| j                   dk7  r³t        j                  |j                  d   |j                  d   d| j                  | j                  g|j
                  ¬«      }| j                  | j                  z
  }| j                  |d d …d d …d d …|| j                  …d d …f<   ||j                  |j                  «      z  }|S )	NrB   r¡   g        rN  r   r   r   ©r£   )rP  r   ÚonesrS  r¢   r£   rQ  r³   rn   r9  Úto)r3   rt   Úvisual_prompt_maskÚpromptÚstart_points        r#   rS   zTvpFrameDownPadPrompter.forward5  s1  € Ø×%Ñ%¨Ò.Ü!&§¡Ø×"Ñ" D×$5Ñ$5Ð6¸l×>PÑ>PÐYe×YlÑYlô"Ðð fiÐ˜t×0Ñ0°4×3JÑ3JÑJÈT×M^ÑM^Ð^Ò`aÐaÑbØÐ.Ñ.ˆLØ×%Ñ%¨Ò1Ü—[‘[Ø×#Ñ# AÑ&¨×(:Ñ(:¸1Ñ(=¸qÀ$×BSÑBSÐUY×UfÑUfÐgØ#×*Ñ*ôˆFð ×+Ñ+¨d×.EÑ.EÑEˆKØBFÇ-Á-ˆF’1’aš˜K¨$×*;Ñ*;Ð;ºQÐ>Ñ?Ø˜FŸI™I l×&8Ñ&8Ó9Ñ9ˆLØÐr"   rÎ   rU   s   @r#   rJ  rJ  "  s   ø„ ñô
ör"   rJ  c                   óp   ‡ — e Zd ZdZˆ fd„Zdej                  dededej                  fd„Zd
de	fd	„Z
ˆ xZS )ÚTvpFramePadPrompterz?
    Pad frames extracted from videos in the surroundings.
    c           
      ó¼  •— |j                   dvrt        d«      ‚t        ‰| �  «        |j                  | _        |j
                  | _        |j                   | _         |j
                  |j                  dz  z
  | _        t        j                  t        j                  d|j                  d|j                  |j
                  g«      «      | _        t        j                  t        j                  d|j                  d|j                  |j
                  g«      «      | _        t        j                  t        j                  d|j                  d|j
                  |j                  dz  z
  |j                  g«      «      | _        t        j                  t        j                  d|j                  d|j
                  |j                  dz  z
  |j                  g«      «      | _        y )NrL  rO  rj   r   r   )rP  r1   r+   r,   rv   rS  rQ  Ú	base_sizer   rT  r   rU  r8  r9  r:  r;  r“   s     €r#   r,   zTvpFramePadPrompter.__init__L  s‹  ø€ Ø×'Ñ'Ð/KÑKÜÐXÓYÐYä‰ÑÔØ ×+Ñ+ˆŒØ"×/Ñ/ˆÔØ%+×%AÑ%AˆÔ"Ø×,Ñ,¨v×/HÑ/HÈ1Ñ/LÑLˆŒÜ—l‘lÜ�K‰K˜˜F×-Ñ-¨q°&×2KÑ2KÈV×M`ÑM`ÐaÓbó
ˆŒô Ÿ™Ü�K‰K˜˜F×-Ñ-¨q°&×2KÑ2KÈV×M`ÑM`ÐaÓbó
ˆŒô Ÿ™Ü�K‰KàØ×%Ñ%ØØ×'Ñ'¨&×*CÑ*CÀaÑ*GÑGØ×-Ñ-ðóó

ˆŒô Ÿ™Ü�K‰KàØ×%Ñ%ØØ×'Ñ'¨&×*CÑ*CÀaÑ*GÑGØ×-Ñ-ðóó

ˆ�r"   r[  rx   ry   r•   c                 ó  — || j                   z  || j                   z  }}|j                  \  }}}}	}
|j                  ||z  ||	|
«      }t        j                  j                  |||fdd¬«      }|j                  |||||«      }|S )z·
        This method allows to interpolate the pre-trained pad weights, to be able to use the model on collection of high
        resolution images (high resolution videos).

        r—   Fr˜   )rS  rn   rì   r   rp   rœ   )r3   r[  rx   ry   r�   rž   Úbatchrv   ÚchannelsÚprompt_heightÚprompt_widths              r#   Úinterpolate_pad_encodingz,TvpFramePadPrompter.interpolate_pad_encodingr  s˜   € ð ˜$×+Ñ+Ñ+¨U°T×5FÑ5FÑ-FˆBˆàCIÇ<Á<Ñ@ˆˆz˜8 ]°Lð —‘ ¨
Ñ 2°H¸mÈ\ÓZˆÜ—‘×*Ñ*ØØ˜b˜ØØð	 +ó 
ˆð —‘  z°8¸VÀUÓKˆØˆr"   rf  c                 óZ  — |r|j                   d   |j                   d   fn| j                  | j                  f\  }}| j                  dvrt        d| j                  › �«      ‚| j                  dv r3t	        j
                  ||g|j                  |j                  ¬«      }||z  }| j                  dv rút	        j                  d| j                  d	| j                  | j                  |j                  ¬
«      }t	        j                  | j                  || j                  gd¬«      }t	        j                  | j                  || j                  gd	¬«      }t	        j                  |j!                  d«      |gz  «      }|r| j#                  |||«      }||j%                  |j                  «      z   }|S )Nrå   rY   )rB   rN  rM  z$Invalid visual_prompter_apply value )rM  rN  r¡   )rM  rB   r   r   rW  rm   ræ   r   )rn   rS  rP  r1   r   rX  r¢   r£   r³   rv   r`  Úcatr:  r;  r8  r9  rÅ   rf  rY  )r3   rt   rf  rx   ry   rZ  Úbaser[  s           r#   rS   zTvpFramePadPrompter.forwardŠ  s{  € ñ (ð ×Ñ Ñ# \×%7Ñ%7¸Ñ%;Ñ<à×#Ñ# T×%6Ñ%6Ð7ñ 	ˆ�ð
 ×%Ñ%Ð-IÑIÜÐCÀD×D^ÑD^ÐC_Ð`ÓaÐaØ×%Ñ%Ð)>Ñ>Ü!&§¡¨V°U¨OÀ<×CUÑCUÐ^j×^qÑ^qÔ!rÐØÐ.Ñ.ˆLØ×%Ñ%Ð);Ñ;Ü—;‘;˜q $§/¡/°1°d·n±nÀdÇnÁnÐ]i×]pÑ]pÔqˆDä—Y‘Y §¡¨t°T·^±^ÐDÈ!ÔLˆFÜ—Y‘Y §¡¨V°T·]±]ÐCÈÔKˆFÜ—Y‘Y˜|×0Ñ0°Ó3°v°hÑ>Ó?ˆFÙ'Ø×6Ñ6°v¸vÀuÓM�Ø'¨&¯)©)°L×4FÑ4FÓ*GÑGˆLØÐr"   r¸   )r   r   r   r   r,   r   r¹   rº   rf  r»   rS   rT   rU   s   @r#   r^  r^  G  sG   ø„ ñô$
ðL¨u¯|©|ð ÀSð ÐQTð ÐY^×YeÑYeó ñ0¸d÷ r"   r^  )ÚframedownpadÚframepadzw
    The bare Tvp Model transformer outputting BaseModelOutputWithPooling object without any specific head on top.
    )Úcustom_introc                   óÒ   ‡ — e Zd Zˆ fd„Zd„ Zd„ Ze	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  de
dz  d	e
dz  d
e
dz  de
deez  fd„«       Zˆ xZS )rA  c                 ó"  •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        t        |«      | _        t        |«      | _
        t        |«      | _        t        j                  t        j                   dd|j"                  g«      «      | _        t        j&                  |j(                  «      | _        |j,                  t.        vrt1        d«      ‚t/        |j,                     |«      | _        | j5                  «        y )Nr   é
   z:`visual_prompter_type` must be in (framedownpad, framepad))r+   r,   rZ   rW   Úvision_modelr½   r·   r�   Úvisual_embeddingsr  Úencoderr(  Úpoolerr   rT  r   rU  r\   rC  r�   r‘   r’   Úvisual_prompter_typeÚTVP_PROMPTER_CLASSES_MAPPINGr1   Úvisual_prompterÚ	post_initr“   s     €r#   r,   zTvpModel.__init__­  sÐ   ø€ Ü‰Ñ˜Ô ØˆŒÜ*¨6Ó2ˆÔÜ0°Ó8ˆŒÜ!8¸Ó!@ˆÔÜ! &Ó)ˆŒÜ Ó'ˆŒÜŸ<™<¬¯©°Q¸¸F×<NÑ<NÐ4OÓ(PÓQˆÔÜ—z‘z &×"<Ñ"<Ó=ˆŒØ×&Ñ&Ô.JÑJÜÐYÓZÐZÜ;¸F×<WÑ<WÑXÐY_Ó`ˆÔà�‰Õr"   c                 ó.   — | j                   j                  S rü   ©r·   rÂ   )r3   s    r#   Úget_input_embeddingszTvpModel.get_input_embeddings½  s   € Ø�‰×.Ñ.Ð.r"   c                 ó&   — || j                   _        y rü   ry  )r3   rÙ   s     r#   Úset_input_embeddingszTvpModel.set_input_embeddingsÀ  s   € Ø*/ˆ�‰Õ'r"   NrÈ   rt   rí   rã   r  r  rŸ   r•   c                 ó–  — |�|n| j                   j                  }| j                  | j                  ||¬«      «      }| j	                  |¬«      }	| j                  ||¬«      }
|�€|j                  |
j                  dd «      }t        j                  |j                  d   d«      j                  |j                  |j                  ¬«      }t        j                  |||gd	¬
«      }| j                  j                  |	j                  d   d	d	«      }t        j                  ||	|
gd¬
«      }t!        | j                   ||¬«      }| j#                  |||||¬«      }|r|j$                  n|d   }| j'                  |«      }| j)                  |«      }| j)                  |«      }|s
||f|dd z   S t+        |||j,                  |j.                  ¬«      S )a  
        Examples:
        ```python
        >>> import torch
        >>> from transformers import AutoConfig, AutoTokenizer, TvpModel

        >>> model = TvpModel.from_pretrained("Jiqing/tiny-random-tvp")

        >>> tokenizer = AutoTokenizer.from_pretrained("Jiqing/tiny-random-tvp")

        >>> pixel_values = torch.rand(1, 1, 3, 448, 448)
        >>> text_inputs = tokenizer("This is an example input", return_tensors="pt")
        >>> output = model(text_inputs.input_ids, pixel_values, text_inputs.attention_mask)
        ```N)rf  )rÈ   r±   rj   r   ro  )r£   r¢   rY   ræ   r   )rZ   rÊ   rí   )rí   rã   r  r  )r   Úpooler_outputr   r   )rZ   r  rp  rv  r·   rq  Únew_onesrn   r   rX  rY  r£   r¢   rh  rC  rÇ   r	   rr  r   rs  r’   r   r   r   )r3   rÈ   rt   rí   rã   r  r  rŸ   ÚkwargsÚtext_embedding_outputÚvisual_embedding_outputÚvisual_attention_maskÚpt_maskrC  Úembedding_outputÚencoder_outputsr   r.  s                     r#   rS   zTvpModel.forwardÃ  sò  € ð4 &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆà×(Ñ(Ø× Ñ  ÐH`Ð Óaó
ˆð !%§¡¸) Ó DÐà"&×"8Ñ"8ØÐ3Kð #9ó #
Ðð Ð%à$2×$;Ñ$;Ð<S×<YÑ<YÐZ\Ð[\Ð<]Ó$^Ð!Ü—j‘j ×!5Ñ!5°aÑ!8¸"Ó=×@Ñ@Ø%×,Ñ,°N×4HÑ4Hð Aó ˆGô #ŸY™Y¨°ÐAVÐ'WÐ]_Ô`ˆNà×&Ñ&×-Ñ-Ð.C×.IÑ.IÈ!Ñ.LÈbÐRTÓUˆä Ÿ9™9 kÐ3HÐJaÐ%bÐhiÔjÐä2Ø—;‘;Ø*Ø)ô
ˆð Ÿ,™,ØØ)Ø/Ø!5Ø#ð 'ó 
ˆñ BM˜O×=Ò=ÐRaÐbcÑRdÐØŸ™Ð$5Ó6ˆØ ŸL™LÐ):Ó;ÐØŸ™ ]Ó3ˆÙØ% }Ð5¸ÈÈÐ8KÑKÐKÜ)Ø/Ø'Ø)×7Ñ7Ø&×1Ñ1ô	
ð 	
r"   )NNNNNNF)r   r   r   r,   rz  r|  r   r   Ú
LongTensorr   r»   r    r   rS   rT   rU   s   @r#   rA  rA  §  sÈ   ø„ ôò /ò0ð ð .2Ø15Ø26Ø)-Ø,0Ø#'Ø).ñI
à×#Ñ# dÑ*ðI
ð ×'Ñ'¨$Ñ.ðI
ð ×(Ñ(¨4Ñ/ð	I
ð
   $™;ðI
ð # T™kðI
ð ˜D‘[ðI
ð #'ðI
ð 
Ð+Ñ	+òI
ó ôI
r"   rA  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚTvpVideoGroundingHeadc                 ó:  •— t         ‰| �  «        t        j                  |j                  |j                  dz  «      | _        t        j                  |j                  dz  d«      | _        t        j                  «       | _        t        j                  «       | _
        y )Nrj   )r+   r,   r   rÖ   r\   Úlayer_0Úlayer_1ÚReLUÚactivation_0ÚSigmoidÚactivation_1r“   s     €r#   r,   zTvpVideoGroundingHead.__init__  sj   ø€ Ü‰ÑÔÜ—y‘y ×!3Ñ!3°V×5GÑ5GÈ!Ñ5KÓLˆŒÜ—y‘y ×!3Ñ!3°aÑ!7¸Ó;ˆŒÜŸG™G›IˆÔÜŸJ™J›LˆÕr"   c                 ó†   — | j                  | j                  |«      «      }| j                  | j                  |«      «      }|S rü   )rŽ  r‹  r�  rŒ  )r3   r~  r   s      r#   rS   zTvpVideoGroundingHead.forward  s9   € Ø×"Ñ" 4§<¡<°Ó#>Ó?ˆØ×"Ñ" 4§<¡<°Ó#7Ó8ˆØˆr"   r   rU   s   @r#   r‰  r‰    s   ø„ ô)ör"   r‰  zb
    Tvp Model with a video grounding head on top computing IoU, distance, and duration loss.
    c                   óì   ‡ — e Zd Zˆ fd„Ze	 	 	 	 	 	 	 	 ddej                  dz  dej                  dz  dej                  dz  deej                     dz  de
dz  de
dz  d	e
dz  d
e
deez  fd„«       Zˆ xZS )ÚTvpForVideoGroundingc                 ó’   •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        | j                  «        y rü   )r+   r,   rZ   rA  r1  r‰  Úvideo_grounding_headrw  r“   s     €r#   r,   zTvpForVideoGrounding.__init__$  s:   ø€ Ü‰Ñ˜Ô ØˆŒÜ˜fÓ%ˆŒ
Ü$9¸&Ó$AˆÔ!à�‰Õr"   NrÈ   rt   rí   rP   rã   r  r  rŸ   r•   c	           	      óê  — |�|n| j                   j                  }| j                  |||||||¬«      }
|
d   }| j                  |«      }d}|�pt	        g d¢«      }|j                  | j                  «        |||«      }|d   | j                   j                  |d   z  z   | j                   j                  |d   z  z   }|s|f|
dd z   }
|�|f|
z   }
|
S t        |||
j                  |
j                  ¬	«      S )
aç  
        labels (`torch.FloatTensor` of shape `(batch_size, 3)`, *optional*):
            The labels contains duration, start time, and end time of the video corresponding to the text.

        Examples:
        ```python
        >>> import torch
        >>> from transformers import AutoConfig, AutoTokenizer, TvpForVideoGrounding

        >>> model = TvpForVideoGrounding.from_pretrained("Jiqing/tiny-random-tvp")

        >>> tokenizer = AutoTokenizer.from_pretrained("Jiqing/tiny-random-tvp")

        >>> pixel_values = torch.rand(1, 1, 3, 448, 448)
        >>> text_inputs = tokenizer("This is an example input", return_tensors="pt")
        >>> output = model(text_inputs.input_ids, pixel_values, text_inputs.attention_mask)
        ```N)rã   r  r  rŸ   r   r'   r(   r)   r*   rj   )r   r   r   r   )rZ   r  r1  r•  r%   rY  r£   Údistance_loss_weightÚduration_loss_weightr   r   r   )r3   rÈ   rt   rí   rP   rã   r  r  rŸ   r€  r÷   r~  r   r   Ú	criterionÚ	loss_dicts                   r#   rS   zTvpForVideoGrounding.forward,  s)  € ð< &1Ð%<‘kÀ$Ç+Á+×BYÑBYˆØ—*‘*ØØØØ/Ø!5Ø#Ø%=ð ó 
ˆð   ™
ˆØ×*Ñ*¨=Ó9ˆàˆØÐÜÒ ?Ó@ˆIØ�L‰L˜Ÿ™Ô%Ù! &¨&Ó1ˆIà˜%Ñ Ø—+‘+×2Ñ2°Y¸zÑ5JÑJñKà—+‘+×2Ñ2°Y¸zÑ5JÑJñKð ñ
 Ø�i '¨!¨" +Ñ-ˆGØÐØ˜' GÑ+�ØˆNä&ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r"   )NNNNNNNF)r   r   r   r,   r   r   r‡  r   r    r¹   r»   r   rS   rT   rU   s   @r#   r“  r“    sÏ   ø„ ôð ð .2Ø15Ø26Ø-1Ø)-Ø,0Ø#'Ø).ñ?
à×#Ñ# dÑ*ð?
ð ×'Ñ'¨$Ñ.ð?
ð ×(Ñ(¨4Ñ/ð	?
ð
 �e—l‘lÑ# dÑ*ð?
ð   $™;ð?
ð # T™kð?
ð ˜D‘[ð?
ð #'ð?
ð 
Ð(Ñ	(ò?
ó ô?
r"   r“  )rA  r0  r“  )3r   ré   Údataclassesr   r   r   Ú r   r=  Úactivationsr   Úbackbone_utilsr   Úmasking_utilsr	   Úmodeling_layersr
   Úmodeling_outputsr   r   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_tvpr   Ú
get_loggerr   Úloggerr   rH  r%   rW   r�   r½   rÐ   rú   r  r
  r  r(  r0  rJ  r^  ru  rA  r‰  r“  Ú__all__r!   r"   r#   ú<module>r¨     sÏ  ðñ ã Ý !ã Ý å &Ý !Ý +Ý 6Ý 9ß XÑ XÝ -ß ,Ý (ð 
ˆ×	Ñ	˜HÓ	%€ð Ø
ô<˜kó <ó ó ð<ô$Mˆb�i‰iô Mô`%�R—Y‘Yô %ôPn˜bŸi™iô nôb!˜RŸY™Yô !ôHB�2—9‘9ô BôL�b—i‘iô ô�R—Y‘Yô ôÐ/ô ô41
�—‘ô 1
ôj�—	‘	ô ð ô+˜ó +ó ð+ô6"˜bŸi™iô "ôJW˜"Ÿ)™)ô Wðv ,Ø#ñ Ð ñ ðôô
a
Ð!ó a
óð
a
ôH˜BŸI™Iô ñ ðôô
I
Ð-ó I
óð
I
òX E�r"   