Ë
    ÜÓ^j Ô  ã                   óÐ  — d dl Zd dlZd dl mZ d dlmZ d dlZd dlZd dl	m
c mZ d dlmZm
Z
 ddlmZ ddlmZ ddlmZmZmZ dd	lmZ dd
lmZmZ ddlmZ ddlmZm Z m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z'  e«       rd dl(m)Z)  e!«       rd dl*m+Z+ d dl,m-Z-  G d„ de
j\                  «      Z/ G d„ de
j\                  «      Z0 G d„ de
j\                  «      Z1 G d„ de
j\                  «      Z2	 dNde
j\                  dej                  dej                  dej                  d ej                  dz  d!e3d"e3fd#„Z4 G d$„ d%e
j\                  «      Z5 G d&„ d'e
j\                  «      Z6 G d(„ d)e
j\                  «      Z7 G d*„ d+e«      Z8 G d,„ d-e
j\                  «      Z9 e d.¬/«      e G d0„ d1e«      «       «       Z:	 dOd2ej                  d3ej                  d4ej                  fd5„Z;d6ed7ed4efd8„Z<d6ej                  d7ej                  d4ej                  fd9„Z= G d:„ d;e
j\                  «      Z>d6ed7ed<e?d4efd=„Z@d6ej                  d7ej                  d<e?d4ej                  fd>„ZA G d?„ d@e
j\                  «      ZBe  G dA„ dBe«      «       ZC G dC„ dDe
jˆ                  «      ZE G dE„ dFe
j\                  «      ZF G dG„ dHe
j\                  «      ZG G dI„ dJe
j\                  «      ZH e dK¬/«       G dL„ dMeC«      «       ZIdBdMgZJy)Pé    N)ÚCallable)Ú	dataclass)ÚTensorÚnné   )Úinitialization)ÚACT2FN)ÚModelOutputÚis_scipy_availableÚrequires_backends)ÚGradientCheckpointingLayer)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚis_accelerate_available)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚVideomtConfig)Úlinear_sum_assignment)ÚPartialState)Úreducec                   óZ   ‡ — e Zd ZdZˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚVideomtPatchEmbeddingszì
    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
    Transformer.
    c                 óÌ  •— t         ‰| �  «        |j                  |j                  }}|j                  |j
                  }}t        |t        j                  j                  «      r|n||f}t        |t        j                  j                  «      r|n||f}|d   |d   z  |d   |d   z  z  }|| _        || _        || _        || _
        t        j                  ||||¬«      | _        y )Nr   r   ©Úkernel_sizeÚstride)ÚsuperÚ__init__Ú
image_sizeÚ
patch_sizeÚnum_channelsÚhidden_sizeÚ
isinstanceÚcollectionsÚabcÚIterableÚnum_patchesr   ÚConv2dÚ
projection)ÚselfÚconfigr#   r$   r%   r&   r+   Ú	__class__s          €úw/var/www/ramen.bs-engineer-server.com/venv/lib/python3.12/site-packages/transformers/models/videomt/modeling_videomt.pyr"   zVideomtPatchEmbeddings.__init__:   sÔ   ø€ Ü‰ÑÔØ!'×!2Ñ!2°F×4EÑ4E�Jˆ
Ø$*×$7Ñ$7¸×9KÑ9K�kˆä#-¨j¼+¿/¹/×:RÑ:RÔ#S‘ZÐZdÐfpÐYqˆ
Ü#-¨j¼+¿/¹/×:RÑ:RÔ#S‘ZÐZdÐfpÐYqˆ
Ø! !‘}¨
°1©Ñ5¸*ÀQ¹-È:ÐVWÉ=Ñ:XÑYˆØ$ˆŒØ$ˆŒØ(ˆÔØ&ˆÔäŸ)™) L°+È:Ð^hÔiˆ�ó    Úpixel_valuesÚreturnc                 ó:  — |j                   d   }|| j                  k7  rt        d| j                  › d|› d�«      ‚|j                  | j                  j
                  j                  ¬«      }| j	                  |«      j                  d«      j                  dd«      }|S )Nr   zoMake sure that the channel dimension of the pixel values match with the one set in the configuration. Expected z	 but got ú.©Údtypeé   )	Úshaper%   Ú
ValueErrorÚtor-   Úweightr8   ÚflattenÚ	transpose)r.   r3   r%   Ú
embeddingss       r1   ÚforwardzVideomtPatchEmbeddings.forwardI   sš   € Ø#×)Ñ)¨!Ñ,ˆØ˜4×,Ñ,Ò,ÜðØ!×.Ñ.Ð/¨y¸¸ÀaðIóð ð
 $—‘¨T¯_©_×-CÑ-C×-IÑ-I�ÓJˆØ—_‘_ \Ó2×:Ñ:¸1Ó=×GÑGÈÈ1ÓMˆ
ØÐr2   )	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r"   Útorchr   rA   Ú__classcell__©r0   s   @r1   r   r   3   s)   ø„ ñôjð
 E§L¡Lð 
°U·\±\÷ 
r2   r   c                   ó„   ‡ — e Zd ZdZdeddfˆ fd„Zd	dej                  dej                  dz  dej                  fd„Zˆ xZ	S )
ÚVideomtEmbeddingszM
    Construct the CLS token, mask token, position and patch embeddings.
    r/   r4   Nc                 ó.  •— t         ‰| �  «        || _        |j                  | _        t	        j
                  t        j                  dd|j                  «      «      | _	        t	        j
                  t        j                  d|j                  |j                  «      «      | _        t        |«      | _        | j                  j                  }t	        j                   |j"                  «      | _        d|j                  z   | _        t	        j(                  ||j                  «      | _        | j-                  dt        j.                  |«      j1                  d«      d¬«       t	        j
                  t        j                  dd|j                  «      «      | _        y )Nr   Úposition_ids©r   éÿÿÿÿF)Ú
persistent)r!   r"   r/   r$   r   Ú	ParameterrF   Úrandnr&   Ú	cls_tokenÚzerosÚnum_register_tokensÚregister_tokensr   Úpatch_embeddingsr+   ÚDropoutÚhidden_dropout_probÚdropoutÚnum_prefix_tokensÚ	EmbeddingÚposition_embeddingsÚregister_bufferÚarangeÚexpandÚ
mask_token)r.   r/   r+   r0   s      €r1   r"   zVideomtEmbeddings.__init__[   s  ø€ Ü‰ÑÔàˆŒØ ×+Ñ+ˆŒäŸ™¤e§k¡k°!°Q¸×8JÑ8JÓ&KÓLˆŒÜ!Ÿ|™|¬E¯K©K¸¸6×;UÑ;UÐW]×WiÑWiÓ,jÓkˆÔÜ 6°vÓ >ˆÔØ×+Ñ+×7Ñ7ˆÜ—z‘z &×"<Ñ"<Ó=ˆŒØ!" V×%?Ñ%?Ñ!?ˆÔÜ#%§<¡<°¸V×=OÑ=OÓ#PˆÔ Ø×Ñ˜^¬U¯\©\¸+Ó-F×-MÑ-MÈgÓ-VÐchÐÔiÜŸ,™,¤u§{¡{°1°a¸×9KÑ9KÓ'LÓMˆ�r2   r3   Úbool_masked_posc                 ó  — |j                   dk(  rA|j                  \  }}}}}|j                  ||z  |||«      }|�F|j                  ||z  d«      }n0|�.|j                   dkD  r|j                  |j                  d   d«      }|j                  d   }| j                  |«      }|�[|j	                  |j
                  t        j                  ¬«      j                  d«      }	t        j                  |	| j                  |«      }| j                  j                  |dd«      }
| j                  j                  |dd«      }|| j                  | j                  «      z   }t        j                   |
||gd¬«      }| j#                  |«      }|S )Né   rN   r9   r   )Údevicer8   r   ©Údim)Úndimr:   ÚreshaperV   r<   rd   rF   ÚboolÚ	unsqueezeÚwherer`   rR   r_   rU   r\   rL   ÚcatrY   )r.   r3   ra   Ú
batch_sizeÚ
num_framesr%   ÚheightÚwidthr@   ÚmaskÚ
cls_tokensrU   s               r1   rA   zVideomtEmbeddings.forwardk   sq  € Ø×Ñ Ò!ØBN×BTÑBTÑ?ˆJ˜
 L°&¸%Ø'×/Ñ/°
¸ZÑ0GÈÐW]Ð_dÓeˆLàÐ*Ø"1×"9Ñ"9¸*ÀzÑ:QÐSUÓ"V‘ØÐ(¨_×-AÑ-AÀAÒ-EØ-×5Ñ5°o×6KÑ6KÈAÑ6NÐPRÓSˆOà!×'Ñ'¨Ñ*ˆ
Ø×*Ñ*¨<Ó8ˆ
àÐ&Ø"×%Ñ%¨Z×->Ñ->ÄeÇjÁjÐ%ÓQ×[Ñ[Ð\^Ó_ˆDÜŸ™ T¨4¯?©?¸JÓGˆJà—^‘^×*Ñ*¨:°r¸2Ó>ˆ
Ø×.Ñ.×5Ñ5°jÀ"ÀbÓIˆà $×":Ñ":¸4×;LÑ;LÓ"MÑMˆ
Ü—Y‘Y 
¨O¸ZÐHÈaÔPˆ
Ø—\‘\ *Ó-ˆ
ØÐr2   ©N©
rB   rC   rD   rE   r   r"   rF   r   rA   rG   rH   s   @r1   rJ   rJ   V   sM   ø„ ñðN˜}ð N°õ Nñ  E§L¡Lð À5Ç<Á<ÐRVÑCVð Ðbg×bnÑbn÷ r2   rJ   c                   óX   ‡ — e Zd Zdˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )Ú
VideomtMLPr4   c                 ó~  •— t         ‰| �  «        |j                  x}}t        |j                  |j                  z  «      }t        j                  ||d¬«      | _        t        |j                  t        «      rt        |j                     | _        n|j                  | _        t        j                  ||d¬«      | _        y )NT©Úbias)r!   r"   r&   ÚintÚ	mlp_ratior   ÚLinearÚfc1r'   Ú
hidden_actÚstrr	   Ú
activationÚfc2©r.   r/   Úin_featuresÚout_featuresÚhidden_featuresr0   s        €r1   r"   zVideomtMLP.__init__†   s�   ø€ Ü‰ÑÔØ%+×%7Ñ%7Ð7ˆ�lÜ˜f×0Ñ0°6×3CÑ3CÑCÓDˆÜ—9‘9˜[¨/ÀÔEˆŒÜ�f×'Ñ'¬Ô-Ü$ V×%6Ñ%6Ñ7ˆD�Oà$×/Ñ/ˆDŒOÜ—9‘9˜_¨lÀÔFˆ�r2   Úhidden_statec                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S rs   )r}   r€   r�   ©r.   r†   s     r1   rA   zVideomtMLP.forward‘   s2   € Ø—x‘x Ó-ˆØ—‘ |Ó4ˆØ—x‘x Ó-ˆØÐr2   ©r4   N©rB   rC   rD   r"   rF   r   rA   rG   rH   s   @r1   rv   rv   …   s$   ø„ õ	Gð E§L¡Lð °U·\±\÷ r2   rv   c                   óX   ‡ — e Zd Zdˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚVideomtGatedMLPr4   c                 ó0  •— t         ‰| �  «        |j                  x}}t        |j                  |j                  z  «      }t        |dz  dz  «      dz   dz  dz  }t        j                  |d|z  d¬«      | _        t        j                  ||d¬«      | _        y ©Nr9   r   é   é   Trx   ©	r!   r"   r&   rz   r{   r   r|   Ú
weights_inÚweights_outr‚   s        €r1   r"   zVideomtGatedMLP.__init__™   óŠ   ø€ Ü‰ÑÔØ%+×%7Ñ%7Ð7ˆ�lÜ˜f×0Ñ0°6×3CÑ3CÑCÓDˆÜ˜°Ñ2°QÑ6Ó7¸!Ñ;ÀÑAÀAÑEˆäŸ)™) K°°_Ñ1DÈ4ÔPˆŒÜŸ9™9 _°lÈÔNˆÕr2   r†   c                 ó¶   — | j                  |«      }|j                  dd¬«      \  }}t        j                  j	                  |«      |z  }| j                  |«      S ©Nr9   rN   re   ©r’   Úchunkr   Ú
functionalÚsilur“   ©r.   r†   Úx1Úx2Úhiddens        r1   rA   zVideomtGatedMLP.forward¢   óS   € Ø—‘ |Ó4ˆØ×#Ñ# A¨2Ð#Ó.‰ˆˆBÜ—‘×#Ñ# BÓ'¨"Ñ,ˆØ×Ñ Ó'Ð'r2   r‰   rŠ   rH   s   @r1   rŒ   rŒ   ˜   ó$   ø„ õOð( E§L¡Lð (°U·\±\÷ (r2   rŒ   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingrY   c                 óÀ  — t        j                  ||j                  dd«      «      |z  }|�||z   }t        j                  j                  |dt         j                  ¬«      j                  |j                  «      }t        j                  j                  ||| j                  ¬«      }t        j                  ||«      }	|	j                  dd«      j                  «       }	|	|fS )NrN   éþÿÿÿ)rf   r8   )ÚpÚtrainingr   r9   )rF   Úmatmulr?   r   r™   ÚsoftmaxÚfloat32r<   r8   rY   rª   Ú
contiguous)
r¡   r¢   r£   r¤   r¥   r¦   rY   ÚkwargsÚattn_weightsÚattn_outputs
             r1   Úeager_attention_forwardr²   ©   sº   € ô —<‘<  s§}¡}°R¸Ó'<Ó=ÀÑG€LØÐ!Ø# nÑ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€Lä—,‘,˜|¨UÓ3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r2   c            
       ó    ‡ — e Zd ZdZˆ fd„Z	 ddej                  dej                  dz  deej                  ej                  dz  f   fd„Zˆ xZ	S )	ÚVideomtAttentionz=Multi-headed attention from 'Attention Is All You Need' paperc                 ó  •— t         ‰| �  «        || _        |j                  | _        |j
                  | _        | j                  | j                  z  | _        | j                  | j                  z  | j                  k7  r&t        d| j                  › d| j                  › d�«      ‚| j                  dz  | _	        |j                  | _        d| _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        t        j                  | j                  | j                  «      | _        y )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      à¿F)r!   r"   r/   r&   Ú	embed_dimÚnum_attention_headsÚ	num_headsÚhead_dimr;   ÚscaleÚattention_dropoutrY   Ú	is_causalr   r|   Úk_projÚv_projÚq_projÚout_proj©r.   r/   r0   s     €r1   r"   zVideomtAttention.__init__Ã   s  ø€ Ü‰ÑÔØˆŒØ×+Ñ+ˆŒØ×3Ñ3ˆŒØŸ™¨$¯.©.Ñ8ˆŒØ�=‰=˜4Ÿ>™>Ñ)¨T¯^©^Ò;ÜØMÈdÏnÉnÐM]ð ^Ø—N‘NÐ# 2ð'óð ð —]‘] DÑ(ˆŒ
Ø×/Ñ/ˆŒØˆŒä—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜ—i‘i §¡°·±Ó?ˆŒÜŸ	™	 $§.¡.°$·.±.ÓAˆ�r2   NÚhidden_statesr¥   r4   c           
      ó®  — |j                   dd }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }| j                  |«      j                  |«      j	                  dd«      }t        j                  | j                  j                  t        «      }	 |	| ||||| j                  | j                  | j                  sdn| j                  ¬«      \  }
} |
j                   g |¢d‘­Ž j#                  «       }
| j%                  |
«      }
|
|fS )z#Input shape: Batch x Time x ChannelNrN   r   r9   ç        )r¼   r¦   rY   )r:   r¹   r¿   Úviewr?   r½   r¾   r   Úget_interfacer/   Ú_attn_implementationr²   r¼   rº   rª   rY   rh   r®   rÀ   )r.   rÂ   r¥   r¯   Úinput_shapeÚhidden_shapeÚqueriesÚkeysÚvaluesÚattention_interfacer±   r°   s               r1   rA   zVideomtAttention.forward×   s<  € ð $×)Ñ)¨#¨2Ð.ˆà8˜Ð8 bÐ8¨$¯-©-Ñ8ˆØ—+‘+˜mÓ,×1Ñ1°,Ó?×IÑIÈ!ÈQÓOˆØ�{‰{˜=Ó)×.Ñ.¨|Ó<×FÑFÀqÈ!ÓLˆØ—‘˜]Ó+×0Ñ0°Ó>×HÑHÈÈAÓNˆä(?×(MÑ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØØ—n‘nØ—J‘JØ#Ÿ}š}‘C°$·,±,ô	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—m‘m KÓ0ˆà˜LÐ(Ð(r2   rs   )
rB   rC   rD   rE   r"   rF   r   ÚtuplerA   rG   rH   s   @r1   r´   r´   À   sV   ø„ ÙGôBð. /3ñ!)à—|‘|ð!)ð Ÿ™ tÑ+ð!)ð
 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷!)r2   r´   c                   óX   ‡ — e Zd Zdˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚVideomtSwiGLUFFNr4   c                 ó0  •— t         ‰| �  «        |j                  x}}t        |j                  |j                  z  «      }t        |dz  dz  «      dz   dz  dz  }t        j                  |d|z  d¬«      | _        t        j                  ||d¬«      | _        y rŽ   r‘   r‚   s        €r1   r"   zVideomtSwiGLUFFN.__init__ü   r”   r2   r†   c                 ó¶   — | j                  |«      }|j                  dd¬«      \  }}t        j                  j	                  |«      |z  }| j                  |«      S r–   r—   r›   s        r1   rA   zVideomtSwiGLUFFN.forward  rŸ   r2   r‰   rŠ   rH   s   @r1   rÐ   rÐ   û   r    r2   rÐ   c                   ór   ‡ — e Zd ZdZd	deddfˆ fd„Zdej                  dej                  fd„Zde	fd„Z
ˆ xZS )
ÚVideomtDropPathzÏStochastic depth (DropPath) per sample, for residual blocks.

    Identity when ``drop_prob`` is 0 or outside training. See `Deep Networks with Stochastic Depth
    <https://arxiv.org/abs/1603.09382>`_.
    Ú	drop_probr4   Nc                 ó0   •— t         ‰| �  «        || _        y rs   )r!   r"   rÕ   )r.   rÕ   r0   s     €r1   r"   zVideomtDropPath.__init__  s   ø€ Ü‰ÑÔØ"ˆ�r2   rÂ   c                 óP  — | j                   dk(  s| j                  s|S d| j                   z
  }|j                  d   fd|j                  dz
  z  z   }t	        j
                  ||j                  |j                  ¬«      }t	        j                  ||z   «      }|j                  |«      |z  S )NrÄ   r   r   )r   ©r8   rd   )
rÕ   rª   r:   rg   rF   Úrandr8   rd   ÚfloorÚdiv)r.   rÂ   Ú	keep_probr:   Úrandom_tensors        r1   rA   zVideomtDropPath.forward  s—   € Ø�>‰>˜SÒ ¨¯ªØ Ð Ø˜Ÿ™Ñ&ˆ	Ø×$Ñ$ QÑ'Ð)¨D°M×4FÑ4FÈÑ4JÑ,KÑKˆÜŸ
™
 5°×0CÑ0CÈM×L`ÑL`ÔaˆÜŸ™ M°IÑ$=Ó>ˆØ× Ñ  Ó+¨mÑ;Ð;r2   c                 ó    — d| j                   › �S )Nzp=)rÕ   ©r.   s    r1   Ú
extra_reprzVideomtDropPath.extra_repr   s   € Ø�D—N‘NÐ#Ð$Ð$r2   ©rÄ   )rB   rC   rD   rE   Úfloatr"   rF   r   rA   r   rà   rG   rH   s   @r1   rÔ   rÔ     sB   ø„ ññ# %ð #°$õ #ð< U§\¡\ð <°e·l±ló <ð%˜C÷ %r2   rÔ   c                   ó†   ‡ — e Zd ZdZdeddfˆ fd„Z	 d	dej                  dej                  dz  dej                  fd„Zˆ xZ	S )
ÚVideomtLayerzCThis corresponds to the Block class in the original implementation.r/   r4   Nc                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  ¬«      | _        t        |«      | _        t        |«      | _
        |j                  dkD  rt        |j                  «      nt        j                  «       | _        t        j                  |j                  |j
                  ¬«      | _        |j                   rt#        |«      | _        nt'        |«      | _        t        |«      | _        y )N©ÚepsrÄ   )r!   r"   r   Ú	LayerNormr&   Úlayer_norm_epsÚnorm1r´   Ú	attentionÚVideomtLayerScaleÚlayer_scale1Údrop_path_raterÔ   ÚIdentityÚ	drop_pathÚnorm2Úuse_swiglu_ffnrÐ   Úmlprv   Úlayer_scale2rÁ   s     €r1   r"   zVideomtLayer.__init__'  s½   ø€ Ü‰ÑÔä—\‘\ &×"4Ñ"4¸&×:OÑ:OÔPˆŒ
Ü)¨&Ó1ˆŒÜ-¨fÓ5ˆÔØCI×CXÑCXÐ[^ÒC^œ¨×)>Ñ)>Ô?Ôdf×doÑdoÓdqˆŒä—\‘\ &×"4Ñ"4¸&×:OÑ:OÔPˆŒ
à× Ò Ü'¨Ó/ˆD�Hä! &Ó)ˆDŒHÜ-¨fÓ5ˆÕr2   rÂ   r¥   c                 ó*  — | j                  |«      }| j                  ||«      \  }}| j                  |«      }| j                  |«      |z   }| j	                  |«      }| j                  |«      }| j                  |«      }| j                  |«      |z   }|S rs   )rê   rë   rí   rð   rñ   ró   rô   )r.   rÂ   r¥   Úhidden_states_normÚself_attention_outputÚ_Úlayer_outputs          r1   rA   zVideomtLayer.forward7  s›   € ð
 "ŸZ™Z¨Ó6ÐØ#'§>¡>Ð2DÀnÓ#UÑ Ð˜qØ $× 1Ñ 1Ð2GÓ HÐð Ÿ™Ð'<Ó=ÀÑMˆð —z‘z -Ó0ˆØ—x‘x Ó-ˆØ×(Ñ(¨Ó6ˆð —~‘~ lÓ3°mÑCˆàÐr2   rs   rt   rH   s   @r1   rä   rä   $  sP   ø„ ÙMð6˜}ð 6°õ 6ð& /3ñà—|‘|ðð Ÿ™ tÑ+ðð 
�‰÷	r2   rä   c                   óX   ‡ — e Zd Zdˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )rì   r4   c                 óª   •— t         ‰| �  «        t        j                  |j                  t        j                  |j                  «      z  «      | _        y rs   )	r!   r"   r   rP   Úlayerscale_valuerF   Úonesr&   Úlambda1rÁ   s     €r1   r"   zVideomtLayerScale.__init__O  s8   ø€ Ü‰ÑÔÜ—|‘| F×$;Ñ$;¼e¿j¹jÈ×I[ÑI[Ó>\Ñ$\Ó]ˆ�r2   r†   c                 ó    — || j                   z  S rs   )rþ   rˆ   s     r1   rA   zVideomtLayerScale.forwardS  s   € Ø˜dŸl™lÑ*Ð*r2   r‰   rŠ   rH   s   @r1   rì   rì   N  s$   ø„ õ^ð+ E§L¡Lð +°U·\±\÷ +r2   rì   a¨  
    Class for outputs of [`VideomtForUniversalSegmentationOutput`].

    This output can be directly passed to [`~VideomtVideoProcessor.post_process_semantic_segmentation`] or
    [`~VideomtVideoProcessor.post_process_instance_segmentation`] or
    [`~VideomtVideoProcessor.post_process_panoptic_segmentation`] to compute final segmentation maps. Please, see
    [`~VideomtVideoProcessor`] for details regarding usage.
    )Úcustom_introc                   ó  — e Zd ZU dZdZej                  dz  ed<   dZej                  dz  ed<   dZ	ej                  dz  ed<   dZ
ej                  dz  ed<   dZeej                     dz  ed<   dZeej                     dz  ed<   y)	Ú%VideomtForUniversalSegmentationOutputa€  
    loss (`torch.Tensor`, *optional*):
        The computed loss, returned when labels are present.
    class_queries_logits (`torch.FloatTensor`):
        A tensor of shape `(batch_size, num_queries, num_labels + 1)` representing the proposed classes for each
        query. Note the `+ 1` is needed because we incorporate the null class.
    masks_queries_logits (`torch.FloatTensor`):
        A tensor of shape `(batch_size, num_queries, height, width)` representing the proposed masks for each
        query.
    last_hidden_state (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
        Last hidden states (final feature map) of the last layer.
    hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
        Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each stage) of
        shape `(batch_size, sequence_length, hidden_size)`. Hidden-states all layers of the model.
    attentions (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
        Tuple of `tuple(torch.FloatTensor)` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
        sequence_length)`. Self and Cross Attentions weights from transformer decoder.
    NÚlossÚclass_queries_logitsÚmasks_queries_logitsÚlast_hidden_staterÂ   Ú
attentions)rB   rC   rD   rE   r  rF   ÚFloatTensorÚ__annotations__r  r  r  rÂ   rÎ   r  © r2   r1   r  r  W  s”   … ñð& &*€Dˆ%×
Ñ
˜dÑ
"Ó)Ø59Ð˜%×+Ñ+¨dÑ2Ó9Ø59Ð˜%×+Ñ+¨dÑ2Ó9Ø26Ð�u×(Ñ(¨4Ñ/Ó6Ø59€M�5˜×*Ñ*Ñ+¨dÑ2Ó9Ø26€J��e×'Ñ'Ñ(¨4Ñ/Ô6r2   r  Úinput_featuresÚpoint_coordinatesr4   c                 óÚ   — |j                  «       dk(  rd}|j                  d«      }t        j                  j                  j
                  | d|z  dz
  fi |¤Ž}|r|j                  d«      }|S )a(  
    A wrapper around `torch.nn.functional.grid_sample` to support 3D point_coordinates tensors.

    Args:
        input_features (`torch.Tensor` of shape (batch_size, channels, height, width)):
            A tensor that contains features map on a height * width grid
        point_coordinates (`torch.Tensor` of shape (batch_size, num_points, 2) or (batch_size, grid_height, grid_width,:
        2)):
            A tensor that contains [0, 1] * [0, 1] normalized point coordinates
        add_dim (`bool`):
            boolean value to keep track of added dimension

    Returns:
        point_features (`torch.Tensor` of shape (batch_size, channels, num_points) or (batch_size, channels,
        height_grid, width_grid):
            A tensor that contains features for points in `point_coordinates`.
    r   Tr9   g       @ç      ð?)rf   rj   rF   r   r™   Úgrid_sampleÚsqueeze)r  r  Úadd_dimr¯   Úpoint_featuress        r1   Úsample_pointr    st   € ð( ×ÑÓ !Ò#ØˆØ-×7Ñ7¸Ó:Ðô —X‘X×(Ñ(×4Ñ4°^ÀSÐK\ÑE\Ð_bÑEbÑmÐflÑm€NÙØ'×/Ñ/°Ó2ˆàÐr2   ÚinputsÚlabelsc                 ó  — | j                  «       j                  d«      } dt        j                  | |j                  «      z  }| j                  d«      dd…df   |j                  d«      ddd…f   z   }d|dz   |dz   z  z
  }|S )aÈ  
    A pair wise version of the dice loss, see `dice_loss` for usage.

    Args:
        inputs (`torch.Tensor`):
            A tensor representing a mask
        labels (`torch.Tensor`):
            A tensor with the same shape as inputs. Stores the binary classification labels for each element in inputs
            (0 for the negative class and 1 for the positive class).

    Returns:
        `torch.Tensor`: The computed loss between each pairs.
    r   r9   rN   N)Úsigmoidr>   rF   r«   ÚTÚsum)r  r  Ú	numeratorÚdenominatorr  s        r1   Úpair_wise_dice_lossr  Ÿ  s|   € ð �^‰^Ó×%Ñ% aÓ(€FØ”E—L‘L ¨¯©Ó2Ñ2€Ià—*‘*˜R“.¢ D Ñ)¨F¯J©J°r«N¸4Â¸7Ñ,CÑC€KØ�	˜A‘ +°¡/Ñ2Ñ2€DØ€Kr2   c                 ó\  — | j                   d   }t        j                  d¬«      } || t        j                  | «      «      } || t        j
                  | «      «      }t        j                  ||z  |j                  «      }t        j                  ||z  d|z
  j                  «      }||z   }|S )aê  
    A pair wise version of the cross entropy loss, see `sigmoid_cross_entropy_loss` for usage.

    Args:
        inputs (`torch.Tensor`):
            A tensor representing a mask.
        labels (`torch.Tensor`):
            A tensor with the same shape as inputs. Stores the binary classification labels for each element in inputs
            (0 for the negative class and 1 for the positive class).

    Returns:
        loss (`torch.Tensor`): The computed loss between each pairs.
    r   Únone©Ú	reduction)r:   r   ÚBCEWithLogitsLossrF   Ú	ones_likeÚ
zeros_liker«   r  )	r  r  Úheight_and_widthÚ	criterionÚcross_entropy_loss_posÚcross_entropy_loss_negÚloss_posÚloss_negr  s	            r1   Ú$pair_wise_sigmoid_cross_entropy_lossr*  µ  s™   € ð —|‘| A‘Ðä×$Ñ$¨vÔ6€IÙ& v¬u¯©¸vÓ/FÓGÐÙ& v¬u×/?Ñ/?ÀÓ/GÓHÐä�|‰|Ð2Ð5EÑEÀvÇxÁxÓP€HÜ�|‰|Ð2Ð5EÑEÈÈFÉ
Ç~Á~ÓV€HØ�hÑ€DØ€Kr2   c                   óØ   ‡ — e Zd ZdZ	 ddedededefˆ fd„Z ej                  «       dej                  dej                  d	ej                  d
ej                  de
ee	      f
d„«       Zˆ xZS )ÚVideomtHungarianMatcheraq  This class computes an assignment between the labels and the predictions of the network.

    For efficiency reasons, the labels don't include the no_object. Because of this, in general, there are more
    predictions than labels. In this case, we do a 1-to-1 matching of the best predictions, while the others are
    un-matched (and thus treated as non-objects).
    Ú
cost_classÚ	cost_maskÚ	cost_diceÚ
num_pointsc                 óŽ   •— t         ‰| �  «        |dk(  r|dk(  r|dk(  rt        d«      ‚|| _        || _        || _        || _        y)aH  Creates the matcher

        Params:
            cost_class (`float`, *optional*, defaults to 1.0):
                Relative weight of the classification error in the matching cost.
            cost_mask (`float`, *optional*,  defaults to 1.0):
                This is the relative weight of the focal loss of the binary mask in the matching cost.
            cost_dice (`float`, *optional*, defaults to 1.0):
                This is the relative weight of the dice loss of the binary mask in the matching cost.
            num_points (`int`, *optional*, defaults to 12544):
                No. of points to sample on which the mask loss will be calculated. The same set of K points are
                uniformly sampled for all prediction and ground truth masks to construct the cost matrix for bipartite
                matching.
        r   zAll costs can't be 0N)r!   r"   r;   r0  r-  r.  r/  )r.   r-  r.  r/  r0  r0   s        €r1   r"   z VideomtHungarianMatcher.__init__Ù  sK   ø€ ô" 	‰ÑÔØ˜Š?˜y¨Aš~°)¸q².ÜÐ3Ó4Ð4à$ˆŒØ$ˆŒØ"ˆŒØ"ˆ�r2   r  r  Úmask_labelsÚclass_labelsr4   c           	      ó¨  — g }|j                   d   }t        |«      D �]Ï  }||   j                  d«      }||   }	|dd…||   f    }
||   j                  |	«      }|dd…df   }|	dd…df   }	t	        j
                  d| j                  d|	j                  ¬«      }|j                  |j                   d   dd«      }t        ||d¬«      j                  d«      }|j                  |	j                   d   dd«      }t        |	|d¬«      j                  d«      }	t        |	|«      }t        |	|«      }| j                  |z  | j                  |
z  z   | j                  |z  z   }t	        j                   |t	        j"                  d	«      «      }t	        j$                  |t	        j"                  d
«      «      }t	        j&                  |d«      }t)        |j+                  «       «      }|j-                  |«       �ŒÒ |D ��cg c]O  \  }}t	        j.                  |t        j0                  ¬«      t	        j.                  |t        j0                  ¬«      f‘ŒQ }}}|S c c}}w )ao  
        Params:
            masks_queries_logits (`torch.Tensor`):
                A tensor of dim `batch_size, num_queries, num_labels` with the classification logits.
            class_queries_logits (`torch.Tensor`):
                A tensor of dim `batch_size, num_queries, height, width` with the predicted masks.
            class_labels (`torch.Tensor`):
                A tensor of dim `num_target_boxes` (where num_target_boxes is the number of ground-truth objects in the
                target) containing the class labels.
            mask_labels (`torch.Tensor`):
                A tensor of dim `num_target_boxes, height, width` containing the target masks.

        Returns:
            matched_indices (`list[tuple[Tensor]]`): A list of size batch_size, containing tuples of (index_i, index_j)
            where:
                - index_i is the indices of the selected predictions (in order)
                - index_j is the indices of the corresponding selected labels (in order)
            For each batch element, it holds:
                len(index_i) = len(index_j) = min(num_queries, num_target_boxes).
        r   rN   Nr   r9   ©rd   F©Úalign_cornersg    _ Bg    _ Âr7   )r:   Úranger¬   r<   rF   rÙ   r0  rd   Úrepeatr  r  r*  r  r.  r-  r/  ÚminimumÚtensorÚmaximumÚ
nan_to_numr   ÚcpuÚappendÚ	as_tensorÚint64)r.   r  r  r2  r3  Úindicesrm   ÚiÚ
pred_probsÚ	pred_maskr-  Útarget_maskr  Útarget_coordinatesÚpred_coordinatesr.  r/  Úcost_matrixÚassigned_indicesÚjÚmatched_indicess                        r1   rA   zVideomtHungarianMatcher.forwardó  s/  € ð8 *,ˆð *×/Ñ/°Ñ2ˆ
Ü�zÓ"ó 	-ˆAØ-¨aÑ0×8Ñ8¸Ó<ˆJØ,¨QÑ/ˆIð %¢Q¨°Q©Ð%7Ñ8Ð8ˆJØ% a™.×+Ñ+¨IÓ6ˆKØ%¢a¨ gÑ.ˆKØ!¢! T 'Ñ*ˆIô !&§
¡
¨1¨d¯o©o¸qÈ×IYÑIYÔ ZÐà!2×!9Ñ!9¸+×:KÑ:KÈAÑ:NÐPQÐSTÓ!UÐÜ& {Ð4FÐV[Ô\×dÑdÐefÓgˆKà0×7Ñ7¸	¿¹ÈÑ8JÈAÈqÓQÐÜ$ YÐ0@ÐPUÔV×^Ñ^Ð_`ÓaˆIô =¸YÈÓTˆIä+¨I°{ÓCˆIàŸ.™.¨9Ñ4°t·±ÈÑ7SÑSÐVZ×VdÑVdÐgpÑVpÑpˆKäŸ-™-¨´U·\±\À$Ó5GÓHˆKÜŸ-™-¨´U·\±\À%Ó5HÓIˆKÜ×*Ñ*¨;¸Ó:ˆKä0EÀkÇoÁoÓFWÓ0XÐØ�N‰NÐ+Ö,ð?	-ðF ho÷
Ù_cÐ_`ÐbcŒU�_‰_˜Q¤e§k¡kÔ2´E·O±OÀAÌUÏ[É[Ô4YÒZð
ˆñ 
ð Ðùó
s   Ç5AI)r  r  r  i 1  )rB   rC   rD   rE   râ   rz   r"   rF   Úno_gradr   ÚlistrÎ   rA   rG   rH   s   @r1   r,  r,  Ñ  s    ø„ ñð joñ#Øð#Ø27ð#ØJOð#Øcfõ#ð4 €U‡]�]ƒ_ðDà#Ÿl™lðDð $Ÿl™lðDð —\‘\ð	Dð
 —l‘lðDð 
ˆe�F‰mÑ	òDó ôDr2   r,  Ú	num_masksc                 óú   — | j                  «       j                  d«      }d||z  j                  d«      z  }|j                  d«      |j                  d«      z   }d|dz   |dz   z  z
  }|j                  «       |z  }|S )a4  
    Compute the DICE loss, similar to generalized IOU for masks as follows:

    $$ \mathcal{L}_{\text{dice}(x, y) = 1 - \frac{2 * x \cap y }{x \cup y + 1}} $$

    In practice, since `labels` is a binary mask, (only 0s and 1s), dice can be computed as follow

    $$ \mathcal{L}_{\text{dice}(x, y) = 1 - \frac{2 * x * y }{x + y + 1}} $$

    Args:
        inputs (`torch.Tensor`):
            A tensor representing a mask.
        labels (`torch.Tensor`):
            A tensor with the same shape as inputs. Stores the binary classification labels for each element in inputs
            (0 for the negative class and 1 for the positive class).
        num_masks (`int`):
            The number of masks present in the current batch, used for normalization.

    Returns:
        `torch.Tensor`: The computed loss.
    r   r9   rN   )r  r>   r  )r  r  rO  Úprobsr  r  r  s          r1   Ú	dice_lossrR  ;  sz   € ð, �N‰NÓ×$Ñ$ QÓ'€EØ�U˜V‘^×(Ñ(¨Ó,Ñ,€IØ—)‘)˜B“- &§*¡*¨R£.Ñ0€KØ�	˜A‘ +°¡/Ñ2Ñ2€DØ�8‰8‹:˜	Ñ!€DØ€Kr2   c                 óˆ   — t        j                  d¬«      } || |«      }|j                  d«      j                  «       |z  }|S )a|  
    Args:
        inputs (`torch.Tensor`):
            A float tensor of arbitrary shape.
        labels (`torch.Tensor`):
            A tensor with the same shape as inputs. Stores the binary classification labels for each element in inputs
            (0 for the negative class and 1 for the positive class).

    Returns:
        loss (`torch.Tensor`): The computed loss.
    r  r  r   )r   r!  Úmeanr  )r  r  rO  r%  Úcross_entropy_lossr  s         r1   Úsigmoid_cross_entropy_lossrV  Y  sD   € ô ×$Ñ$¨vÔ6€IÙ" 6¨6Ó2Ðà×"Ñ" 1Ó%×)Ñ)Ó+¨iÑ7€DØ€Kr2   c                   óÒ  ‡ — e Zd Zdedeeef   fˆ fd„Zdeee	      dee	   fd„Z
dee   deeef   fd„Zd	ed
ee   deej                     deeef   fd„Zdej                  deej                     deej                     de	deeej                  f   f
d„Zd„ Zd„ Zdej                  dej                  fd„Zdej                  de	de	dedej                  f
d„Z	 ddej                  d	ej                  deej                     d
eej                     deeej                  f   dz  deeej                  f   fd„Zd
ej                  dej0                  dej                  fd„Zˆ xZS )ÚVideomtLossr/   Úweight_dictc                 óò  •— t         ‰| �  «        t        | dg«       |j                  | _        || _        |j
                  | _        t        j                  | j                  dz   «      }| j                  |d<   | j                  d|«       |j                  | _        |j                  | _        |j                  | _        t        |j                  |j                   |j"                  | j                  ¬«      | _        y)aQ  
        The Videomt Loss. The loss is computed very similar to DETR. The process happens in two steps: 1) we
        compute hungarian assignment between ground truth masks and the outputs of the model 2) we supervise each pair
        of matched ground-truth / prediction (supervise class and mask)

        Args:
            config (`VideomtConfig`):
                The configuration for Videomt model also containing loss calculation specific parameters.
            weight_dict (`dict[str, float]`):
                A dictionary of weights to be applied to the different losses.
        Úscipyr   rN   Úempty_weight)r-  r/  r.  r0  N)r!   r"   r   Ú
num_labelsrY  Úno_object_weightÚeos_coefrF   rý   r]   Útrain_num_pointsr0  Úoversample_ratioÚimportance_sample_ratior,  Úclass_weightÚdice_weightÚmask_weightÚmatcher)r.   r/   rY  r\  r0   s       €r1   r"   zVideomtLoss.__init__n  sÎ   ø€ ô 	‰ÑÔÜ˜$  	Ô*Ø ×+Ñ+ˆŒØ&ˆÔð ×/Ñ/ˆŒÜ—z‘z $§/¡/°AÑ"5Ó6ˆØŸ=™=ˆ�RÑØ×Ñ˜^¨\Ô:ð !×1Ñ1ˆŒØ &× 7Ñ 7ˆÔØ'-×'EÑ'EˆÔ$ä.Ø×*Ñ*Ø×(Ñ(Ø×(Ñ(Ø—‘ô	
ˆ�r2   Úsizesr4   c                 ón   — |d   }|dd  D ]'  }t        |«      D ]  \  }}t        ||   |«      ||<   Œ Œ) |S )Nr   r   )Ú	enumerateÚmax)r.   rg  ÚmaxesÚsublistÚindexÚitems         r1   Ú_max_by_axiszVideomtLoss._max_by_axis‘  sS   € Ø�a‘ˆØ˜Q˜R�yò 	7ˆGÜ(¨Ó1ò 7‘��tÜ" 5¨¡<°Ó6��e’ñ7ð	7ð ˆr2   Útensorsc                 ó`  — | j                  |D �cg c]  }t        |j                  «      ‘Œ c}«      }t        |«      g|z   }|\  }}}}|d   j                  }	|d   j
                  }
t        j                  ||	|
¬«      }t        j                  |||ft        j                  |
¬«      }t        |||«      D ]o  \  }}}|d |j                  d   …d |j                  d   …d |j                  d   …f   j                  |«       d|d |j                  d   …d |j                  d   …f<   Œq ||fS c c}w )Nr   rØ   r   r9   F)ro  rN  r:   Úlenr8   rd   rF   rS   rý   ri   ÚzipÚcopy_)r.   rp  r;  Úmax_sizeÚbatch_shaperm   rø   ro   rp   r8   rd   Úpadded_tensorsÚpadding_masksÚpadded_tensorÚpadding_masks                  r1   Ú_pad_images_to_max_in_batchz'VideomtLoss._pad_images_to_max_in_batch™  s7  € à×$Ñ$ÀwÖ%O¸V¤d¨6¯<©<Õ&8Ò%OÓPˆä˜7“|�n xÑ/ˆØ'2Ñ$ˆ
�A�v˜uØ˜‘
× Ñ ˆØ˜‘×"Ñ"ˆÜŸ™ [¸ÀfÔMˆÜŸ
™
 J°¸Ð#>ÄeÇjÁjÐY_Ô`ˆä36°wÀÐP]Ó3^ò 	GÑ/ˆF�M <ØÐ+˜FŸL™L¨™OÐ+Ð->¨v¯|©|¸A©Ð->Ð@QÀ&Ç,Á,ÈqÁ/Ð@QÐQÑR×XÑXÐY_Ô`ØAFˆLÐ*˜6Ÿ<™<¨™?Ð*Ð,=¨f¯l©l¸1©oÐ,=Ð=Ò>ð	Gð ˜}Ð,Ð,ùò &Ps   �D+r  r3  rB  c           	      óÐ  — |}|j                   \  }}}t        j                  | j                  ¬«      }| j	                  |«      }	t        j                  t        ||«      D �
��cg c]  \  }
\  }}|
|   ‘Œ c}}}
«      }t        j                  ||f| j                  t
        j                  |j                  ¬«      }|||	<   |j                  dd«      } |||«      }d|i}|S c c}}}
w )a…  Compute the losses related to the labels using cross entropy.

        Args:
            class_queries_logits (`torch.Tensor`):
                A tensor of shape `batch_size, num_queries, num_labels`
            class_labels (`list[torch.Tensor]`):
                List of class labels of shape `(labels)`.
            indices (`tuple[np.array])`:
                The indices computed by the Hungarian matcher.

        Returns:
            `dict[str, Tensor]`: A dict of `torch.Tensor` containing the following key:
            - **loss_cross_entropy** -- The loss computed using cross entropy on the predicted and ground truth labels.
        )r=   )Ú
fill_valuer8   rd   r   r9   Úloss_cross_entropy)r:   r   ÚCrossEntropyLossr\  Ú$_get_predictions_permutation_indicesrF   rl   rs  Úfullr]  rA  rd   r?   )r.   r  r3  rB  Úpred_logitsrm   Únum_queriesrø   r%  ÚidxÚtargetrK  Útarget_classes_oÚtarget_classesÚpred_logits_transposedÚloss_ceÚlossess                    r1   Úloss_labelszVideomtLoss.loss_labelsª  sâ   € ð" +ˆØ%0×%6Ñ%6Ñ"ˆ
�K Ü×'Ñ'¨t×/@Ñ/@ÔAˆ	Ø×7Ñ7¸Ó@ˆÜ Ÿ9™9Ü-0°¸wÓ-G×HÐH™>˜6¡6 A qˆV�A‹YÔHó
Ðô Ÿ™Ø˜Ð%°$·/±/ÌÏÉÐ]h×]oÑ]oô
ˆð /ˆ�sÑà!,×!6Ñ!6°q¸!Ó!<ÐÙÐ2°NÓCˆØ&¨Ð0ˆØˆùô Is   Á#C!r  r2  rO  c                 ó  ‡ — ‰ j                  |«      }‰ j                  |«      }||   }‰ j                  |«      \  }}	||   }|dd…df   }|dd…df   }t        j                  «       5  ‰ j                  |ˆ fd„‰ j                  ‰ j                  ‰ j                  «      }
t        ||
d¬«      j                  d«      }ddd«       t        |
d¬«      j                  d«      }t        ||«      t        |||«      dœ}~~|S # 1 sw Y   ŒExY w)a¤  Compute the losses related to the masks using sigmoid_cross_entropy_loss and dice loss.

        Args:
            masks_queries_logits (`torch.Tensor`):
                A tensor of shape `(batch_size, num_queries, height, width)`.
            mask_labels (`torch.Tensor`):
                List of mask labels of shape `(labels, height, width)`.
            indices (`tuple[np.array])`:
                The indices computed by the Hungarian matcher.
            num_masks (`int)`:
                The number of masks, used for normalization.

        Returns:
            losses (`dict[str, Tensor]`): A dict of `torch.Tensor` containing two keys:
            - **loss_mask** -- The loss computed using sigmoid cross entropy loss on the predicted and ground truth.
              masks.
            - **loss_dice** -- The loss computed using dice loss on the predicted on the predicted and ground truth,
              masks.
        Nc                 ó&   •— ‰j                  | «      S rs   )Úcalculate_uncertainty)Úlogitsr.   s    €r1   ú<lambda>z(VideomtLoss.loss_masks.<locals>.<lambda>÷  s   ø€ ˜t×9Ñ9¸&ÓA€ r2   Fr6  r   )Ú	loss_maskÚ	loss_dice)r€  Ú _get_targets_permutation_indicesr{  rF   rM  Úsample_points_using_uncertaintyr0  ra  rb  r  r  rV  rR  )r.   r  r2  rB  rO  Úsrc_idxÚtgt_idxÚ
pred_masksÚtarget_masksrø   r  Úpoint_labelsÚpoint_logitsrŠ  s   `             r1   Ú
loss_maskszVideomtLoss.loss_masksÌ  s,  ø€ ð4 ×;Ñ;¸GÓDˆØ×7Ñ7¸Ó@ˆà)¨'Ñ2ˆ
ð ×:Ñ:¸;ÓG‰ˆ�aØ# GÑ,ˆð  ¢ 4 Ñ(ˆ
Ø#¢A t GÑ,ˆô �]‰]‹_ñ 		iØ $× DÑ DØÛAØ—‘Ø×%Ñ%Ø×,Ñ,ó!Ðô (¨Ð6GÐW\Ô]×eÑeÐfgÓhˆL÷		iô $ JÐ0AÐQVÔW×_Ñ_Ð`aÓbˆô 4°LÀ,ÐPYÓZÜ" <°¸yÓIñ
ˆð
 ØØˆ÷)		ið 		iús   Á(AD Ä D	c                 ó  — t        j                  t        |«      D ���cg c]  \  }\  }}t        j                  ||«      ‘Œ  c}}}«      }t        j                  |D ��cg c]  \  }}|‘Œ	 c}}«      }||fS c c}}}w c c}}w rs   ©rF   rl   ri  Ú	full_like)r.   rB  rC  Úsrcrø   Úbatch_indicesÚpredictions_indicess          r1   r€  z0VideomtLoss._get_predictions_permutation_indices
  sj   € äŸ	™	ÌiÐX_ÓN`×"aÐ"a¹{¸qÁ(À3È¤5§?¡?°3¸Õ#:Ô"aÓbˆÜ#Ÿi™i¸W×(E±°#°qªÓ(EÓFÐØÐ1Ð1Ð1ùô #bùÛ(Eó   Ÿ#A7ÁA>
c                 ó  — t        j                  t        |«      D ���cg c]  \  }\  }}t        j                  ||«      ‘Œ  c}}}«      }t        j                  |D ��cg c]  \  }}|‘Œ	 c}}«      }||fS c c}}}w c c}}w rs   r�  )r.   rB  rC  rø   Útgtr   Útarget_indicess          r1   r“  z,VideomtLoss._get_targets_permutation_indices  sh   € äŸ	™	ÌiÐX_ÓN`×"aÐ"a¹{¸qÁ(À1Àc¤5§?¡?°3¸Õ#:Ô"aÓbˆÜŸ™¸×#@©H¨Q°¢CÓ#@ÓAˆØ˜nÐ,Ð,ùô #bùÛ#@r¢  r�  c                 ó2   — t        j                  |«       }|S )a…  
        In Videomt paper, uncertainty is estimated as L1 distance between 0.0 and the logit prediction in 'logits'
        for the foreground class in `classes`.

        Args:
            logits (`torch.Tensor`):
            A tensor of shape (R, 1, ...) for class-specific or class-agnostic, where R is the total number of predicted masks in all images and C is:
            the number of foreground classes. The values are logits.

        Returns:
            scores (`torch.Tensor`): A tensor of shape (R, 1, ...) that contains uncertainty scores with the most
            uncertain locations having the highest uncertainty score.
        )rF   Úabs)r.   r�  Úuncertainty_scoress      r1   rŽ  z!VideomtLoss.calculate_uncertainty  s   € ô  %Ÿy™y¨Ó0Ð1ÐØ!Ð!r2   r0  ra  rb  c           	      ó�  — |j                   d   }t        ||z  «      }t        j                  ||d|j                  ¬«      }t        ||d¬«      }	 ||	«      }
t        ||z  «      }||z
  }t        j                  |
dd…ddd…f   |d¬«      d   }|t        j                  |t        j                  |j                  ¬	«      z  }||dd…df   z  }|j                  d
d«      |j                  d
«      dd…f   j                  ||d«      }|dkD  r:t        j                  |t        j                  ||d|j                  ¬«      gd¬«      }|S )a€  
        This function is meant for sampling points in [0, 1] * [0, 1] coordinate space based on their uncertainty. The
        uncertainty is calculated for each point using the passed `uncertainty function` that takes points logit
        prediction as input.

        Args:
            logits (`float`):
                Logit predictions for P points.
            uncertainty_function:
                A function that takes logit predictions for P points and returns their uncertainties.
            num_points (`int`):
                The number of points P to sample.
            oversample_ratio (`int`):
                Oversampling parameter.
            importance_sample_ratio (`float`):
                Ratio of points that are sampled via importance sampling.

        Returns:
            point_coordinates (`torch.Tensor`):
                Coordinates for P sampled points.
        r   r9   r5  Fr6  Nr   )Úkrf   rØ   rN   re   )r:   rz   rF   rÙ   rd   r  Útopkr^   ÚlongrÅ   rl   )r.   r�  Úuncertainty_functionr0  ra  rb  Ú	num_boxesÚnum_points_sampledr  rš  Úpoint_uncertaintiesÚnum_uncertain_pointsÚnum_random_pointsr„  Úshifts                  r1   r”  z+VideomtLoss.sample_points_using_uncertainty'  sI  € ð< —L‘L ‘Oˆ	Ü  Ð.>Ñ!>Ó?Ðô "ŸJ™J yÐ2DÀaÐPV×P]ÑP]Ô^Ðä# FÐ,=ÈUÔSˆá2°<Ó@Ðä"Ð#:¸ZÑ#GÓHÐØ&Ð)=Ñ=Ðä�j‰jÐ,ªQ°²1¨WÑ5Ð9MÐSTÔUÐVWÑXˆØ"¤U§\¡\°)Ä5Ç:Á:ÐV\×VcÑVcÔ%dÑdˆØˆu’Q˜�W‰~ÑˆØ-×2Ñ2°2°qÓ9¸#¿(¹(À2»,Ê¸/ÑJ×OÑOÐPYÐ[oÐqrÓsÐà˜qÒ Ü %§	¡	Ø"¤E§J¡J¨yÐ:KÈQÐW]×WdÑWdÔ$eÐfØô!Ðð !Ð r2   NÚauxiliary_predictionsc                 ó¤  — | j                  ||||«      }| j                  ||d   j                  ¬«      }i | j                  ||||«      ¥| j	                  |||«      ¥}|�jt        |«      D ]\  \  }	}
|
d   }|
d   }| j                  ||||«      }|j                  «       D ��ci c]  \  }}|› d|	› �|“Œ }}}|j                  |«       Œ^ |S c c}}w )a»  
        This performs the loss computation.

        Args:
            masks_queries_logits (`torch.Tensor`):
                A tensor of shape `(batch_size, num_queries, height, width)`.
            class_queries_logits (`torch.Tensor`):
                A tensor of shape `(batch_size, num_queries, num_labels)`.
            mask_labels (`torch.Tensor`):
                List of mask labels of shape `(labels, height, width)`.
            class_labels (`list[torch.Tensor]`):
                List of class labels of shape `(labels)`.
            auxiliary_predictions (`dict[str, torch.Tensor]`, *optional*):
                if `use_auxiliary_loss` was set to `true` in [`VideomtConfig`], then it contains the logits from
                the inner layers of the VideomtMaskedAttentionDecoder.

        Returns:
            losses (`dict[str, Tensor]`): A dict of `torch.Tensor` containing three keys:
            - **loss_cross_entropy** -- The loss computed using cross entropy on the predicted and ground truth labels.
            - **loss_mask** -- The loss computed using sigmoid cross_entropy loss on the predicted and ground truth
              masks.
            - **loss_dice** -- The loss computed using dice loss on the predicted on the predicted and ground truth
              masks.
            if `use_auxiliary_loss` was set to `true` in [`VideomtConfig`], the dictionary contains additional
            losses for each auxiliary predictions.
        r   r5  r  r  rø   )	rf  Úget_num_masksrd   r›  r‹  ri  rA   ÚitemsÚupdate)r.   r  r  r2  r3  r´  rB  rO  rŠ  r„  Úaux_outputsÚ	loss_dictr£   r¤   s                 r1   rA   zVideomtLoss.forward^  s
  € ðH —,‘,Ð3Ð5IÈ;ÐXdÓeˆà×&Ñ& |¸LÈ¹O×<RÑ<RÐ&ÓSˆ	ð%
Ø�o‰oÐ2°KÀÈ)ÓTð%
à×ÑÐ3°\À7ÓKð%
ˆð
 !Ð,Ü$-Ð.CÓ$Dò )Ñ ��[Ø'2Ð3IÑ'JÐ$Ø'2Ð3IÑ'JÐ$Ø ŸL™LÐ)=Ð?SÐU`ÐbnÓo�	ØENÇ_Á_ÓEV×W±z°s¸E ˜u A c U˜^¨UÑ2ÐW�	ÑWØ—‘˜iÕ(ð)ð ˆùó Xs   Â"Crd   c                 ó&  — t        d„ |D «       «      }t        j                  |t        j                  |¬«      }d}t	        «       r2t
        j                  i k7  rt        |«      }t        «       j                  }t        j                  ||z  d¬«      }|S )zk
        Computes the average number of target masks across the batch, for normalization purposes.
        c              3   ó2   K  — | ]  }t        |«      –— Œ y ­wrs   )rr  )Ú.0Úclassess     r1   ú	<genexpr>z,VideomtLoss.get_num_masks.<locals>.<genexpr>™  s   è ø€ ÒA¨œ˜GŸÑAùs   ‚rØ   r   )Úmin)
r  rF   r@  râ   r   r   Ú_shared_stater   Únum_processesÚclamp)r.   r3  rd   rO  Ú
world_sizes        r1   r¶  zVideomtLoss.get_num_masks•  su   € ô ÑA°LÔAÓAˆ	Ü—O‘O I´U·[±[ÈÔPˆ	Øˆ
Ü"Ô$Ü×)Ñ)¨RÒ/Ü" 9Ó-�	Ü)›^×9Ñ9�
ä—K‘K 	¨JÑ 6¸AÔ>ˆ	ØÐr2   rs   )rB   rC   rD   r   Údictr   râ   r"   rN  rz   ro  r   rÎ   r{  ÚnpÚarrayr‹  rF   r›  r€  r“  rŽ  r”  rA   rd   r¶  rG   rH   s   @r1   rX  rX  m  s  ø„ ð!
˜}ð !
¸4ÀÀUÀ
Ñ;Kõ !
ðF $ t¨C¡y¡/ð °d¸3±ió ð-°4¸±<ð -ÀEÈ&ÐRXÈ.ÑDYó -ð" Ø$*ð Ø:>¸v¹,ð ØQVÐWY×W_ÑW_ÑQ`ð à	ˆc�6ˆkÑ	ó ðD<à#Ÿl™lð<ð ˜%Ÿ,™,Ñ'ð<ð �r—x‘x‘ð	<ð
 ð<ð 
ˆc�5—<‘<ÐÑ	 ó<ò|2ò-ð"¨E¯L©Lð "¸U¿\¹\ó "ð"5!à—‘ð5!ð ð	5!ð
 ð5!ð "'ð5!ð 
�‰ó5!ðz AEñ5à#Ÿl™lð5ð $Ÿl™lð5ð ˜%Ÿ,™,Ñ'ð	5ð
 ˜5Ÿ<™<Ñ(ð5ð  $ C¨¯©Ð$5Ñ6¸Ñ=ð5ð 
ˆc�5—<‘<ÐÑ	 ó5ðn¨%¯,©,ð ÀÇÁð ÐQV×Q]ÑQ]÷ r2   rX  c                   ó˜   ‡ — e Zd ZU dZeed<   dZdZdZdZ	dgZ
dZeed	œZ ej                   «       d
ej$                  ddfˆ fd„«       Zˆ xZS )ÚVideomtPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    r/   ÚvideomtÚpixel_values_videos)ÚvideoFrä   T)rÂ   r  r¡   r4   Nc                 óæ  •— t         ‰| �  |«       | j                  j                  }t	        |t
        j                  t
        j                  t
        j                  f«      rºt        j                  |j                  t        j                  d«      ¬«       |j                  ��—t        j
                  j                  j!                  |j                  «      \  }}|dkD  rdt        j                  |«      z  nd}t        j"                  |j                  | |«       �nt	        |t
        j$                  «      rut        j&                  |j                  dd¬«       |j(                  ��Õt+        |j                  dd«      �s½t        j,                  |j                  |j(                     «       �n�t	        |t.        «      rCt1        |d	«      �rrt        j2                  |j4                  | j                  j6                  «       �n<t	        |t8        «      r›t        j:                  |j<                  d|¬«       t        j,                  |j>                  «       t        j@                  |jB                  t        jD                  |jB                  jF                  d
   «      jI                  d«      «       n‘t	        |tJ        «      rRt        jL                  |jN                  dz   «      }|jP                  |d
<   t        j@                  |jR                  |«       n/t	        |tT        «      rt        jV                  |jX                  «       t	        |t8        «      r*t
        j                  j-                  |jZ                  «       y y )Nrc   )Úar   r   rÄ   )rT  ÚstdÚ_is_hf_initializedFrþ   rN   rM   ).r!   Ú_init_weightsr/   Úinitializer_ranger'   r   r|   r,   ÚConvTranspose2dÚinitÚkaiming_uniform_r=   ÚmathÚsqrtry   rF   Ú_calculate_fan_in_and_fan_outÚuniform_r[   Únormal_Úpadding_idxÚgetattrÚzeros_rì   ÚhasattrÚ	constant_rþ   rü   rJ   Útrunc_normal_rR   rU   rt  rL   r^   r:   r_   rX  rý   r]  r_  r\  ÚVideomtForUniversalSegmentationÚones_Úattn_mask_probsr`   )r.   r¡   rÏ  Úfan_inrø   Úboundr\  r0   s          €r1   rÑ  z$VideomtPreTrainedModel._init_weights¸  s8  ø€ ä‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fœrŸy™y¬"¯)©)´R×5GÑ5GÐHÔIÜ×!Ñ! &§-¡-´4·9±9¸Q³<Õ@Ø�{‰{Ñ&Ü!ŸH™HŸM™M×GÑGÈÏÉÓV‘	�˜Ø17¸!²˜œDŸI™I fÓ-Ò-À�Ü—‘˜fŸk™k¨E¨6°5Ö9Ü˜¤§¡Ô-Ü�L‰L˜Ÿ™¨S°aÕ8à×!Ñ!Ñ-´g¸f¿m¹mÐMaÐchÕ6iÜ—‘˜FŸM™M¨&×*<Ñ*<Ñ=Ö>Ü˜Ô 1Ô2Ü�v˜yÕ)Ü—‘˜vŸ~™~¨t¯{©{×/KÑ/KÖLÜ˜Ô 1Ô2Ü×Ñ˜v×/Ñ/°c¸sÕCÜ�K‰K˜×.Ñ.Ô/Ü�J‰J�v×*Ñ*¬E¯L©L¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÕhÜ˜¤Ô,Ü Ÿ:™: f×&7Ñ&7¸!Ñ&;Ó<ˆLØ%Ÿ™ˆL˜ÑÜ�J‰J�v×*Ñ*¨LÕ9Ü˜Ô ?Ô@Ü�J‰J�v×-Ñ-Ô.Ü�fÔ/Ô0Ü�G‰G�N‰N˜6×,Ñ,Õ-ð 1r2   )rB   rC   rD   rE   r   r	  Úbase_model_prefixÚmain_input_nameÚinput_modalitiesÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_sdparä   r´   Ú_can_record_outputsrF   rM  r   ÚModulerÑ  rG   rH   s   @r1   rÉ  rÉ  ¥  sq   ø… ñð
 ÓØ!ÐØ+€OØ!ÐØ&+Ð#Ø'Ð(ÐØ€Nà%Ø&ñÐð
 €U‡]�]ƒ_ð. B§I¡Ið .°$ô .ó ô.r2   rÉ  c                   óX   ‡ — e Zd Zdˆ fd„	Zdej
                  dej
                  fd„Zˆ xZS )ÚVideomtLayerNorm2dc                 ó*   •— t         ‰| �  |||¬«       y )N)rç   Úelementwise_affine)r!   r"   )r.   r%   rç   Úaffiner0   s       €r1   r"   zVideomtLayerNorm2d.__init__Ù  s   ø€ Ü‰Ñ˜¨3À6ÐÕJr2   r†   r4   c                 óØ   — |j                  dddd«      }t        j                  || j                  | j                  | j
                  | j                  «      }|j                  dddd«      }|S )Nr   r9   r   r   )ÚpermuteÚFÚ
layer_normÚnormalized_shaper=   ry   rç   rˆ   s     r1   rA   zVideomtLayerNorm2d.forwardÜ  sb   € Ø#×+Ñ+¨A¨q°!°QÓ7ˆÜ—|‘| L°$×2GÑ2GÈÏÉÐVZ×V_ÑV_Ðae×aiÑaiÓjˆØ#×+Ñ+¨A¨q°!°QÓ7ˆØÐr2   )g�íµ ÷Æ°>TrŠ   rH   s   @r1   rï  rï  Ø  s$   ø„ õKð E§L¡Lð °U·\±\÷ r2   rï  c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚVideomtScaleLayerr/   c                 ó  •— t         ‰| �  «        |j                  }t        j                  ||dd¬«      | _        t        |j                     | _        t        j                  ||dd|d¬«      | _
        t        |«      | _        y )Nr9   r   r   r   F)r   ÚpaddingÚgroupsry   )r!   r"   r&   r   rÓ  Úconv1r	   r~   r€   r,   Úconv2rï  Úlayernorm2d©r.   r/   r&   r0   s      €r1   r"   zVideomtScaleLayer.__init__ä  su   ø€ Ü‰ÑÔØ×(Ñ(ˆÜ×'Ñ'¨°[ÈaÐXYÔZˆŒ
Ü  ×!2Ñ!2Ñ3ˆŒÜ—Y‘YØØØØØØô
ˆŒ
ô .¨kÓ:ˆÕr2   rÂ   r4   c                 óŽ   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|S rs   )rý  r€   rþ  rÿ  ©r.   rÂ   s     r1   rA   zVideomtScaleLayer.forwardô  sB   € ØŸ
™
 =Ó1ˆØŸ™¨Ó6ˆØŸ
™
 =Ó1ˆØ×(Ñ(¨Ó7ˆØÐr2   ©	rB   rC   rD   r   r"   rF   r   rA   rG   rH   s   @r1   rù  rù  ã  s*   ø„ ð;˜}õ ;ð  U§\¡\ð °e·l±l÷ r2   rù  c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚVideomtScaleBlockr/   c                 óÖ   •— t         ‰| �  «        |j                  | _        t	        j
                  t        | j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        y c c}w rs   )	r!   r"   Únum_upscale_blocksÚ
num_blocksr   Ú
ModuleListr8  rù  Úblock©r.   r/   rø   r0   s      €r1   r"   zVideomtScaleBlock.__init__ý  sH   ø€ Ü‰ÑÔØ ×3Ñ3ˆŒÜ—]‘]ÄuÈTÏ_É_ÓG]Ö#^À!Ô$5°fÕ$=Ò#^Ó_ˆ�
ùÒ#^s   ÁA&rÂ   r4   c                 ó8   — | j                   D ]
  } ||«      }Œ |S rs   )r
  )r.   rÂ   r
  s      r1   rA   zVideomtScaleBlock.forward  s%   € Ø—Z‘Zò 	1ˆEÙ! -Ó0‰Mð	1àÐr2   r  rH   s   @r1   r  r  ü  s,   ø„ ð`˜}õ `ð
 U§\¡\ð °e·l±l÷ r2   r  c                   ó\   ‡ — e Zd Zdefˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚVideomtMaskHeadr/   c                 ó  •— t         ‰| �  «        |j                  }t        j                  ||«      | _        t        j                  ||«      | _        t        j                  ||«      | _        t        |j                     | _
        y rs   )r!   r"   r&   r   r|   r}   r�   Úfc3r	   r~   r€   r   s      €r1   r"   zVideomtMaskHead.__init__	  sa   ø€ Ü‰ÑÔà×(Ñ(ˆÜ—9‘9˜[¨+Ó6ˆŒÜ—9‘9˜[¨+Ó6ˆŒÜ—9‘9˜[¨+Ó6ˆŒÜ  ×!2Ñ!2Ñ3ˆ�r2   rÂ   r4   c                 ó¨   — | j                  | j                  |«      «      }| j                  | j                  |«      «      }| j                  |«      }|S rs   )r€   r}   r�   r  r  s     r1   rA   zVideomtMaskHead.forward  sD   € ØŸ™¨¯©°Ó(?Ó@ˆØŸ™¨¯©°Ó(?Ó@ˆØŸ™ Ó/ˆØÐr2   r  rH   s   @r1   r  r    s*   ø„ ð4˜}õ 4ð U§\¡\ð °e·l±l÷ r2   r  zY
    The Videomt Model with head on top for instance/semantic/panoptic segmentation.
    c                   ód  ‡ — e Zd ZdZdefˆ fd„Zdededededeeef   d	eeef   fd
„Z	deeef   d	efd„Z
eee	 	 	 	 ddej                  dz  deej                     dz  deej                     dz  deej                     dz  dee   d	efd„«       «       «       Zd„ Zdej                  fd„Zˆ xZS )rá  rË  r/   c                 ó<  •— t         ‰| �  |«       || _        |j                  | _        t	        |«      | _        t        j                  |j                  |j                  ¬«      | _
        t        j                  |j                  |j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t!        |«      ‘Œ c}«      | _        t%        |«      | _        t)        |«      | _        t        j,                  |j                  |j.                  dz   «      | _        |j2                  |j4                  z  |j2                  |j4                  z  f| _        |j8                  |j:                  |j<                  dœ| _        tA        || j>                  ¬«      | _!        | jE                  dtG        jH                  |jJ                  «      «       t        j,                  |j                  |j                  «      | _&        | jO                  «        y c c}w )Nræ   r   )r~  r‘  r’  )r/   rY  rã  )(r!   r"   r/   Únum_hidden_layersrJ   r@   r   rè   r&   ré   Ú	layernormr[   rƒ  r¢   r	  r8  rä   Úlayersr  Úupscale_blockr  Ú	mask_headr|   r]  Úclass_predictorr#   r$   Ú	grid_sizerc  re  rd  rY  rX  r%  r]   rF   rý   r  Úquery_updaterÚ	post_initr  s      €r1   r"   z(VideomtForUniversalSegmentation.__init__!  s�  ø€ Ü‰Ñ˜Ô ØˆŒØ!'×!9Ñ!9ˆÔÜ+¨FÓ3ˆŒÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒä—\‘\ &×"4Ñ"4°f×6HÑ6HÓIˆŒ
Ü—m‘mÄ5È×IaÑIaÓCbÖ$c¸a¤\°&Õ%9Ò$cÓdˆŒä.¨vÓ6ˆÔÜ(¨Ó0ˆŒä!Ÿy™y¨×);Ñ);¸V×=NÑ=NÐQRÑ=RÓSˆÔà ×+Ñ+¨v×/@Ñ/@Ñ@À&×BSÑBSÐW]×WhÑWhÑBhÐiˆŒà"(×"5Ñ"5Ø×+Ñ+Ø×+Ñ+ñ.
ˆÔô %¨FÀ×@PÑ@PÔQˆŒà×ÑÐ.´·
±
¸6×;LÑ;LÓ0MÔNÜŸY™Y v×'9Ñ'9¸6×;MÑ;MÓNˆÔà�‰Õùò' %ds   Â>Hr  r  r2  r3  r´  r4   c                 óº   — | j                  |||||¬«      }| j                  j                  «       D ]'  \  }}|j                  «       D ]  \  }	}
||	v sŒ|
|z  }
Œ Œ) |S )N)r  r  r2  r3  r´  )r%  rY  r·  )r.   r  r  r2  r3  r´  rº  r£   r=   Úloss_keyr  s              r1   Úget_loss_dictz-VideomtForUniversalSegmentation.get_loss_dict>  s|   € ð (,§~¡~Ø!5Ø!5Ø#Ø%Ø"7ð (6ó (
ˆ	ð  ×+Ñ+×1Ñ1Ó3ò 	#‰KˆC�Ø"+§/¡/Ó"3ò #‘�˜$Ø˜(’?Ø˜F‘N‘Dñ#ð	#ð
 Ðr2   rº  c                 ó4   — t        |j                  «       «      S rs   )r  rÌ   )r.   rº  s     r1   Úget_lossz(VideomtForUniversalSegmentation.get_lossV  s   € Ü�9×#Ñ#Ó%Ó&Ð&r2   NÚpatch_offsetsr¯   c           	      ó|  — d|v rt        d«      ‚|€t        d«      ‚|j                  dk7  rt        d«      ‚|€|�t        d«      ‚|j                  \  }}}}	}
|j                  ||z  ||	|
«      }| j	                  |«      }| j
                  | j                  j                  z
  }| j                  d| D ]
  } ||«      }Œ |j                  |||j                  d   |j                  d	   «      }g }g }g }d}t        |«      D �]l  }|dd…|f   }|€K| j                  j                  ddd…dd…f   j                  |d
d
«      j                  |j                  «      }nd| j!                  |«      j                  |j                  «      | j                  j                  ddd…dd…f   j                  |j                  «      z   }t#        j$                  ||fd¬«      }| j                  |d D ]
  } ||«      }Œ | j'                  |«      }| j)                  |«      \  }}|j+                  |«       |j+                  |«       |j+                  |«       |dd…d| j                  j,                  …dd…f   }�Œo t/        dt#        j$                  |d¬«      t#        j$                  |d¬«      t#        j$                  |d¬«      ¬«      S )aø  
        pixel_values_videos (`torch.Tensor`, *optional*):
            Video inputs of shape `(batch_size, num_frames, num_channels, height, width)`.
        mask_labels (`list[torch.Tensor]`, *optional*):
            Not supported for 5D video inputs.
        class_labels (`list[torch.LongTensor]`, *optional*):
            Not supported for 5D video inputs.
        patch_offsets (`list[torch.Tensor]`, *optional*):
            Unused for video inputs and only kept for modular compatibility.
        r3   zAUse `pixel_values_videos` with `VideomtForUniversalSegmentation`.Nz'You have to specify pixel_values_videosrc   zyVideomtForUniversalSegmentation only supports 5D video inputs of shape (batch_size, num_frames, channels, height, width).z“Training with 5D video inputs is not supported in `VideomtForUniversalSegmentation`. Flatten frames and use `EomtForUniversalSegmentation` instead.r   r9   rN   re   r   )r  r  r  r  )r;   rg   r:   rh   r@   r  r/   r  r  rÅ   r8  r¢   r=   r_   r<   rd   r  rF   rl   r  Úpredictr?  rƒ  r  )r.   rË  r2  r3  r"  r¯   rm   rn   r%   ro   rp   Úflat_pixel_valuesrÂ   Úquery_start_idxÚlayer_moduleÚall_masks_queries_logitsÚall_class_queries_logitsÚall_last_hidden_statesÚpropagated_queryÚ	frame_idxÚframe_hidden_statesÚquery_tokensÚsequence_outputr  r  s                            r1   rA   z'VideomtForUniversalSegmentation.forwardY  sð  € ð* ˜VÑ#ÜÐ`ÓaÐaàÐ&ÜÐFÓGÐGà×#Ñ# qÒ(ÜðEóð ð
 Ð" lÐ&>ÜðQóð ð
 ?R×>WÑ>WÑ;ˆ
�J ¨f°eØ/×7Ñ7¸
ÀZÑ8OÐQ]Ð_eÐglÓmÐàŸ™Ð(9Ó:ˆØ×0Ñ0°4·;±;×3IÑ3IÑIˆà ŸK™KÐ(8¨Ð9ò 	8ˆLÙ(¨Ó7‰Mð	8ð &×*Ñ*¨:°zÀ=×CVÑCVÐWXÑCYÐ[h×[nÑ[nÐopÑ[qÓrˆà#%Ð Ø#%Ð Ø!#ÐØÐä˜zÓ*ó 	TˆIØ"/²°9°Ñ"=ÐàÐ'Ø#Ÿz™z×0Ñ0°²qº!°Ñ<×CÑCÀJÐPRÐTVÓW×ZÑZÐ[n×[uÑ[uÓv‘à#×1Ñ1Ð2BÓC×FÑFÐGZ×GaÑGaÓbÐei×eoÑeo×evÑevØš!šQ�Jñfç‘"Ð(×/Ñ/Ó0ñ 1�ô #(§)¡)¨\Ð;NÐ,OÐUVÔ"WÐà $§¡¨OÐ,<Ð =ò H�Ù&2Ð3FÓ&GÑ#ðHð #Ÿn™nÐ-@ÓAˆOØ9=¿¹ÀoÓ9VÑ6Ð Ð"6à$×+Ñ+Ð,@ÔAØ$×+Ñ+Ð,@ÔAØ"×)Ñ)¨/Ô:Ø2²1Ð6O¸¿¹×8OÑ8OÐ6OÒQRÐ3RÑSÒð)	Tô, 5ØÜ!&§¡Ð+CÈÔ!KÜ!&§¡Ð+CÈÔ!KÜ#Ÿi™iÐ(>ÀAÔFô	
ð 	
r2   c                 ó.   — | j                   j                  S rs   )r@   rV   rß   s    r1   Úget_input_embeddingsz4VideomtForUniversalSegmentation.get_input_embeddings­  s   € Ø�‰×/Ñ/Ð/r2   r�  c                 óÎ  — |d d …d | j                   j                  …d d …f   }| j                  |«      }|d d …| j                   j                  | j                  j                  z   d …d d …f   }|j                  dd«      } |j                  |j                  d   dg| j                  ¢­Ž }| j                  |«      }| j                  |«      }t        j                  d||«      }||fS )Nr   r9   r   rN   zbqc, bchw -> bqhw)r/   rƒ  r  r@   rZ   r?   rh   r:   r  r  r  rF   Úeinsum)r.   r�  r.  Úclass_logitsÚprefix_tokensÚmask_logitss         r1   r$  z'VideomtForUniversalSegmentation.predict°  s×   € ØšaÐ!: 4§;¡;×#:Ñ#:Ð!:ºAÐ=Ñ>ˆØ×+Ñ+¨LÓ9ˆàšq $§+¡+×"9Ñ"9¸D¿O¹O×<]Ñ<]Ñ"]Ñ"_ÒabÐbÑcˆØ%×/Ñ/°°1Ó5ˆà-˜×-Ñ-¨m×.AÑ.AÀ!Ñ.DÀbÐZÈ4Ï>É>ÒZˆà—~‘~ lÓ3ˆØ×*Ñ*¨=Ó9ˆä—l‘lÐ#6¸ÀmÓTˆà˜LÐ(Ð(r2   )NNNN)rB   rC   rD   rç  r   r"   r   rÅ  r   r  r!  r   r   r   rF   rN  r   r   r  rA   r1  r$  rG   rH   s   @r1   rá  rá    sF  ø„ ð ,€Oð˜}õ ð:à$ðð %ðð ð	ð
 ðð  $ C¨ KÑ0ðð 
ˆc�6ˆkÑ	óð0' $ s¨F {Ñ"3ð '¸ó 'ð  ØØð 48Ø15Ø26Ø37ñO
à"Ÿ\™\¨DÑ0ðO
ð ˜%Ÿ,™,Ñ'¨$Ñ.ðO
ð ˜5Ÿ<™<Ñ(¨4Ñ/ð	O
ð
 ˜EŸL™LÑ)¨DÑ0ðO
ð Ð+Ñ,ðO
ð 
/òO
ó ó ó  ðO
òb0ð)˜eŸl™l÷ )r2   rá  rá   )F)KÚcollections.abcr(   rÖ  r   Údataclassesr   ÚnumpyrÆ  rF   Útorch.nn.functionalr   r™   rõ  r   Ú r   rÔ  Úactivationsr	   Ú
file_utilsr
   r   r   Úmodeling_layersr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   Úconfiguration_videomtr   Úscipy.optimizer   Ú
accelerater   Úaccelerate.utilsr   rí  r   rJ   rv   rŒ   râ   r²   r´   rÐ   rÔ   rä   rì   r  r  r  r*  r,  rz   rR  rV  rX  rÉ  rè   rï  rù  r  r  rá  Ú__all__r
  r2   r1   ú<module>rI     s  ðó* Û Ý $Ý !ã Û ß Ð ß å &Ý !ß LÑ LÝ 9ß FÝ &ß PÑ PÝ 7Ý 5Ý 0ñ ÔÝ4áÔÝ'Ý'ô ˜RŸY™Yô  ôF,˜Ÿ	™	ô ,ô^�—‘ô ô&(�b—i‘iô (ð0 ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ó%ô.8)�r—y‘yô 8)ôv(�r—y‘yô (ô"%�b—i‘iô %ô0'Ð-ô 'ôT+˜Ÿ	™	ô +ñ ðô	ð ô7¨Kó 7ó ó	ð7ð< LQñØ—L‘LðØ5:·\±\ðà
‡\�\óð@ ð °ð ¸6ó ð,°·±ð ÀuÇ|Á|ð ÐX]×XdÑXdó ô8g˜bŸi™iô gðT�fð  fð ¸ð Àó ð< u§|¡|ð ¸U¿\¹\ð ÐVYð Ð^c×^jÑ^jó ô(u�"—)‘)ô uðp	 ô/.˜_ó /.ó ð/.ôd˜Ÿ™ô ô˜Ÿ	™	ô ô2	˜Ÿ	™	ô 	ô�b—i‘iô ñ" ðôô
`)Ð&<ó `)óð
`)ðF $Ð%FÐ
G�r2   